Forgejo does not support build cache

https://github.com/matrix-org/sytest/pull/1383 updates some sytests in
line with MSC3967. Dendrite does not support MSC3967, so these tests
fail.

.dockerignore

@@ -1,3 +1,2 @@
 bin
-*.wasm
-.git
+*.wasm

.forgejo/workflows/docker.yml

@@ -0,0 +1,59 @@
+on:
+  push:
+    tags:
+      - 'v*'
+
+env:
+  GHCR_NAMESPACE: sigb.us
+  PLATFORMS: linux/amd64
+  FORGEJO_USER: signaryk
+
+jobs:
+  monolith:
+    name: Monolith image
+    runs-on: docker
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Get release tag & build flags
+        if: github.event_name == 'release' # Only for GitHub releases
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to sigb.us container registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.sigb.us
+          username: ${{ env.FORGEJO_USER }}
+          password: ${{ secrets.FORGEJO_TOKEN }}
+
+      - name: Build main monolith image
+        id: docker_build_monolith
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ github.ref_name }}
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:devel
+
+      - name: Build release monolith image
+        if: github.event_name == 'release' # Only for GitHub releases
+        id: docker_build_monolith_release
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:stable
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ env.RELEASE_VERSION }}

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -62,6 +62,6 @@ If you can identify any relevant log snippets from server logs, please include
 those (please be careful to remove any personal or private data). Please surround them with
 ``` (three backticks, on a line on their own), so that they are formatted legibly.
 
-Alternatively, please send logs to @kegan:matrix.org or @neilalexander:matrix.org
+Alternatively, please send logs to @kegan:matrix.org, @s7evink:matrix.org or @devonh:one.ems.host
 with a link to the respective Github issue, thanks!
 -->

.github/codecov.yaml

@@ -7,7 +7,7 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 0%
+        threshold: 0.1%
         base: auto
         flags:
           - unittests

.github/workflows/dendrite.yml

@@ -28,10 +28,10 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ false }} # disable for now
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
@@ -41,7 +41,7 @@ jobs:
         with:
           node-version: 14
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -66,11 +66,11 @@ jobs:
     name: Linting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
       - name: golangci-lint
@@ -102,14 +102,14 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
         with:
           path: |
@@ -123,7 +123,7 @@ jobs:
         with:
           # Optional: pass GITHUB_TOKEN to avoid rate limiting.
           token: ${{ secrets.GITHUB_TOKEN }}
-      - run: go test -json -v ./... 2>&1 | gotestfmt
+      - run: go test -json -v ./... 2>&1 | gotestfmt -hide all
         env:
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
@@ -141,12 +141,12 @@ jobs:
         goos: ["linux"]
         goarch: ["amd64", "386"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -174,12 +174,12 @@ jobs:
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -235,11 +235,11 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
       - name: Set up gotestfmt
@@ -247,7 +247,7 @@ jobs:
         with:
           # Optional: pass GITHUB_TOKEN to avoid rate limiting.
           token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -255,17 +255,18 @@ jobs:
           key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-stable-test-race-
-      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
+      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all
         env:
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: dendrite
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           flags: unittests
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # run database upgrade tests
   upgrade_test:
@@ -274,12 +275,22 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade-test-
+      - name: Docker version
+        run: docker version
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
       - name: Test upgrade (PostgreSQL)
@@ -294,12 +305,22 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade-direct-test-
+      - name: Docker version
+        run: docker version
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
       - name: Test upgrade (PostgreSQL)
@@ -336,8 +357,8 @@ jobs:
         SYTEST_BRANCH: ${{ github.head_ref }}
         CGO_ENABLED: ${{ matrix.cgo && 1 }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -360,7 +381,7 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
@@ -400,8 +421,8 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v3 for dendrite
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v4 for dendrite
+        uses: actions/checkout@v4
         with:
           path: dendrite
 
@@ -436,7 +457,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:

.github/workflows/docker.yml

@@ -27,26 +27,22 @@ jobs:
       security-events: write # To upload Trivy sarif files
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -57,10 +53,9 @@ jobs:
         id: docker_build_monolith
         uses: docker/build-push-action@v3
         with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache
+          cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -75,7 +70,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -104,26 +98,22 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -137,7 +127,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -153,7 +142,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -171,26 +159,22 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -204,7 +188,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -220,7 +203,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true

.github/workflows/gh-pages.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Build with Jekyll

.github/workflows/helm.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -32,9 +32,10 @@ jobs:
           version: v3.10.0
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.4.1
+        uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         with:
           config: helm/cr.yaml
           charts_dir: helm/
+          mark_as_latest: false

.github/workflows/k8s.yml

@@ -17,7 +17,7 @@ jobs:
     outputs:
       changed: ${{ steps.list-changed.outputs.changed }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: azure/setup-helm@v3
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ inputs.checkoutCommit }}
@@ -66,7 +66,7 @@ jobs:
       - name: Create k3d cluster
         uses: nolar/setup-k3d-k3s@v1
         with:
-          version: v1.21
+          version: v1.28
       - name: Remove node taints
         run: |
           kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true

.github/workflows/schedules.yaml

@@ -10,8 +10,26 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  check_date: # https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where
+    runs-on: ubuntu-latest
+    name: Check latest commit
+    outputs:
+      should_run: ${{ steps.should_run.outputs.should_run }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: print latest_commit
+        run: echo ${{ github.sha }}
+
+      - id: should_run
+        continue-on-error: true
+        name: check latest commit is less than a day
+        if: ${{ github.event_name == 'schedule' }}
+        run: test -z $(git rev-list  --after="24 hours"  ${{ github.sha }}) && echo "::set-output name=should_run::false"
+
   # run Sytest in different variations
   sytest:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
     timeout-minutes: 60
     name: "Sytest (${{ matrix.label }})"
     runs-on: ubuntu-latest
@@ -38,8 +56,8 @@ jobs:
         RACE_DETECTION: 1
         COVER: 1
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -62,7 +80,7 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }})
@@ -75,31 +93,34 @@ jobs:
     timeout-minutes: 5
     name: "Sytest Coverage"
     runs-on: ubuntu-latest
-    needs: sytest # only run once Sytest is done
-    if: ${{ always() }}
+    needs: [ sytest, check_date ] # only run once Sytest is done and there was a commit
+    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 'stable'
           cache: true
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: Collect coverage
         run: |
           go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov
           grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov
           go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./final.cov
           flags: sytest
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # run Complement
   complement:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
     name: "Complement (${{ matrix.label }})"
     timeout-minutes: 60
     runs-on: ubuntu-latest
@@ -128,9 +149,9 @@ jobs:
         # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v3 for dendrite
-        uses: actions/checkout@v3
+          go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+      - name: Run actions/checkout@v4 for dendrite
+        uses: actions/checkout@v4
         with:
           path: dendrite
 
@@ -174,7 +195,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:
@@ -185,7 +206,7 @@ jobs:
         working-directory: complement
 
       - name: Upload Complement logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }})
@@ -196,30 +217,32 @@ jobs:
     timeout-minutes: 5
     name: "Complement Coverage"
     runs-on: ubuntu-latest
-    needs: complement # only run once Complement is done
-    if: ${{ always() }}
+    needs: [ complement, check_date ] # only run once Complements is done and there was a commit
+    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 'stable'
           cache: true
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: Collect coverage
         run: |
           go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov
           grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
           go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./final.cov
           flags: complement
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }} # required
 
   element-web:
+    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -228,7 +251,7 @@ jobs:
           # Our test suite includes some screenshot tests with unusual diacritics, which are
           # supposed to be covered by STIXGeneral.
           tools: fonts-stix
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           repository: matrix-org/matrix-react-sdk
       - uses: actions/setup-node@v3
@@ -259,6 +282,7 @@ jobs:
           TMPDIR: ${{ runner.temp }}
 
   element-web-pinecone:
+    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -267,7 +291,7 @@ jobs:
           # Our test suite includes some screenshot tests with unusual diacritics, which are
           # supposed to be covered by STIXGeneral.
           tools: fonts-stix
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           repository: matrix-org/matrix-react-sdk
       - uses: actions/setup-node@v3

.gitignore

@@ -5,6 +5,7 @@
 
 # Allow GitHub config
 !.github
+!.forgejo
 
 # Downloads
 /.downloads
@@ -74,4 +75,10 @@ complement/
 docs/_site
 
 media_store/
-build
+build
+
+# golang workspaces
+go.work*
+
+# helm chart
+helm/dendrite/charts/

.golangci.yml

@@ -6,7 +6,7 @@ run:
   concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  deadline: 30m
+  timeout: 5m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
@@ -18,24 +18,6 @@ run:
   #build-tags:
   #  - mytag
 
-  # which dirs to skip: they won't be analyzed;
-  # can use regexp here: generated.*, regexp is applied on full path;
-  # default value is empty list, but next dirs are always skipped independently
-  # from this option's value:
-  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs:
-    - bin
-    - docs
-
-  # which files to skip: they will be analyzed, but issues from them
-  # won't be reported. Default value is empty list, but there is
-  # no need to include all autogenerated files, we confidently recognize
-  # autogenerated files. If it's not please let us know.
-  skip-files:
-    - ".*\\.md$"
-    - ".*\\.sh$"
-    - "^cmd/syncserver-integration-tests/testdata.go$"
-
   # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
   # If invoked with -mod=readonly, the go command is disallowed from the implicit
   # automatic updating of go.mod described above. Instead, it fails when any changes
@@ -50,7 +32,8 @@ run:
 # output configuration options
 output:
   # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
-  format: colored-line-number
+  formats:
+    - format: colored-line-number
 
   # print lines of code with issue, default is true
   print-issued-lines: true
@@ -79,9 +62,8 @@ linters-settings:
     # see https://github.com/kisielk/errcheck#excluding-functions for details
     #exclude: /path/to/file.txt
   govet:
-    # report about shadowed variables
-    check-shadowing: true
-
+    enable:
+      - shadow
     # settings per analyzer
     settings:
       printf: # analyzer name, run `go tool vet help` to see all analyzers
@@ -180,7 +162,6 @@ linters-settings:
 linters:
   enable:
     - errcheck
-    - goconst
     - gocyclo
     - goimports # Does everything gofmt does
     - gosimple
@@ -211,12 +192,31 @@ linters:
     - stylecheck
     - typecheck # Should turn back on soon
     - unconvert # Should turn back on soon
+    - goconst # Slightly annoying, as it reports "issues" in SQL statements
   disable-all: false
   presets:
   fast: false
 
 
 issues:
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  exclude-files:
+    - ".*\\.md$"
+    - ".*\\.sh$"
+    - "^cmd/syncserver-integration-tests/testdata.go$"
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  exclude-dirs:
+    - bin
+    - docs
+
   # List of regexps of issue texts to exclude, empty list by default.
   # But independently from this option we use default exclude patterns,
   # it can be disabled by `exclude-use-default: false`. To list all

CHANGES.md

@@ -1,5 +1,185 @@
 # Changelog
 
+## Dendrite 0.13.7 (2024-04-09)
+
+### Fixes
+
+- Fixed an issue where the displayname/avatar of an invited user was replaced with the inviter's details
+- Improved server startup performance by avoiding unnecessary room ACL queries
+  - This change reduces memory footprint as it caches ACL regex patterns once instead of for each room
+  - Unnecessary Relay related queries have been removed. **Note**: To use relays, you now need to explicitly enable them using the `federation_api.enable_relays` config
+- Fixed space summaries over federation
+- Improved usage of external NATS JetStream by reusing existing connections instead of opening new ones unnecessarily
+
+### Features
+
+- Modernized Appservices (contributed by [tulir](https://github.com/tulir))
+- Added event reporting with Synapse Admin endpoints for querying them
+- Updated dependencies
+
+## Dendrite 0.13.6 (2024-01-26)
+
+Upgrading to this version is **highly** recommended, as it contains several QoL improvements.
+
+### Fixes
+
+- Use `AckExplicitPolicy` for JetStream consumers, so messages don't pile up in NATS
+- A rare panic when assigning a state key NID has been fixed
+- A rare panic when checking powerlevels has been fixed
+- Notary keys requests for all keys now work correctly
+- Spec compliance:
+  - Return `M_INVALID_PARAM` when querying room aliases
+  - Handle empty `from` parameter when requesting `/messages`
+  - Add CORP headers on media endpoints
+  - Remove `aliases` from `/publicRooms` responses
+  - Allow `+` in MXIDs (Contributed by [RosstheRoss](https://github.com/RosstheRoss))
+- Fixes membership transitions from `knock` to `join` in `knock_restricted` rooms
+- Incremental syncs now batch querying events (Contributed by [recht](https://github.com/recht))
+- Move `/joined_members` back to the clientAPI/roomserver, which should make bridges happier again
+- Backfilling from other servers now only uses at max 100 events instead of potentially thousands
+
+## Dendrite 0.13.5 (2023-12-12)
+
+Upgrading to this version is **highly** recommended, as it fixes several long-standing bugs in
+our CanonicalJSON implementation.
+
+### Fixes
+
+- Convert unicode escapes to lowercase (gomatrixserverlib)
+- Fix canonical json utf-16 surrogate pair detection logic (gomatrixserverlib)
+- Handle negative zero and exponential numbers in Canonical JSON verification (gomatrixserverlib)
+- Avoid logging unnecessary messages when unable to fetch server keys if multiple fetchers are used (gomatrixserverlib)
+- Issues around the device list updater have been fixed, which should ensure that there are always
+  workers available to process incoming device list updates.
+- A panic in the `/hierarchy` endpoints used for spaces has been fixed (client-server and server-server API)
+- Fixes around the way we handle database transactions (including a potential connection leak)
+- ACLs are now updated when received as outliers
+- A race condition, which could lead to bridges instantly leaving a room after joining it, between the SyncAPI and
+  Appservices has been fixed
+
+### Features
+
+- **Appservice login is now supported!**
+- Users can now kick themselves (used by some bridges)
+
+## Dendrite 0.13.4 (2023-10-25)
+
+Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution
+algorithm.
+
+### Fixes:
+
+- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also 
+  avoids spamming the logs when shutting down.)
+- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset"
+  out of rooms and other missing state events due to calculating the wrong state.
+- A bug when setting notifications from Element Android has been fixed by implementing MSC3987
+
+### Features
+
+- Updated dependencies
+  - Internal NATS Server has been updated from v2.9.19 to v2.9.23
+
+## Dendrite 0.13.3 (2023-09-28)
+
+### Fixes:
+
+- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir))
+- Invitations are now correctly pushed to devices
+- A bug which could result in the corruption of `m.direct` account data has been fixed 
+
+### Features
+
+- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response
+- Room version 11 is now supported
+- Clients can request the `federation` `event_format` when creating filters
+- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+
+### Other
+
+- Dendrite now requires Go 1.20 if building from source
+
+## Dendrite 0.13.2 (2023-08-23)
+
+### Fixes:
+
+- Migrations in SQLite are now prepared on the correct context (transaction or database)
+- The `InputRoomEvent` stream now has a maximum age of 24h, which should help with slow start up times of NATS JetStream (contributed by [neilalexander](https://github.com/neilalexander))
+- Event size checks are more in line with Synapse
+- Requests to `/messages` have been optimized, possibly reducing database round trips
+- Re-add the revision of Dendrite when building from source (Note: This only works if git is installed)
+- Getting local members to notify has been optimized, which should significantly reduce memory allocation and cache usage
+- When getting queried about user profiles, we now return HTTP404 if the user/profiles does not exist
+- Background federated joins should now be fixed and not timeout after a short time
+- Database connections are now correctly re-used
+- Restored the old behavior of the `/purgeRoom` admin endpoint (does not evacuate the room before purging)
+- Don't expose information about the system when trying to download files that don't exist
+
+### Features
+
+- Further improvements and fixes for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+  - Lookup correct prev events in the sync API
+  - Populate `prev_sender` correctly in the sync API
+  - Event federation should work better
+- Added new `dendrite_up` Prometheus metric, containing the version of Dendrite
+- Space summaries ([MSC2946](https://github.com/matrix-org/matrix-spec-proposals/pull/2946)) have been moved from MSC to being natively supported
+- For easier issue investigation, logs for application services now contain the application service ID (contributed by [maxberger](https://github.com/maxberger))
+- The default room version to use when creating rooms can now be configured using `room_server.default_room_version`
+
+## Dendrite 0.13.1 (2023-07-06)
+
+This releases fixes a long-standing "off-by-one" error which could result in state resets. Upgrading to this version is **highly** recommended.
+
+When deduplicating state events, we were checking if the event in question was already in a state snapshot. If it was in a previous state snapshot, we would 
+then remove it from the list of events to store. If this happened, we were, unfortunately, skipping the next event to check. This resulted in 
+events getting stored in state snapshots where they may not be needed. When we now compared two of those state snapshots, one of them
+contained the skipped event, while the other didn't. This difference possibly shouldn't exist, resulting in unexpected state resets and explains
+reports of missing state events as well.
+
+Rooms where a state reset occurred earlier should, hopefully, reconcile over time.
+
+### Fixes:
+
+- A long-standing "off-by-one" error has been fixed, which could result in state resets
+- Roomserver Prometheus Metrics are available again
+
+### Features
+
+- Updated dependencies
+  - Internal NATS Server has been updated from v2.9.15 to v2.9.19
+
+## Dendrite 0.13.0 (2023-06-30)
+
+### Features
+
+- Results in responses to `/search` now highlight words more accurately and not only the search terms as before
+- Support for connecting to appservices listening on unix sockets has been added (contributed by [cyberb](https://github.com/cyberb))
+- Admin APIs for token authenticated registration have been added (contributed by [santhoshivan23](https://github.com/santhoshivan23))
+- Initial support for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+  - This is **highly experimental**, things like changing usernames/avatars, inviting users, upgrading rooms isn't working
+
+### Fixes
+
+- `m.upload.size` is now optional, finally allowing uploads with unlimited file size
+- A bug while resolving server names has been fixed (contributed by [anton-molyboha](https://github.com/anton-molyboha))
+- Application services should only receive one invitation instead of 2 (or worse), which could result in state resets previously
+- Several admin endpoints are now using `POST` instead of `GET`
+- `/delete_devices` now uses user-interactive authentication
+- Several "membership" (e.g `/kick`, `/ban`) endpoints are using less heavy database queries to check if the user is allowed to perform this action
+- `/3pid` endpoints are now available on `/v3` instead of the `/unstable` prefix
+- Upgrading rooms ignores state events of other users, which could result in failed upgrades before
+- Uploading key backups with a wrong version now returns `M_WRONG_ROOM_KEYS_VERSION`
+- A potential state reset when joining the same room multiple times in short sequence has been fixed
+- A bug where we returned the full event as `redacted_because` in redaction events has been fixed
+- The `displayname` and `avatar_url` can now be set to empty strings
+- Unsafe hotserving of files has been fixed (contributed by [joshqou](https://github.com/joshqou))
+- Joining new rooms would potentially return "redacted" events, due to history visibility not being set correctly, this could result in events being rejected 
+- Backfilling resulting in `unsuported room version ''` should now be solved
+
+### Other
+
+- Huge refactoring of Dendrite and gomatrixserverlib
+
 ## Dendrite 0.12.0 (2023-03-13)
 
 ### Features

Dockerfile

@@ -3,8 +3,9 @@
 #
 # base installs required dependencies and runs go mod download to cache dependencies
 #
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base
-RUN apk --update --no-cache add bash build-base curl
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine3.18 AS base
+RUN apk --update --no-cache add bash build-base curl git
 
 #
 # build creates all needed binaries
@@ -13,7 +14,6 @@ FROM --platform=${BUILDPLATFORM} base AS build
 WORKDIR /src
 ARG TARGETOS
 ARG TARGETARCH
-ARG FLAGS
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
@@ -21,7 +21,7 @@ RUN --mount=target=. \
     GOARCH="$TARGETARCH" \
     GOOS="linux" \
     CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
-    go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
+    go build -v -trimpath -o /out/ ./cmd/...
 
 
 #

README.md

@@ -13,7 +13,7 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ
 
 Dendrite is **beta** software, which means:
 
-- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database.
+- Dendrite is ready for early adopters. We recommend running Dendrite with a PostgreSQL database.
 - Dendrite has periodic releases. We intend to release new versions as we fix bugs and land significant features.
 - Dendrite supports database schema upgrades between releases. This means you should never lose your messages when upgrading Dendrite.
 
@@ -21,7 +21,7 @@ This does not mean:
 
 - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
 - Dendrite is feature-complete. There may be client or federation APIs that are not implemented.
-- Dendrite is ready for massive homeserver deployments. There is no sharding of microservices (although it is possible to run them on separate machines) and there is no high-availability/clustering support.
+- Dendrite is ready for massive homeserver deployments. There is no high-availability/clustering support.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
 
@@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for
 more information on requirements.
 
-To build Dendrite, you will need Go 1.18 or later.
+To build Dendrite, you will need Go 1.20 or later.
 
 For a usable federating Dendrite deployment, you will also need:
 
@@ -47,7 +47,7 @@ For a usable federating Dendrite deployment, you will also need:
 Also recommended are:
 
 - A PostgreSQL database engine, which will perform better than SQLite with many users and/or larger rooms
-- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/master/docs/nginx/monolith-sample.conf)
+- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/dendrite-sample.conf)
 
 The [Federation Tester](https://federationtester.matrix.org) can be used to verify your deployment.
 
@@ -60,7 +60,7 @@ The following instructions are enough to get Dendrite started as a non-federatin
 ```bash
 $ git clone https://github.com/matrix-org/dendrite
 $ cd dendrite
-$ ./build.sh
+$ go build -o bin/ ./cmd/...
 
 # Generate a Matrix signing key for federation (required)
 $ ./bin/generate-keys --private-key matrix_key.pem
@@ -85,7 +85,7 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l
 
 ## Progress
 
-We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
+We use a script called "Are We Synapse Yet" which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
 updates with CI. As of January 2023, we have 100% server-server parity with Synapse, and the client-server parity is at 93% , though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse

appservice/api/query.go

@@ -82,9 +82,17 @@ type UserIDExistsResponse struct {
 }
 
 const (
-	ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
-	ASUserPath     = "/_matrix/app/unstable/thirdparty/user"
-	ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
+	ASProtocolLegacyPath        = "/_matrix/app/unstable/thirdparty/protocol/"
+	ASUserLegacyPath            = "/_matrix/app/unstable/thirdparty/user"
+	ASLocationLegacyPath        = "/_matrix/app/unstable/thirdparty/location"
+	ASRoomAliasExistsLegacyPath = "/rooms/"
+	ASUserExistsLegacyPath      = "/users/"
+
+	ASProtocolPath        = "/_matrix/app/v1/thirdparty/protocol/"
+	ASUserPath            = "/_matrix/app/v1/thirdparty/user"
+	ASLocationPath        = "/_matrix/app/v1/thirdparty/location"
+	ASRoomAliasExistsPath = "/_matrix/app/v1/rooms/"
+	ASUserExistsPath      = "/_matrix/app/v1/users/"
 )
 
 type ProtocolRequest struct {

appservice/appservice_test.go

@@ -14,7 +14,18 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/syncapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/appservice/api"
@@ -32,6 +43,10 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 )
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestAppserviceInternalAPI(t *testing.T) {
 
 	// Set expected results
@@ -134,7 +149,6 @@ func TestAppserviceInternalAPI(t *testing.T) {
 		}
 		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
 		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
-
 		t.Cleanup(func() {
 			ctx.ShutdownDendrite()
 			ctx.WaitForShutdown()
@@ -144,7 +158,8 @@ func TestAppserviceInternalAPI(t *testing.T) {
 		natsInstance := jetstream.NATSInstance{}
 		cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 		runCases(t, asAPI)
@@ -238,7 +253,8 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) {
 	natsInstance := jetstream.NATSInstance{}
 	cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 	rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+	rsAPI.SetFederationAPI(nil, nil)
+	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 	t.Run("UserIDExists", func(t *testing.T) {
@@ -377,7 +393,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 		// Create required internal APIs
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// start the consumer
 		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
 
@@ -401,3 +417,190 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 		close(evChan)
 	})
 }
+
+// Note: If this test panics, it is because we timed out waiting for the
+// join event to come through to the appservice and we close the DB/shutdown Dendrite. This makes the
+// syncAPI unhappy, as it is unable to write to the database.
+func TestOutputAppserviceEvent(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		natsInstance := &jetstream.NATSInstance{}
+
+		evChan := make(chan struct{})
+
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		// Create required internal APIs
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+
+		// Create the router, so we can hit `/joined_members`
+		routers := httputil.NewRouters()
+
+		accessTokens := map[*test.User]userDevice{
+			bob: {},
+		}
+
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		clientapi.AddPublicRoutes(processCtx, routers, cfg, natsInstance, nil, rsAPI, nil, nil, nil, usrAPI, nil, nil, caching.DisableMetrics)
+		createAccessTokens(t, accessTokens, usrAPI, processCtx.Context(), routers)
+
+		room := test.NewRoom(t, alice)
+
+		// Invite Bob
+		room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
+			"membership": "invite",
+		}, test.WithStateKey(bob.ID))
+
+		// create a dummy AS url, handling the events
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var txn consumers.ApplicationServiceTransaction
+			err := json.NewDecoder(r.Body).Decode(&txn)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, ev := range txn.Events {
+				if ev.Type != spec.MRoomMember {
+					continue
+				}
+				if ev.StateKey != nil && *ev.StateKey == bob.ID {
+					membership := gjson.GetBytes(ev.Content, "membership").Str
+					t.Logf("Processing membership: %s", membership)
+					switch membership {
+					case spec.Invite:
+						// Accept the invite
+						joinEv := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+							"membership": "join",
+						}, test.WithStateKey(bob.ID))
+
+						if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, []*types.HeaderedEvent{joinEv}, "test", "test", "test", nil, false); err != nil {
+							t.Fatalf("failed to send events: %v", err)
+						}
+					case spec.Join: // the AS has received the join event, now hit `/joined_members` to validate that
+						rec := httptest.NewRecorder()
+						req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/rooms/"+room.ID+"/joined_members", nil)
+						req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+						routers.Client.ServeHTTP(rec, req)
+						if rec.Code != http.StatusOK {
+							t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
+						}
+
+						// Both Alice and Bob should be joined. If not, we have a race condition
+						if !gjson.GetBytes(rec.Body.Bytes(), "joined."+alice.ID).Exists() {
+							t.Errorf("Alice is not joined to the room") // in theory should not happen
+						}
+						if !gjson.GetBytes(rec.Body.Bytes(), "joined."+bob.ID).Exists() {
+							t.Errorf("Bob is not joined to the room")
+						}
+						evChan <- struct{}{}
+					default:
+						t.Fatalf("Unexpected membership: %s", membership)
+					}
+				}
+			}
+		}))
+		defer srv.Close()
+
+		as := &config.ApplicationService{
+			ID:              "someID",
+			URL:             srv.URL,
+			ASToken:         "",
+			HSToken:         "",
+			SenderLocalpart: "senderLocalPart",
+			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+				"users":   {{RegexpObject: regexp.MustCompile(bob.ID)}},
+				"aliases": {{RegexpObject: regexp.MustCompile(room.ID)}},
+			},
+		}
+		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
+
+		// Create a dummy application service
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+		// Prepare AS Streams on the old topic to validate that they get deleted
+		jsCtx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+
+		token := jetstream.Tokenise(as.ID)
+		if err := jetstream.JetStreamConsumer(
+			processCtx.Context(), jsCtx, cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
+			cfg.Global.JetStream.Durable("Appservice_"+token),
+			50, // maximum number of events to send in a single transaction
+			func(ctx context.Context, msgs []*nats.Msg) bool {
+				return true
+			},
+		); err != nil {
+			t.Fatal(err)
+		}
+
+		// Start the syncAPI to have `/joined_members` available
+		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, natsInstance, usrAPI, rsAPI, caches, caching.DisableMetrics)
+
+		// start the consumer
+		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
+
+		// At this point, the old JetStream consumers should be deleted
+		for consumer := range jsCtx.Consumers(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) {
+			if consumer.Name == cfg.Global.JetStream.Durable("Appservice_"+token)+"Pull" {
+				t.Fatalf("Consumer still exists")
+			}
+		}
+
+		// Create the room, this triggers the AS to receive an invite for Bob.
+		if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		select {
+		// Pretty generous timeout duration...
+		case <-time.After(time.Millisecond * 1000): // wait for the AS to process the events
+			t.Errorf("Timed out waiting for join event")
+		case <-evChan:
+		}
+		close(evChan)
+	})
+}
+
+type userDevice struct {
+	accessToken string
+	deviceID    string
+	password    string
+}
+
+func createAccessTokens(t *testing.T, accessTokens map[*test.User]userDevice, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) {
+	t.Helper()
+	for u := range accessTokens {
+		localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+		userRes := &uapi.PerformAccountCreationResponse{}
+		password := util.RandomString(8)
+		if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+			AccountType: u.AccountType,
+			Localpart:   localpart,
+			ServerName:  serverName,
+			Password:    password,
+		}, userRes); err != nil {
+			t.Errorf("failed to create account: %s", err)
+		}
+		req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+			"type": authtypes.LoginTypePassword,
+			"identifier": map[string]interface{}{
+				"type": "m.id.user",
+				"user": u.ID,
+			},
+			"password": password,
+		}))
+		rec := httptest.NewRecorder()
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("failed to login: %s", rec.Body.String())
+		}
+		accessTokens[u] = userDevice{
+			accessToken: gjson.GetBytes(rec.Body.Bytes(), "access_token").String(),
+			deviceID:    gjson.GetBytes(rec.Body.Bytes(), "device_id").String(),
+			password:    password,
+		}
+	}
+}

appservice/consumers/roomserver.go

@@ -71,13 +71,14 @@ func NewOutputRoomEventConsumer(
 		ctx:       process.Context(),
 		cfg:       cfg,
 		jetstream: js,
-		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
+		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputAppserviceEvent),
 		rsAPI:     rsAPI,
 	}
 }
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
+	durableNames := make([]string, 0, len(s.cfg.Derived.ApplicationServices))
 	for _, as := range s.cfg.Derived.ApplicationServices {
 		appsvc := as
 		state := &appserviceState{
@@ -95,6 +96,15 @@ func (s *OutputRoomEventConsumer) Start() error {
 		); err != nil {
 			return fmt.Errorf("failed to create %q consumer: %w", token, err)
 		}
+		durableNames = append(durableNames, s.cfg.Matrix.JetStream.Durable("Appservice_"+token))
+	}
+	// Cleanup any consumers still existing on the OutputRoomEvent stream
+	// to avoid messages not being deleted
+	for _, consumerName := range durableNames {
+		err := s.jetstream.DeleteConsumer(s.cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), consumerName+"Pull")
+		if err != nil && err != nats.ErrConsumerNotFound {
+			return err
+		}
 	}
 	return nil
 }
@@ -128,7 +138,7 @@ func (s *OutputRoomEventConsumer) onMessage(
 			if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
 				newEventID := output.NewRoomEvent.Event.EventID()
 				eventsReq := &api.QueryEventsByIDRequest{
-					RoomID:   output.NewRoomEvent.Event.RoomID(),
+					RoomID:   output.NewRoomEvent.Event.RoomID().String(),
 					EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
 				}
 				eventsRes := &api.QueryEventsByIDResponse{}
@@ -181,7 +191,9 @@ func (s *OutputRoomEventConsumer) sendEvents(
 	// Create the transaction body.
 	transaction, err := json.Marshal(
 		ApplicationServiceTransaction{
-			Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatAll),
+			Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+				return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+			}),
 		},
 	)
 	if err != nil {
@@ -194,13 +206,21 @@ func (s *OutputRoomEventConsumer) sendEvents(
 	}
 
 	// Send the transaction to the appservice.
-	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
-	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.RequestUrl(), txnID, url.QueryEscape(state.HSToken))
+	// https://spec.matrix.org/v1.9/application-service-api/#pushing-events
+	path := "_matrix/app/v1/transactions"
+	if s.cfg.LegacyPaths {
+		path = "transactions"
+	}
+	address := fmt.Sprintf("%s/%s/%s", state.RequestUrl(), path, txnID)
+	if s.cfg.LegacyAuth {
+		address += "?access_token=" + url.QueryEscape(state.HSToken)
+	}
 	req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
 	if err != nil {
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", state.HSToken))
 	resp, err := state.HTTPClient.Do(req)
 	if err != nil {
 		return state.backoffAndPause(err)
@@ -233,12 +253,18 @@ func (s *appserviceState) backoffAndPause(err error) error {
 //
 // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
 func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool {
+	user := ""
+	userID, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
+	if err == nil {
+		user = userID.String()
+	}
+
 	switch {
 	case appservice.URL == "":
 		return false
-	case appservice.IsInterestedInUserID(event.Sender()):
+	case appservice.IsInterestedInUserID(user):
 		return true
-	case appservice.IsInterestedInRoomID(event.RoomID()):
+	case appservice.IsInterestedInRoomID(event.RoomID().String()):
 		return true
 	}
 
@@ -249,7 +275,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	}
 
 	// Check all known room aliases of the room the event came from
-	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
+	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID().String()}
 	var queryRes api.GetAliasesForRoomIDResponse
 	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
 		for _, alias := range queryRes.Aliases {
@@ -260,7 +286,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID(),
+			"room_id":    event.RoomID().String(),
 		}).WithError(err).Errorf("Unable to get aliases for room")
 	}
 
@@ -276,7 +302,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 	// until we have a lighter way of checking the state before the event that
 	// doesn't involve state res, then this is probably OK.
 	membershipReq := &api.QueryMembershipsForRoomRequest{
-		RoomID:     event.RoomID(),
+		RoomID:     event.RoomID().String(),
 		JoinedOnly: true,
 	}
 	membershipRes := &api.QueryMembershipsForRoomResponse{}
@@ -305,7 +331,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID(),
+			"room_id":    event.RoomID().String(),
 		}).WithError(err).Errorf("Unable to get membership for room")
 	}
 	return false

appservice/query/query.go

@@ -19,10 +19,10 @@ package query
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"strings"
 	"sync"
 
 	log "github.com/sirupsen/logrus"
@@ -32,9 +32,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-const roomAliasExistsPath = "/rooms/"
-const userIDExistsPath = "/users/"
-
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
 	Cfg           *config.AppServiceAPI
@@ -55,14 +52,23 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
+			path := api.ASRoomAliasExistsPath
+			if a.Cfg.LegacyPaths {
+				path = api.ASRoomAliasExistsLegacyPath
+			}
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.RequestUrl() + roomAliasExistsPath)
+			URL, err := url.Parse(appservice.RequestUrl() + path)
 			if err != nil {
 				return err
 			}
 
 			URL.Path += request.Alias
-			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+			if a.Cfg.LegacyAuth {
+				q := URL.Query()
+				q.Set("access_token", appservice.HSToken)
+				URL.RawQuery = q.Encode()
+			}
+			apiURL := URL.String()
 
 			// Send a request to each application service. If one responds that it has
 			// created the room, immediately return.
@@ -70,6 +76,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			if err != nil {
 				return err
 			}
+			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
 			req = req.WithContext(ctx)
 
 			resp, err := appservice.HTTPClient.Do(req)
@@ -123,12 +130,21 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.RequestUrl() + userIDExistsPath)
+			path := api.ASUserExistsPath
+			if a.Cfg.LegacyPaths {
+				path = api.ASUserExistsLegacyPath
+			}
+			URL, err := url.Parse(appservice.RequestUrl() + path)
 			if err != nil {
 				return err
 			}
 			URL.Path += request.UserID
-			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+			if a.Cfg.LegacyAuth {
+				q := URL.Query()
+				q.Set("access_token", appservice.HSToken)
+				URL.RawQuery = q.Encode()
+			}
+			apiURL := URL.String()
 
 			// Send a request to each application service. If one responds that it has
 			// created the user, immediately return.
@@ -136,6 +152,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
+			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
 			resp, err := appservice.HTTPClient.Do(req.WithContext(ctx))
 			if resp != nil {
 				defer func() {
@@ -176,25 +193,22 @@ type thirdpartyResponses interface {
 	api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
 }
 
-func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
-	origURL := url
-	// try v1 and unstable appservice endpoints
-	for _, version := range []string{"v1", "unstable"} {
-		var resp *http.Response
-		var body []byte
-		asURL := strings.Replace(origURL, "unstable", version, 1)
-		resp, err = client.Get(asURL)
-		if err != nil {
-			continue
-		}
-		defer resp.Body.Close() // nolint: errcheck
-		body, err = io.ReadAll(resp.Body)
-		if err != nil {
-			continue
-		}
-		return json.Unmarshal(body, &response)
+func requestDo[T thirdpartyResponses](as *config.ApplicationService, url string, response *T) error {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return err
 	}
-	return err
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", as.HSToken))
+	resp, err := as.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close() // nolint: errcheck
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(body, &response)
 }
 
 func (a *AppServiceQueryAPI) Locations(
@@ -207,17 +221,23 @@ func (a *AppServiceQueryAPI) Locations(
 		return err
 	}
 
+	path := api.ASLocationPath
+	if a.Cfg.LegacyPaths {
+		path = api.ASLocationLegacyPath
+	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asLocations []api.ASLocationResponse
-		params.Set("access_token", as.HSToken)
+		if a.Cfg.LegacyAuth {
+			params.Set("access_token", as.HSToken)
+		}
 
-		url := as.RequestUrl() + api.ASLocationPath
+		url := as.RequestUrl() + path
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
-			log.WithError(err).Error("unable to get 'locations' from application service")
+		if err := requestDo[[]api.ASLocationResponse](&as, url+"?"+params.Encode(), &asLocations); err != nil {
+			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service")
 			continue
 		}
 
@@ -242,17 +262,23 @@ func (a *AppServiceQueryAPI) User(
 		return err
 	}
 
+	path := api.ASUserPath
+	if a.Cfg.LegacyPaths {
+		path = api.ASUserLegacyPath
+	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asUsers []api.ASUserResponse
-		params.Set("access_token", as.HSToken)
+		if a.Cfg.LegacyAuth {
+			params.Set("access_token", as.HSToken)
+		}
 
-		url := as.RequestUrl() + api.ASUserPath
+		url := as.RequestUrl() + path
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
-			log.WithError(err).Error("unable to get 'user' from application service")
+		if err := requestDo[[]api.ASUserResponse](&as, url+"?"+params.Encode(), &asUsers); err != nil {
+			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service")
 			continue
 		}
 
@@ -272,6 +298,10 @@ func (a *AppServiceQueryAPI) Protocols(
 	req *api.ProtocolRequest,
 	resp *api.ProtocolResponse,
 ) error {
+	protocolPath := api.ASProtocolPath
+	if a.Cfg.LegacyPaths {
+		protocolPath = api.ASProtocolLegacyPath
+	}
 
 	// get a single protocol response
 	if req.Protocol != "" {
@@ -289,8 +319,8 @@ func (a *AppServiceQueryAPI) Protocols(
 		response := api.ASProtocolResponse{}
 		for _, as := range a.Cfg.Derived.ApplicationServices {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil {
-				log.WithError(err).Error("unable to get 'protocol' from application service")
+			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+req.Protocol, &proto); err != nil {
+				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
 
@@ -319,8 +349,8 @@ func (a *AppServiceQueryAPI) Protocols(
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		for _, p := range as.Protocols {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil {
-				log.WithError(err).Error("unable to get 'protocol' from application service")
+			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+p, &proto); err != nil {
+				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
 			existing, ok := response[p]

are-we-synapse-yet.list

@@ -944,4 +944,12 @@ rmv remote user can join room with version 10
 rmv User can invite remote user to room with version 10
 rmv Remote user can backfill in a room with version 10
 rmv Can reject invites over federation for rooms with version 10
-rmv Can receive redactions from regular users over federation in room version 10
+rmv Can receive redactions from regular users over federation in room version 10
+rmv User can create and send/receive messages in a room with version 11
+rmv local user can join room with version 11
+rmv User can invite local user to room with version 11
+rmv remote user can join room with version 11
+rmv User can invite remote user to room with version 11
+rmv Remote user can backfill in a room with version 11
+rmv Can reject invites over federation for rooms with version 11
+rmv Can receive redactions from regular users over federation in room version 11

build.cmd

@@ -1,51 +0,0 @@
-@echo off
-
-:ENTRY_POINT
-    setlocal EnableDelayedExpansion
-
-    REM script base dir
-    set SCRIPTDIR=%~dp0
-    set PROJDIR=%SCRIPTDIR:~0,-1%
-
-    REM Put installed packages into ./bin
-    set GOBIN=%PROJDIR%\bin
-
-    set FLAGS=
-
-    REM Check if sources are under Git control
-    if not exist ".git" goto :CHECK_BIN
-
-    REM set BUILD=`git rev-parse --short HEAD \\ ""`
-    FOR /F "tokens=*" %%X IN ('git rev-parse --short HEAD') DO (
-        set BUILD=%%X
-    )
-
-    REM set BRANCH=`(git symbolic-ref --short HEAD \ tr -d \/ ) \\ ""`
-    FOR /F "tokens=*" %%X IN ('git symbolic-ref --short HEAD') DO (
-        set BRANCHRAW=%%X
-        set BRANCH=!BRANCHRAW:/=!
-    )
-    if "%BRANCH%" == "main" set BRANCH=
-
-    set FLAGS=-X github.com/matrix-org/dendrite/internal.branch=%BRANCH% -X github.com/matrix-org/dendrite/internal.build=%BUILD%
-
-:CHECK_BIN
-    if exist "bin" goto :ALL_SET
-    mkdir "bin"
-
-:ALL_SET
-    set CGO_ENABLED=1
-    for /D %%P in (cmd\*) do (
-        go build -trimpath -ldflags "%FLAGS%" -v -o ".\bin" ".\%%P"
-    )
-
-    set CGO_ENABLED=0
-    set GOOS=js
-    set GOARCH=wasm
-    go build -trimpath -ldflags "%FLAGS%" -o bin\main.wasm .\cmd\dendritejs-pinecone
-
-    goto :DONE
-
-:DONE
-    echo Done
-    endlocal

build.sh

@@ -1,24 +0,0 @@
-#!/bin/sh -eu
-
-# Put installed packages into ./bin
-export GOBIN=$PWD/`dirname $0`/bin
-
-if [ -d ".git" ] 
-then
-    export BUILD=`git rev-parse --short HEAD || ""`
-    export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""`
-    if [ "$BRANCH" = main ]
-    then
-        export BRANCH=""
-    fi
-
-    export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD"
-else
-    export FLAGS=""
-fi
-
-mkdir -p bin
-
-CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
-
-# CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone

build/dendritejs-pinecone/main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -190,13 +191,13 @@ func startup() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fedSenderAPI.IsBlacklistedOrBackingOff)
 
 	asQuery := appservice.NewInternalAPI(
 		processCtx, cfg, &natsInstance, userAPI, rsAPI,
 	)
 	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
 	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
 
 	monolith := setup.Monolith{

build/docker/Dockerfile.demo-pinecone

@@ -1,4 +1,5 @@
-FROM docker.io/golang:1.19-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM docker.io/golang:1.21-alpine3.18 AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/docker/Dockerfile.demo-yggdrasil

@@ -1,4 +1,5 @@
-FROM docker.io/golang:1.19-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM docker.io/golang:1.21-alpine3.18 AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/docker/README.md

@@ -6,23 +6,20 @@ They can be found on Docker Hub:
 
 - [matrixdotorg/dendrite-monolith](https://hub.docker.com/r/matrixdotorg/dendrite-monolith) for monolith deployments
 
-## Dockerfiles
+## Dockerfile
 
-The `Dockerfile` is a multistage file which can build all four Dendrite
-images depending on the supplied `--target`. From the root of the Dendrite
+The `Dockerfile` is a multistage file which can build Dendrite. From the root of the Dendrite
 repository, run:
 
 ```
-docker build . --target monolith -t matrixdotorg/dendrite-monolith
-docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone
-docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil
+docker build . -t matrixdotorg/dendrite-monolith
 ```
 
-## Compose files
+## Compose file
 
-There are two sample `docker-compose` files:
+There is one sample `docker-compose` files:
 
-- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment
+- `docker-compose.yml` which runs a Dendrite deployment with Postgres
 
 ## Configuration
 
@@ -55,7 +52,7 @@ Create your config based on the [`dendrite-sample.yaml`](https://github.com/matr
 Then start the deployment:
 
 ```
-docker-compose -f docker-compose.monolith.yml up
+docker-compose -f docker-compose.yml up
 ```
 
 ## Building the images

build/docker/docker-compose.monolith.yml

@@ -1,44 +0,0 @@
-version: "3.4"
-services:
-  postgres:
-    hostname: postgres
-    image: postgres:14
-    restart: always
-    volumes:
-      - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
-      # To persist your PostgreSQL databases outside of the Docker image, 
-      # to prevent data loss, modify the following ./path_to path:
-      - ./path_to/postgresql:/var/lib/postgresql/data
-    environment:
-      POSTGRES_PASSWORD: itsasecret
-      POSTGRES_USER: dendrite
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U dendrite"]
-      interval: 5s
-      timeout: 5s
-      retries: 5
-    networks:
-      - internal
-
-  monolith:
-    hostname: monolith
-    image: matrixdotorg/dendrite-monolith:latest
-    command: [
-      "--tls-cert=server.crt",
-      "--tls-key=server.key"
-    ]
-    ports:
-      - 8008:8008
-      - 8448:8448
-    volumes:
-      - ./config:/etc/dendrite
-      - ./media:/var/dendrite/media
-    depends_on:
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-
-networks:
-  internal:
-    attachable: true

build/docker/docker-compose.yml

@@ -0,0 +1,52 @@
+version: "3.4"
+
+services:
+  postgres:
+    hostname: postgres
+    image: postgres:15-alpine
+    restart: always
+    volumes:
+      # This will create a docker volume to persist the database files in.
+      # If you prefer those files to be outside of docker, you'll need to change this.
+      - dendrite_postgres_data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: itsasecret
+      POSTGRES_USER: dendrite
+      POSTGRES_DATABASE: dendrite
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U dendrite"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    networks:
+      - internal
+
+  monolith:
+    hostname: monolith
+    image: matrixdotorg/dendrite-monolith:latest
+    ports:
+      - 8008:8008
+      - 8448:8448
+    volumes:
+      - ./config:/etc/dendrite
+      # The following volumes use docker volumes, change this
+      # if you prefer to have those files outside of docker.
+      - dendrite_media:/var/dendrite/media
+      - dendrite_jetstream:/var/dendrite/jetstream
+      - dendrite_search_index:/var/dendrite/searchindex
+    depends_on:
+      postgres:
+        condition: service_healthy
+    networks:
+      - internal
+    restart: unless-stopped
+
+networks:
+  internal:
+    attachable: true
+
+volumes:
+  dendrite_postgres_data:
+  dendrite_media:
+  dendrite_jetstream:
+  dendrite_search_index:

build/docker/postgres/create_db.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-for db in userapi_accounts mediaapi syncapi roomserver keyserver federationapi appservice mscs; do
-    createdb -U dendrite -O dendrite dendrite_$db
-done

build/gobind-pinecone/monolith_test.go

@@ -22,7 +22,10 @@ import (
 )
 
 func TestMonolithStarts(t *testing.T) {
-	monolith := DendriteMonolith{}
+	monolith := DendriteMonolith{
+		StorageDirectory: t.TempDir(),
+		CacheDirectory:   t.TempDir(),
+	}
 	monolith.Start()
 	monolith.PublicKey()
 	monolith.Stop()
@@ -60,7 +63,10 @@ func TestMonolithSetRelayServers(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		monolith := DendriteMonolith{}
+		monolith := DendriteMonolith{
+			StorageDirectory: t.TempDir(),
+			CacheDirectory:   t.TempDir(),
+		}
 		monolith.Start()
 
 		inputRelays := tc.relays

build/gobind-yggdrasil/monolith.go

@@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() {
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)

build/scripts/complement.sh

@@ -15,5 +15,5 @@ tar -xzf master.tar.gz
 
 # Run the tests!
 cd complement-master
-COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests
+COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi
 

clientapi/api/api.go

@@ -21,3 +21,11 @@ type ExtraPublicRoomsProvider interface {
 	// Rooms returns the extra rooms. This is called on-demand by clients, so cache appropriately.
 	Rooms() []fclient.PublicRoom
 }
+
+type RegistrationToken struct {
+	Token       *string `json:"token"`
+	UsesAllowed *int32  `json:"uses_allowed"`
+	Pending     *int32  `json:"pending"`
+	Completed   *int32  `json:"completed"`
+	ExpiryTime  *int64  `json:"expiry_time"`
+}

clientapi/auth/auth.go

@@ -23,8 +23,8 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -58,7 +58,7 @@ func VerifyUserFromRequest(
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.MissingToken(err.Error()),
+			JSON: spec.MissingToken(err.Error()),
 		}
 	}
 	var res api.QueryAccessTokenResponse
@@ -68,21 +68,23 @@ func VerifyUserFromRequest(
 	}, &res)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccessToken failed")
-		jsonErr := jsonerror.InternalServerError()
-		return nil, &jsonErr
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if res.Err != "" {
 		if strings.HasPrefix(strings.ToLower(res.Err), "forbidden:") { // TODO: use actual error and no string comparison
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden(res.Err),
+				JSON: spec.Forbidden(res.Err),
 			}
 		}
 	}
 	if res.Device == nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.UnknownToken("Unknown token"),
+			JSON: spec.UnknownToken("Unknown token"),
 		}
 	}
 	return res.Device, nil

clientapi/auth/login.go

@@ -15,15 +15,14 @@
 package auth
 
 import (
-	"context"
 	"encoding/json"
 	"io"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -32,12 +31,17 @@ import (
 // called after authorization has completed, with the result of the authorization.
 // If the final return value is non-nil, an error occurred and the cleanup function
 // is nil.
-func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
-	reqBytes, err := io.ReadAll(r)
+func LoginFromJSONReader(
+	req *http.Request,
+	useraccountAPI uapi.UserLoginAPI,
+	userAPI UserInternalAPIForLogin,
+	cfg *config.ClientAPI,
+) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	reqBytes, err := io.ReadAll(req.Body)
 	if err != nil {
 		err := &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Reading request body failed: " + err.Error()),
+			JSON: spec.BadJSON("Reading request body failed: " + err.Error()),
 		}
 		return nil, nil, err
 	}
@@ -48,7 +52,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 	if err := json.Unmarshal(reqBytes, &header); err != nil {
 		err := &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Reading request body failed: " + err.Error()),
+			JSON: spec.BadJSON("Reading request body failed: " + err.Error()),
 		}
 		return nil, nil, err
 	}
@@ -57,23 +61,37 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 	switch header.Type {
 	case authtypes.LoginTypePassword:
 		typ = &LoginTypePassword{
-			GetAccountByPassword: useraccountAPI.QueryAccountByPassword,
-			Config:               cfg,
+			UserAPI: useraccountAPI,
+			Config:  cfg,
 		}
 	case authtypes.LoginTypeToken:
 		typ = &LoginTypeToken{
 			UserAPI: userAPI,
 			Config:  cfg,
 		}
+	case authtypes.LoginTypeApplicationService:
+		token, err := ExtractAccessToken(req)
+		if err != nil {
+			err := &util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.MissingToken(err.Error()),
+			}
+			return nil, nil, err
+		}
+
+		typ = &LoginTypeApplicationService{
+			Config: cfg,
+			Token:  token,
+		}
 	default:
 		err := util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue("unhandled login type: " + header.Type),
+			JSON: spec.InvalidParam("unhandled login type: " + header.Type),
 		}
 		return nil, nil, &err
 	}
 
-	return typ.LoginFromJSON(ctx, reqBytes)
+	return typ.LoginFromJSON(req.Context(), reqBytes)
 }
 
 // UserInternalAPIForLogin contains the aspects of UserAPI required for logging in.

clientapi/auth/login_application_service.go

@@ -0,0 +1,55 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/util"
+)
+
+// LoginTypeApplicationService describes how to authenticate as an
+// application service
+type LoginTypeApplicationService struct {
+	Config *config.ClientAPI
+	Token  string
+}
+
+// Name implements Type
+func (t *LoginTypeApplicationService) Name() string {
+	return authtypes.LoginTypeApplicationService
+}
+
+// LoginFromJSON implements Type
+func (t *LoginTypeApplicationService) LoginFromJSON(
+	ctx context.Context, reqBytes []byte,
+) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	var r Login
+	if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil {
+		return nil, nil, err
+	}
+
+	_, err := internal.ValidateApplicationServiceRequest(t.Config, r.Identifier.User, t.Token)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cleanup := func(ctx context.Context, j *util.JSONResponse) {}
+	return &r, cleanup, nil
+}

clientapi/auth/login_test.go

@@ -17,15 +17,17 @@ package auth
 import (
 	"context"
 	"net/http"
+	"net/http/httptest"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -33,8 +35,9 @@ func TestLoginFromJSONReader(t *testing.T) {
 	ctx := context.Background()
 
 	tsts := []struct {
-		Name string
-		Body string
+		Name  string
+		Body  string
+		Token string
 
 		WantUsername      string
 		WantDeviceID      string
@@ -62,6 +65,30 @@ func TestLoginFromJSONReader(t *testing.T) {
 			WantDeviceID:      "adevice",
 			WantDeletedTokens: []string{"atoken"},
 		},
+		{
+			Name: "appServiceWorksUserID",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
+				"device_id": "adevice"
+			}`,
+			Token: "astoken",
+
+			WantUsername: "@alice:example.com",
+			WantDeviceID: "adevice",
+		},
+		{
+			Name: "appServiceWorksLocalpart",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "alice" },
+				"device_id": "adevice"
+			}`,
+			Token: "astoken",
+
+			WantUsername: "alice",
+			WantDeviceID: "adevice",
+		},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
@@ -72,11 +99,35 @@ func TestLoginFromJSONReader(t *testing.T) {
 						ServerName: serverName,
 					},
 				},
+				Derived: &config.Derived{
+					ApplicationServices: []config.ApplicationService{
+						{
+							ID:      "anapplicationservice",
+							ASToken: "astoken",
+							NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+								"users": {
+									{
+										Exclusive:    true,
+										Regex:        "@alice:example.com",
+										RegexpObject: regexp.MustCompile("@alice:example.com"),
+									},
+								},
+							},
+						},
+					},
+				},
 			}
-			login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
-			if err != nil {
-				t.Fatalf("LoginFromJSONReader failed: %+v", err)
+
+			req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body))
+			if tst.Token != "" {
+				req.Header.Add("Authorization", "Bearer "+tst.Token)
 			}
+
+			login, cleanup, jsonErr := LoginFromJSONReader(req, &userAPI, &userAPI, cfg)
+			if jsonErr != nil {
+				t.Fatalf("LoginFromJSONReader failed: %+v", jsonErr)
+			}
+
 			cleanup(ctx, &util.JSONResponse{Code: http.StatusOK})
 
 			if login.Username() != tst.WantUsername {
@@ -104,16 +155,17 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 	ctx := context.Background()
 
 	tsts := []struct {
-		Name string
-		Body string
+		Name  string
+		Body  string
+		Token string
 
-		WantErrCode string
+		WantErrCode spec.MatrixErrorCode
 	}{
-		{Name: "empty", WantErrCode: "M_BAD_JSON"},
+		{Name: "empty", WantErrCode: spec.ErrorBadJSON},
 		{
 			Name:        "badUnmarshal",
 			Body:        `badsyntaxJSON`,
-			WantErrCode: "M_BAD_JSON",
+			WantErrCode: spec.ErrorBadJSON,
 		},
 		{
 			Name: "badPassword",
@@ -123,7 +175,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 				"password": "invalidpassword",
 				"device_id": "adevice"
             }`,
-			WantErrCode: "M_FORBIDDEN",
+			WantErrCode: spec.ErrorForbidden,
 		},
 		{
 			Name: "badToken",
@@ -132,7 +184,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 				"token": "invalidtoken",
 				"device_id": "adevice"
             }`,
-			WantErrCode: "M_FORBIDDEN",
+			WantErrCode: spec.ErrorForbidden,
 		},
 		{
 			Name: "badType",
@@ -140,7 +192,46 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 				"type": "m.login.invalid",
 				"device_id": "adevice"
             }`,
-			WantErrCode: "M_INVALID_ARGUMENT_VALUE",
+			WantErrCode: spec.ErrorInvalidParam,
+		},
+		{
+			Name: "noASToken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_MISSING_TOKEN",
+		},
+		{
+			Name:  "badASToken",
+			Token: "badastoken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_UNKNOWN_TOKEN",
+		},
+		{
+			Name:  "badASNamespace",
+			Token: "astoken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@bob:example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_EXCLUSIVE",
+		},
+		{
+			Name:  "badASUserID",
+			Token: "astoken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:wrong.example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_INVALID_USERNAME",
 		},
 	}
 	for _, tst := range tsts {
@@ -152,12 +243,34 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 						ServerName: serverName,
 					},
 				},
+				Derived: &config.Derived{
+					ApplicationServices: []config.ApplicationService{
+						{
+							ID:      "anapplicationservice",
+							ASToken: "astoken",
+							NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+								"users": {
+									{
+										Exclusive:    true,
+										Regex:        "@alice:example.com",
+										RegexpObject: regexp.MustCompile("@alice:example.com"),
+									},
+								},
+							},
+						},
+					},
+				},
 			}
-			_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
+			req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body))
+			if tst.Token != "" {
+				req.Header.Add("Authorization", "Bearer "+tst.Token)
+			}
+
+			_, cleanup, errRes := LoginFromJSONReader(req, &userAPI, &userAPI, cfg)
 			if errRes == nil {
 				cleanup(ctx, nil)
 				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
-			} else if merr, ok := errRes.JSON.(*jsonerror.MatrixError); ok && merr.ErrCode != tst.WantErrCode {
+			} else if merr, ok := errRes.JSON.(spec.MatrixError); ok && merr.ErrCode != tst.WantErrCode {
 				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
 			}
 		})
@@ -179,6 +292,14 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *
 	return nil
 }
 
+func (ua *fakeUserInternalAPI) QueryAccountByLocalpart(ctx context.Context, req *uapi.QueryAccountByLocalpartRequest, res *uapi.QueryAccountByLocalpartResponse) error {
+	return nil
+}
+
+func (ua *fakeUserInternalAPI) PerformAccountCreation(ctx context.Context, req *uapi.PerformAccountCreationRequest, res *uapi.PerformAccountCreationResponse) error {
+	return nil
+}
+
 func (ua *fakeUserInternalAPI) PerformLoginTokenDeletion(ctx context.Context, req *uapi.PerformLoginTokenDeletionRequest, res *uapi.PerformLoginTokenDeletionResponse) error {
 	ua.DeletedTokens = append(ua.DeletedTokens, req.Token)
 	return nil

clientapi/auth/login_token.go

@@ -20,9 +20,9 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -48,13 +48,15 @@ func (t *LoginTypeToken) LoginFromJSON(ctx context.Context, reqBytes []byte) (*L
 	var res uapi.QueryLoginTokenResponse
 	if err := t.UserAPI.QueryLoginToken(ctx, &uapi.QueryLoginTokenRequest{Token: r.Token}, &res); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("UserAPI.QueryLoginToken failed")
-		jsonErr := jsonerror.InternalServerError()
-		return nil, nil, &jsonErr
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if res.Data == nil {
 		return nil, nil, &util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("invalid login token"),
+			JSON: spec.Forbidden("invalid login token"),
 		}
 	}
 

clientapi/auth/password.go

@@ -16,20 +16,21 @@ package auth
 
 import (
 	"context"
+	"database/sql"
+	"github.com/go-ldap/ldap/v3"
+	"github.com/google/uuid"
 	"net/http"
 	"strings"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
-type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error
-
 type PasswordRequest struct {
 	Login
 	Password string `json:"password"`
@@ -37,8 +38,8 @@ type PasswordRequest struct {
 
 // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based
 type LoginTypePassword struct {
-	GetAccountByPassword GetAccountByPassword
-	Config               *config.ClientAPI
+	Config  *config.ClientAPI
+	UserAPI api.UserLoginAPI
 }
 
 func (t *LoginTypePassword) Name() string {
@@ -59,73 +60,227 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
 	return login, func(context.Context, *util.JSONResponse) {}, nil
 }
 
-func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
-	r := req.(*PasswordRequest)
-	username := r.Username()
-	if username == "" {
+func (t *LoginTypePassword) Login(ctx context.Context, request *PasswordRequest) (*Login, *util.JSONResponse) {
+	fullUsername := request.Username()
+	if fullUsername == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.BadJSON("A username must be supplied."),
+			JSON: spec.BadJSON("A username must be supplied."),
 		}
 	}
-	if len(r.Password) == 0 {
+	if len(request.Password) == 0 {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.BadJSON("A password must be supplied."),
+			JSON: spec.BadJSON("A password must be supplied."),
 		}
 	}
-	localpart, domain, err := userutil.ParseUsernameParam(username, t.Config.Matrix)
+	username, domain, err := userutil.ParseUsernameParam(fullUsername, t.Config.Matrix)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.InvalidUsername(err.Error()),
+			JSON: spec.InvalidUsername(err.Error()),
 		}
 	}
 	if !t.Config.Matrix.IsLocalServerName(domain) {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.InvalidUsername("The server name is not known."),
+			JSON: spec.InvalidUsername("The server name is not known."),
 		}
 	}
-	// Squash username to all lowercase letters
+
+	var account *api.Account
+	if t.Config.Ldap.Enabled {
+		isAdmin, err := t.authenticateLdap(username, request.Password)
+		if err != nil {
+			return nil, err
+		}
+		acc, err := t.getOrCreateAccount(ctx, username, domain, isAdmin)
+		if err != nil {
+			return nil, err
+		}
+		account = acc
+	} else {
+		acc, err := t.authenticateDb(ctx, username, domain, request.Password)
+		if err != nil {
+			return nil, err
+		}
+		account = acc
+	}
+
+	// Set the user, so login.Username() can do the right thing
+	request.Identifier.User = account.UserID
+	request.User = account.UserID
+	return &request.Login, nil
+}
+
+func (t *LoginTypePassword) authenticateDb(ctx context.Context, username string, domain spec.ServerName, password string) (*api.Account, *util.JSONResponse) {
 	res := &api.QueryAccountByPasswordResponse{}
-	err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-		Localpart:         strings.ToLower(localpart),
+	err := t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+		Localpart:         strings.ToLower(username),
 		ServerName:        domain,
-		PlaintextPassword: r.Password,
+		PlaintextPassword: password,
 	}, res)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("Unable to fetch account by password."),
+			JSON: spec.Unknown("Unable to fetch account by password."),
 		}
 	}
 
-	// If we couldn't find the user by the lower cased localpart, try the provided
-	// localpart as is.
 	if !res.Exists {
-		err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-			Localpart:         localpart,
+		err = t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+			Localpart:         username,
 			ServerName:        domain,
-			PlaintextPassword: r.Password,
+			PlaintextPassword: password,
 		}, res)
 		if err != nil {
 			return nil, &util.JSONResponse{
 				Code: http.StatusInternalServerError,
-				JSON: jsonerror.Unknown("Unable to fetch account by password."),
+				JSON: spec.Unknown("Unable to fetch account by password."),
 			}
 		}
-		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
-		// but that would leak the existence of the user.
 		if !res.Exists {
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."),
+				JSON: spec.Forbidden("The username or password was incorrect or the account does not exist."),
 			}
 		}
 	}
-	// Set the user, so login.Username() can do the right thing
-	r.Identifier.User = res.Account.UserID
-	r.User = res.Account.UserID
-	return &r.Login, nil
+	return res.Account, nil
+}
+func (t *LoginTypePassword) authenticateLdap(username, password string) (bool, *util.JSONResponse) {
+	var conn *ldap.Conn
+	conn, err := ldap.DialURL(t.Config.Ldap.Uri)
+	if err != nil {
+		return false, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("unable to connect to ldap: " + err.Error()),
+		}
+	}
+	defer conn.Close()
+
+	if t.Config.Ldap.AdminBindEnabled {
+		err = conn.Bind(t.Config.Ldap.AdminBindDn, t.Config.Ldap.AdminBindPassword)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("unable to bind to ldap: " + err.Error()),
+			}
+		}
+		filter := strings.ReplaceAll(t.Config.Ldap.SearchFilter, "{username}", username)
+		searchRequest := ldap.NewSearchRequest(
+			t.Config.Ldap.BaseDn, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases,
+			0, 0, false, filter, []string{t.Config.Ldap.SearchAttribute}, nil,
+		)
+		result, err := conn.Search(searchRequest)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("unable to bind to search ldap: " + err.Error()),
+			}
+		}
+		if len(result.Entries) > 1 {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.BadJSON("'user' must be duplicated."),
+			}
+		}
+		if len(result.Entries) < 1 {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.BadJSON("'user' not found."),
+			}
+		}
+
+		userDN := result.Entries[0].DN
+		err = conn.Bind(userDN, password)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.InvalidUsername(err.Error()),
+			}
+		}
+	} else {
+		bindDn := strings.ReplaceAll(t.Config.Ldap.UserBindDn, "{username}", username)
+		err = conn.Bind(bindDn, password)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.InvalidUsername(err.Error()),
+			}
+		}
+	}
+
+	isAdmin, err := t.isLdapAdmin(conn, username)
+	if err != nil {
+		return false, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: spec.InvalidUsername(err.Error()),
+		}
+	}
+	return isAdmin, nil
+}
+
+func (t *LoginTypePassword) isLdapAdmin(conn *ldap.Conn, username string) (bool, error) {
+	searchRequest := ldap.NewSearchRequest(
+		t.Config.Ldap.AdminGroupDn,
+		ldap.ScopeWholeSubtree, ldap.DerefAlways, 0, 0, false,
+		strings.ReplaceAll(t.Config.Ldap.AdminGroupFilter, "{username}", username),
+		[]string{t.Config.Ldap.AdminGroupAttribute},
+		nil)
+
+	sr, err := conn.Search(searchRequest)
+	if err != nil {
+		return false, err
+	}
+
+	if len(sr.Entries) < 1 {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, username string, domain spec.ServerName, admin bool) (*api.Account, *util.JSONResponse) {
+	var existing api.QueryAccountByLocalpartResponse
+	err := t.UserAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{
+		Localpart:  username,
+		ServerName: domain,
+	}, &existing)
+
+	if err == nil {
+		return existing.Account, nil
+	}
+	if err != sql.ErrNoRows {
+		return nil, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: spec.InvalidUsername(err.Error()),
+		}
+	}
+
+	accountType := api.AccountTypeUser
+	if admin {
+		accountType = api.AccountTypeAdmin
+	}
+	var created api.PerformAccountCreationResponse
+	err = t.UserAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{
+		AppServiceID: "ldap",
+		Localpart:    username,
+		Password:     uuid.New().String(),
+		AccountType:  accountType,
+		OnConflict:   api.ConflictAbort,
+	}, &created)
+
+	if err != nil {
+		if _, ok := err.(*api.ErrorConflict); ok {
+			return nil, &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.UserInUse("Desired user ID is already taken."),
+			}
+		}
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("failed to create account: " + err.Error()),
+		}
+	}
+	return created.Account, nil
 }

clientapi/auth/user_interactive.go

@@ -20,9 +20,9 @@ import (
 	"net/http"
 	"sync"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
@@ -55,7 +55,7 @@ type LoginCleanupFunc func(context.Context, *util.JSONResponse)
 // https://matrix.org/docs/spec/client_server/r0.6.1#identifier-types
 type LoginIdentifier struct {
 	Type string `json:"type"`
-	// when type = m.id.user
+	// when type = m.id.user or m.id.application_service
 	User string `json:"user"`
 	// when type = m.id.thirdparty
 	Medium  string `json:"medium"`
@@ -113,8 +113,8 @@ type UserInteractive struct {
 
 func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
-		GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
-		Config:               cfg,
+		UserAPI: userAccountAPI,
+		Config:  cfg,
 	}
 	return &UserInteractive{
 		Flows: []userInteractiveFlow{
@@ -178,8 +178,10 @@ func (u *UserInteractive) NewSession() *util.JSONResponse {
 	sessionID, err := GenerateAccessToken()
 	if err != nil {
 		logrus.WithError(err).Error("failed to generate session ID")
-		res := jsonerror.InternalServerError()
-		return &res
+		return &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	u.Lock()
 	u.Sessions[sessionID] = []string{}
@@ -193,15 +195,19 @@ func (u *UserInteractive) ResponseWithChallenge(sessionID string, response inter
 	mixedObjects := make(map[string]interface{})
 	b, err := json.Marshal(response)
 	if err != nil {
-		ise := jsonerror.InternalServerError()
-		return &ise
+		return &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	_ = json.Unmarshal(b, &mixedObjects)
 	challenge := u.challenge(sessionID)
 	b, err = json.Marshal(challenge.JSON)
 	if err != nil {
-		ise := jsonerror.InternalServerError()
-		return &ise
+		return &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	_ = json.Unmarshal(b, &mixedObjects)
 
@@ -234,7 +240,7 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 	if !ok {
 		return nil, &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Unknown auth.type: " + authType),
+			JSON: spec.BadJSON("Unknown auth.type: " + authType),
 		}
 	}
 
@@ -250,7 +256,7 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 		if !u.IsSingleStageFlow(authType) {
 			return nil, &util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.Unknown("The auth.session is missing or unknown."),
+				JSON: spec.Unknown("The auth.session is missing or unknown."),
 			}
 		}
 	}

clientapi/auth/user_interactive_test.go

@@ -45,6 +45,14 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a
 	return nil
 }
 
+func (d *fakeAccountDatabase) QueryAccountByLocalpart(ctx context.Context, req *api.QueryAccountByLocalpartRequest, res *api.QueryAccountByLocalpartResponse) error {
+	return nil
+}
+
+func (d *fakeAccountDatabase) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error {
+	return nil
+}
+
 func setup() *UserInteractive {
 	cfg := &config.ClientAPI{
 		Matrix: &config.Global{

clientapi/clientapi_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/pushrules"
@@ -33,6 +34,7 @@ import (
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/stretchr/testify/assert"
 	"github.com/tidwall/gjson"
@@ -48,6 +50,10 @@ type userDevice struct {
 	password    string
 }
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestGetPutDevices(t *testing.T) {
 	alice := test.NewUser(t)
 	bob := test.NewUser(t)
@@ -119,7 +125,8 @@ func TestGetPutDevices(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -167,7 +174,8 @@ func TestDeleteDevice(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -271,7 +279,8 @@ func TestDeleteDevices(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -438,7 +447,7 @@ func TestSetDisplayname(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -550,7 +559,7 @@ func TestSetAvatarURL(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -628,7 +637,7 @@ func TestTyping(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -712,7 +721,7 @@ func TestMembership(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		rsAPI.SetUserAPI(userAPI)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -919,13 +928,17 @@ func TestCapabilities(t *testing.T) {
 		}
 	}
 
+	var tempRoomServerCfg config.RoomServer
+	tempRoomServerCfg.Defaults(config.DefaultOpts{})
+	defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion
+
 	expectedMap := map[string]interface{}{
 		"capabilities": map[string]interface{}{
 			"m.change_password": map[string]bool{
 				"enabled": true,
 			},
 			"m.room_versions": map[string]interface{}{
-				"default":   version.DefaultRoomVersion(),
+				"default":   defaultRoomVersion,
 				"available": versionsMap,
 			},
 		},
@@ -945,8 +958,10 @@ func TestCapabilities(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 		// Needed to create accounts
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -991,8 +1006,10 @@ func TestTurnserver(t *testing.T) {
 	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 	// Needed to create accounts
-	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+	rsAPI.SetFederationAPI(nil, nil)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	//rsAPI.SetUserAPI(userAPI)
 	// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 	AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1088,8 +1105,10 @@ func Test3PID(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 		// Needed to create accounts
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -1105,7 +1124,7 @@ func Test3PID(t *testing.T) {
 				resp := threepid.GetValidatedResponse{}
 				switch r.URL.Query().Get("client_secret") {
 				case "fail":
-					resp.ErrCode = "M_SESSION_NOT_VALIDATED"
+					resp.ErrCode = string(spec.ErrorSessionNotValidated)
 				case "fail2":
 					resp.ErrCode = "some other error"
 				case "fail3":
@@ -1264,7 +1283,8 @@ func TestPushRules(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1406,7 +1426,7 @@ func TestPushRules(t *testing.T) {
 				validateFunc: func(t *testing.T, respBody *bytes.Buffer) {
 					actions := gjson.GetBytes(respBody.Bytes(), "actions").Array()
 					// only a basic check
-					assert.Equal(t, 1, len(actions))
+					assert.Equal(t, 0, len(actions))
 				},
 			},
 			{
@@ -1650,7 +1670,8 @@ func TestKeys(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -2111,7 +2132,8 @@ func TestKeyBackup(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -2132,3 +2154,284 @@ func TestKeyBackup(t *testing.T) {
 		}
 	})
 }
+
+func TestGetMembership(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	testCases := []struct {
+		name             string
+		roomID           string
+		user             *test.User
+		additionalEvents func(t *testing.T, room *test.Room)
+		request          func(t *testing.T, room *test.Room, accessToken string) *http.Request
+		wantOK           bool
+		wantMemberCount  int
+	}{
+
+		{
+			name: "/joined_members - Bob never joined",
+			user: bob,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Alice joined",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK:          true,
+			wantMemberCount: 1,
+		},
+		{
+			name: "/joined_members - Alice leaves, shouldn't be able to see members ",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
+					"membership": "leave",
+				}, test.WithStateKey(alice.ID))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Bob joins, Alice sees two members",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+					"membership": "join",
+				}, test.WithStateKey(bob.ID))
+			},
+			wantOK:          true,
+			wantMemberCount: 2,
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				room := test.NewRoom(t, alice)
+				t.Cleanup(func() {
+					t.Logf("running cleanup for %s", tc.name)
+				})
+				// inject additional events
+				if tc.additionalEvents != nil {
+					tc.additionalEvents(t, room)
+				}
+				if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+					t.Fatalf("failed to send events: %v", err)
+				}
+
+				w := httptest.NewRecorder()
+				routers.Client.ServeHTTP(w, tc.request(t, room, accessTokens[tc.user].accessToken))
+				if w.Code != 200 && tc.wantOK {
+					t.Logf("%s", w.Body.String())
+					t.Fatalf("got HTTP %d want %d", w.Code, 200)
+				}
+				t.Logf("[%s] Resp: %s", tc.name, w.Body.String())
+
+				// check we got the expected events
+				if tc.wantOK {
+					memberCount := len(gjson.GetBytes(w.Body.Bytes(), "joined").Map())
+					if memberCount != tc.wantMemberCount {
+						t.Fatalf("expected %d members, got %d", tc.wantMemberCount, memberCount)
+					}
+				}
+			})
+		}
+	})
+}
+
+func TestCreateRoomInvite(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"invite": []string{bob.ID},
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/createRoom", strings.NewReader(string(body)))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+		routers.Client.ServeHTTP(w, req)
+
+		if w.Code != http.StatusOK {
+			t.Fatalf("expected room creation to be successful, got HTTP %d instead: %s", w.Code, w.Body.String())
+		}
+
+		roomID := gjson.GetBytes(w.Body.Bytes(), "room_id").Str
+		validRoomID, _ := spec.NewRoomID(roomID)
+		// Now ask the roomserver about the membership event of Bob
+		ev, err := rsAPI.CurrentStateEvent(context.Background(), *validRoomID, spec.MRoomMember, bob.ID)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if ev == nil {
+			t.Fatal("Membership event for Bob does not exist")
+		}
+
+		// Validate that there is NO displayname in content
+		if gjson.GetBytes(ev.Content(), "displayname").Exists() {
+			t.Fatal("Found displayname in invite")
+		}
+	})
+}
+
+func TestReportEvent(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	charlie := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(charlie.ID))
+	eventToReport := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice:   {},
+			bob:     {},
+			charlie: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		t.Run("Bob is not joined and should not be able to report the event", func(t *testing.T) {
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusNotFound {
+				t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Charlie is joined but the event does not exist", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/$doesNotExist", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusNotFound {
+				t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Charlie is joined and allowed to report the event", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected report to be successful, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+	})
+}

clientapi/httputil/httputil.go

@@ -20,7 +20,7 @@ import (
 	"net/http"
 	"unicode/utf8"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -32,8 +32,10 @@ func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONRespon
 	body, err := io.ReadAll(req.Body)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
-		resp := jsonerror.InternalServerError()
-		return &resp
+		return &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return UnmarshalJSON(body, iface)
@@ -43,7 +45,7 @@ func UnmarshalJSON(body []byte, iface interface{}) *util.JSONResponse {
 	if !utf8.Valid(body) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.NotJSON("Body contains invalid UTF-8"),
+			JSON: spec.NotJSON("Body contains invalid UTF-8"),
 		}
 	}
 
@@ -53,7 +55,7 @@ func UnmarshalJSON(body []byte, iface interface{}) *util.JSONResponse {
 		// valid JSON with incorrect types for values.
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
+			JSON: spec.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
 		}
 	}
 	return nil

clientapi/jsonerror/jsonerror.go

@@ -1,246 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package jsonerror
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
-)
-
-// MatrixError represents the "standard error response" in Matrix.
-// http://matrix.org/docs/spec/client_server/r0.2.0.html#api-standards
-type MatrixError struct {
-	ErrCode string `json:"errcode"`
-	Err     string `json:"error"`
-}
-
-func (e MatrixError) Error() string {
-	return fmt.Sprintf("%s: %s", e.ErrCode, e.Err)
-}
-
-// InternalServerError returns a 500 Internal Server Error in a matrix-compliant
-// format.
-func InternalServerError() util.JSONResponse {
-	return util.JSONResponse{
-		Code: http.StatusInternalServerError,
-		JSON: Unknown("Internal Server Error"),
-	}
-}
-
-// Unknown is an unexpected error
-func Unknown(msg string) *MatrixError {
-	return &MatrixError{"M_UNKNOWN", msg}
-}
-
-// Forbidden is an error when the client tries to access a resource
-// they are not allowed to access.
-func Forbidden(msg string) *MatrixError {
-	return &MatrixError{"M_FORBIDDEN", msg}
-}
-
-// BadJSON is an error when the client supplies malformed JSON.
-func BadJSON(msg string) *MatrixError {
-	return &MatrixError{"M_BAD_JSON", msg}
-}
-
-// BadAlias is an error when the client supplies a bad alias.
-func BadAlias(msg string) *MatrixError {
-	return &MatrixError{"M_BAD_ALIAS", msg}
-}
-
-// NotJSON is an error when the client supplies something that is not JSON
-// to a JSON endpoint.
-func NotJSON(msg string) *MatrixError {
-	return &MatrixError{"M_NOT_JSON", msg}
-}
-
-// NotFound is an error when the client tries to access an unknown resource.
-func NotFound(msg string) *MatrixError {
-	return &MatrixError{"M_NOT_FOUND", msg}
-}
-
-// MissingArgument is an error when the client tries to access a resource
-// without providing an argument that is required.
-func MissingArgument(msg string) *MatrixError {
-	return &MatrixError{"M_MISSING_ARGUMENT", msg}
-}
-
-// InvalidArgumentValue is an error when the client tries to provide an
-// invalid value for a valid argument
-func InvalidArgumentValue(msg string) *MatrixError {
-	return &MatrixError{"M_INVALID_ARGUMENT_VALUE", msg}
-}
-
-// MissingToken is an error when the client tries to access a resource which
-// requires authentication without supplying credentials.
-func MissingToken(msg string) *MatrixError {
-	return &MatrixError{"M_MISSING_TOKEN", msg}
-}
-
-// UnknownToken is an error when the client tries to access a resource which
-// requires authentication and supplies an unrecognised token
-func UnknownToken(msg string) *MatrixError {
-	return &MatrixError{"M_UNKNOWN_TOKEN", msg}
-}
-
-// WeakPassword is an error which is returned when the client tries to register
-// using a weak password. http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based
-func WeakPassword(msg string) *MatrixError {
-	return &MatrixError{"M_WEAK_PASSWORD", msg}
-}
-
-// InvalidUsername is an error returned when the client tries to register an
-// invalid username
-func InvalidUsername(msg string) *MatrixError {
-	return &MatrixError{"M_INVALID_USERNAME", msg}
-}
-
-// UserInUse is an error returned when the client tries to register an
-// username that already exists
-func UserInUse(msg string) *MatrixError {
-	return &MatrixError{"M_USER_IN_USE", msg}
-}
-
-// RoomInUse is an error returned when the client tries to make a room
-// that already exists
-func RoomInUse(msg string) *MatrixError {
-	return &MatrixError{"M_ROOM_IN_USE", msg}
-}
-
-// ASExclusive is an error returned when an application service tries to
-// register an username that is outside of its registered namespace, or if a
-// user attempts to register a username or room alias within an exclusive
-// namespace.
-func ASExclusive(msg string) *MatrixError {
-	return &MatrixError{"M_EXCLUSIVE", msg}
-}
-
-// GuestAccessForbidden is an error which is returned when the client is
-// forbidden from accessing a resource as a guest.
-func GuestAccessForbidden(msg string) *MatrixError {
-	return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg}
-}
-
-// InvalidSignature is an error which is returned when the client tries
-// to upload invalid signatures.
-func InvalidSignature(msg string) *MatrixError {
-	return &MatrixError{"M_INVALID_SIGNATURE", msg}
-}
-
-// InvalidParam is an error that is returned when a parameter was invalid,
-// traditionally with cross-signing.
-func InvalidParam(msg string) *MatrixError {
-	return &MatrixError{"M_INVALID_PARAM", msg}
-}
-
-// MissingParam is an error that is returned when a parameter was incorrect,
-// traditionally with cross-signing.
-func MissingParam(msg string) *MatrixError {
-	return &MatrixError{"M_MISSING_PARAM", msg}
-}
-
-// UnableToAuthoriseJoin is an error that is returned when a server can't
-// determine whether to allow a restricted join or not.
-func UnableToAuthoriseJoin(msg string) *MatrixError {
-	return &MatrixError{"M_UNABLE_TO_AUTHORISE_JOIN", msg}
-}
-
-// LeaveServerNoticeError is an error returned when trying to reject an invite
-// for a server notice room.
-func LeaveServerNoticeError() *MatrixError {
-	return &MatrixError{
-		ErrCode: "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM",
-		Err:     "You cannot reject this invite",
-	}
-}
-
-// ErrRoomKeysVersion is an error returned by `PUT /room_keys/keys`
-type ErrRoomKeysVersion struct {
-	MatrixError
-	CurrentVersion string `json:"current_version"`
-}
-
-// WrongBackupVersionError is an error returned by `PUT /room_keys/keys`
-func WrongBackupVersionError(currentVersion string) *ErrRoomKeysVersion {
-	return &ErrRoomKeysVersion{
-		MatrixError: MatrixError{
-			ErrCode: "M_WRONG_ROOM_KEYS_VERSION",
-			Err:     "Wrong backup version.",
-		},
-		CurrentVersion: currentVersion,
-	}
-}
-
-type IncompatibleRoomVersionError struct {
-	RoomVersion string `json:"room_version"`
-	Error       string `json:"error"`
-	Code        string `json:"errcode"`
-}
-
-// IncompatibleRoomVersion is an error which is returned when the client
-// requests a room with a version that is unsupported.
-func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *IncompatibleRoomVersionError {
-	return &IncompatibleRoomVersionError{
-		Code:        "M_INCOMPATIBLE_ROOM_VERSION",
-		RoomVersion: string(roomVersion),
-		Error:       "Your homeserver does not support the features required to join this room",
-	}
-}
-
-// UnsupportedRoomVersion is an error which is returned when the client
-// requests a room with a version that is unsupported.
-func UnsupportedRoomVersion(msg string) *MatrixError {
-	return &MatrixError{"M_UNSUPPORTED_ROOM_VERSION", msg}
-}
-
-// LimitExceededError is a rate-limiting error.
-type LimitExceededError struct {
-	MatrixError
-	RetryAfterMS int64 `json:"retry_after_ms,omitempty"`
-}
-
-// LimitExceeded is an error when the client tries to send events too quickly.
-func LimitExceeded(msg string, retryAfterMS int64) *LimitExceededError {
-	return &LimitExceededError{
-		MatrixError:  MatrixError{"M_LIMIT_EXCEEDED", msg},
-		RetryAfterMS: retryAfterMS,
-	}
-}
-
-// NotTrusted is an error which is returned when the client asks the server to
-// proxy a request (e.g. 3PID association) to a server that isn't trusted
-func NotTrusted(serverName string) *MatrixError {
-	return &MatrixError{
-		ErrCode: "M_SERVER_NOT_TRUSTED",
-		Err:     fmt.Sprintf("Untrusted server '%s'", serverName),
-	}
-}
-
-// InternalAPIError is returned when Dendrite failed to reach an internal API.
-func InternalAPIError(ctx context.Context, err error) util.JSONResponse {
-	logrus.WithContext(ctx).WithError(err).Error("Error reaching an internal API")
-	return util.JSONResponse{
-		Code: http.StatusInternalServerError,
-		JSON: &MatrixError{
-			ErrCode: "M_INTERNAL_SERVER_ERROR",
-			Err:     "Dendrite encountered an error reaching an internal API.",
-		},
-	}
-}

clientapi/jsonerror/jsonerror_test.go

@@ -1,44 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package jsonerror
-
-import (
-	"encoding/json"
-	"testing"
-)
-
-func TestLimitExceeded(t *testing.T) {
-	e := LimitExceeded("too fast", 5000)
-	jsonBytes, err := json.Marshal(&e)
-	if err != nil {
-		t.Fatalf("TestLimitExceeded: Failed to marshal LimitExceeded error. %s", err.Error())
-	}
-	want := `{"errcode":"M_LIMIT_EXCEEDED","error":"too fast","retry_after_ms":5000}`
-	if string(jsonBytes) != want {
-		t.Errorf("TestLimitExceeded: want %s, got %s", want, string(jsonBytes))
-	}
-}
-
-func TestForbidden(t *testing.T) {
-	e := Forbidden("you shall not pass")
-	jsonBytes, err := json.Marshal(&e)
-	if err != nil {
-		t.Fatalf("TestForbidden: Failed to marshal Forbidden error. %s", err.Error())
-	}
-	want := `{"errcode":"M_FORBIDDEN","error":"you shall not pass"}`
-	if string(jsonBytes) != want {
-		t.Errorf("TestForbidden: want %s, got %s", want, string(jsonBytes))
-	}
-}

clientapi/routing/account_data.go

@@ -21,11 +21,11 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/util"
 )
@@ -38,7 +38,7 @@ func GetAccountData(
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("userID does not match the current user"),
+			JSON: spec.Forbidden("userID does not match the current user"),
 		}
 	}
 
@@ -69,7 +69,7 @@ func GetAccountData(
 
 	return util.JSONResponse{
 		Code: http.StatusNotFound,
-		JSON: jsonerror.NotFound("data not found"),
+		JSON: spec.NotFound("data not found"),
 	}
 }
 
@@ -81,7 +81,7 @@ func SaveAccountData(
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("userID does not match the current user"),
+			JSON: spec.Forbidden("userID does not match the current user"),
 		}
 	}
 
@@ -90,27 +90,30 @@ func SaveAccountData(
 	if req.Body == http.NoBody {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.NotJSON("Content not JSON"),
+			JSON: spec.NotJSON("Content not JSON"),
 		}
 	}
 
 	if dataType == "m.fully_read" || dataType == "m.push_rules" {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden(fmt.Sprintf("Unable to modify %q using this API", dataType)),
+			JSON: spec.Forbidden(fmt.Sprintf("Unable to modify %q using this API", dataType)),
 		}
 	}
 
 	body, err := io.ReadAll(req.Body)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	if !json.Valid(body) {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Bad JSON content"),
+			JSON: spec.BadJSON("Bad JSON content"),
 		}
 	}
 
@@ -142,8 +145,16 @@ func SaveReadMarker(
 	userAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
 ) util.JSONResponse {
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("userID for this device is invalid"),
+		}
+	}
+
 	// Verify that the user is a member of this room
-	resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}
@@ -157,7 +168,10 @@ func SaveReadMarker(
 	if r.FullyRead != "" {
 		data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead})
 		if err != nil {
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 
 		dataReq := api.InputAccountDataRequest{

clientapi/routing/admin.go

@@ -6,6 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"regexp"
+	"strconv"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -16,15 +18,254 @@ import (
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/exp/constraints"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	clientapi "github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/userapi/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
+var validRegistrationTokenRegex = regexp.MustCompile("^[[:ascii:][:digit:]_]*$")
+
+func AdminCreateNewRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	if !cfg.RegistrationRequiresToken {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("Registration via tokens is not enabled on this homeserver"),
+		}
+	}
+	request := struct {
+		Token       string `json:"token"`
+		UsesAllowed *int32 `json:"uses_allowed,omitempty"`
+		ExpiryTime  *int64 `json:"expiry_time,omitempty"`
+		Length      int32  `json:"length"`
+	}{}
+
+	if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON(fmt.Sprintf("Failed to decode request body: %s", err)),
+		}
+	}
+
+	token := request.Token
+	usesAllowed := request.UsesAllowed
+	expiryTime := request.ExpiryTime
+	length := request.Length
+
+	if len(token) == 0 {
+		if length == 0 {
+			// length not provided in request. Assign default value of 16.
+			length = 16
+		}
+		// token not present in request body. Hence, generate a random token.
+		if length <= 0 || length > 64 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.BadJSON("length must be greater than zero and not greater than 64"),
+			}
+		}
+		token = util.RandomString(int(length))
+	}
+
+	if len(token) > 64 {
+		//Token present in request body, but is too long.
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("token must not be longer than 64"),
+		}
+	}
+
+	isTokenValid := validRegistrationTokenRegex.Match([]byte(token))
+	if !isTokenValid {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("token must consist only of characters matched by the regex [A-Za-z0-9-_]"),
+		}
+	}
+	// At this point, we have a valid token, either through request body or through random generation.
+	if usesAllowed != nil && *usesAllowed < 0 {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("uses_allowed must be a non-negative integer or null"),
+		}
+	}
+	if expiryTime != nil && spec.Timestamp(*expiryTime).Time().Before(time.Now()) {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("expiry_time must not be in the past"),
+		}
+	}
+	pending := int32(0)
+	completed := int32(0)
+	// If usesAllowed or expiryTime is 0, it means they are not present in the request. NULL (indicating unlimited uses / no expiration will be persisted in DB)
+	registrationToken := &clientapi.RegistrationToken{
+		Token:       &token,
+		UsesAllowed: usesAllowed,
+		Pending:     &pending,
+		Completed:   &completed,
+		ExpiryTime:  expiryTime,
+	}
+	created, err := userAPI.PerformAdminCreateRegistrationToken(req.Context(), registrationToken)
+	if !created {
+		return util.JSONResponse{
+			Code: http.StatusConflict,
+			JSON: map[string]string{
+				"error": fmt.Sprintf("token: %s already exists", token),
+			},
+		}
+	}
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: err,
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{
+			"token":        token,
+			"uses_allowed": getReturnValue(usesAllowed),
+			"pending":      pending,
+			"completed":    completed,
+			"expiry_time":  getReturnValue(expiryTime),
+		},
+	}
+}
+
+func getReturnValue[t constraints.Integer](in *t) any {
+	if in == nil {
+		return nil
+	}
+	return *in
+}
+
+func AdminListRegistrationTokens(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	queryParams := req.URL.Query()
+	returnAll := true
+	valid := true
+	validQuery, ok := queryParams["valid"]
+	if ok {
+		returnAll = false
+		validValue, err := strconv.ParseBool(validQuery[0])
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.BadJSON("invalid 'valid' query parameter"),
+			}
+		}
+		valid = validValue
+	}
+	tokens, err := userAPI.PerformAdminListRegistrationTokens(req.Context(), returnAll, valid)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.ErrorUnknown,
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{
+			"registration_tokens": tokens,
+		},
+	}
+}
+
+func AdminGetRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	tokenText := vars["token"]
+	token, err := userAPI.PerformAdminGetRegistrationToken(req.Context(), tokenText)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound(fmt.Sprintf("token: %s not found", tokenText)),
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: token,
+	}
+}
+
+func AdminDeleteRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	tokenText := vars["token"]
+	err = userAPI.PerformAdminDeleteRegistrationToken(req.Context(), tokenText)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: err,
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{},
+	}
+}
+
+func AdminUpdateRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	tokenText := vars["token"]
+	request := make(map[string]*int64)
+	if err = json.NewDecoder(req.Body).Decode(&request); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON(fmt.Sprintf("Failed to decode request body: %s", err)),
+		}
+	}
+	newAttributes := make(map[string]interface{})
+	usesAllowed, ok := request["uses_allowed"]
+	if ok {
+		// Only add usesAllowed to newAtrributes if it is present and valid
+		if usesAllowed != nil && *usesAllowed < 0 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.BadJSON("uses_allowed must be a non-negative integer or null"),
+			}
+		}
+		newAttributes["usesAllowed"] = usesAllowed
+	}
+	expiryTime, ok := request["expiry_time"]
+	if ok {
+		// Only add expiryTime to newAtrributes if it is present and valid
+		if expiryTime != nil && spec.Timestamp(*expiryTime).Time().Before(time.Now()) {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.BadJSON("expiry_time must not be in the past"),
+			}
+		}
+		newAttributes["expiryTime"] = expiryTime
+	}
+	if len(newAttributes) == 0 {
+		// No attributes to update. Return existing token
+		return AdminGetRegistrationToken(req, cfg, userAPI)
+	}
+	updatedToken, err := userAPI.PerformAdminUpdateRegistrationToken(req.Context(), tokenText, newAttributes)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound(fmt.Sprintf("token: %s not found", tokenText)),
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: *updatedToken,
+	}
+}
+
 func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
@@ -32,12 +273,12 @@ func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAP
 	}
 
 	affected, err := rsAPI.PerformAdminEvacuateRoom(req.Context(), vars["roomID"])
-	switch err {
+	switch err.(type) {
 	case nil:
 	case eventutil.ErrRoomNoExists:
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(err.Error()),
+			JSON: spec.NotFound(err.Error()),
 		}
 	default:
 		logrus.WithError(err).WithField("roomID", vars["roomID"]).Error("Failed to evacuate room")
@@ -91,7 +332,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.De
 	if req.Body == nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown("Missing request body"),
+			JSON: spec.Unknown("Missing request body"),
 		}
 	}
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -104,7 +345,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.De
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+			JSON: spec.InvalidParam(err.Error()),
 		}
 	}
 	accAvailableResp := &api.QueryAccountAvailabilityResponse{}
@@ -114,28 +355,29 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.De
 	}, accAvailableResp); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.InternalAPIError(req.Context(), err),
+			JSON: spec.InternalServerError{},
 		}
 	}
 	if accAvailableResp.Available {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.Unknown("User does not exist"),
+			JSON: spec.Unknown("User does not exist"),
 		}
 	}
 	request := struct {
-		Password string `json:"password"`
+		Password      string `json:"password"`
+		LogoutDevices bool   `json:"logout_devices"`
 	}{}
 	if err = json.NewDecoder(req.Body).Decode(&request); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown("Failed to decode request body: " + err.Error()),
+			JSON: spec.Unknown("Failed to decode request body: " + err.Error()),
 		}
 	}
 	if request.Password == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting non-empty password."),
+			JSON: spec.MissingParam("Expecting non-empty password."),
 		}
 	}
 
@@ -147,13 +389,13 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.De
 		Localpart:     localpart,
 		ServerName:    serverName,
 		Password:      request.Password,
-		LogoutDevices: true,
+		LogoutDevices: request.LogoutDevices,
 	}
 	updateRes := &api.PerformPasswordUpdateResponse{}
 	if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown("Failed to perform password update: " + err.Error()),
+			JSON: spec.Unknown("Failed to perform password update: " + err.Error()),
 		}
 	}
 	return util.JSONResponse{
@@ -170,7 +412,10 @@ func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *api.Device,
 	_, err := natsClient.RequestMsg(nats.NewMsg(cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex)), time.Second*10)
 	if err != nil {
 		logrus.WithError(err).Error("failed to publish nats message")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -192,7 +437,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien
 	if cfg.Matrix.IsLocalServerName(domain) {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidParam("Can not mark local device list as stale"),
+			JSON: spec.InvalidParam("Can not mark local device list as stale"),
 		}
 	}
 
@@ -203,7 +448,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown(fmt.Sprintf("Failed to mark device list as stale: %s", err)),
+			JSON: spec.Unknown(fmt.Sprintf("Failed to mark device list as stale: %s", err)),
 		}
 	}
 	return util.JSONResponse{
@@ -221,21 +466,21 @@ func AdminDownloadState(req *http.Request, device *api.Device, rsAPI roomserverA
 	if !ok {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting room ID."),
+			JSON: spec.MissingParam("Expecting room ID."),
 		}
 	}
 	serverName, ok := vars["serverName"]
 	if !ok {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting remote server name."),
+			JSON: spec.MissingParam("Expecting remote server name."),
 		}
 	}
 	if err = rsAPI.PerformAdminDownloadState(req.Context(), roomID, device.UserID, spec.ServerName(serverName)); err != nil {
-		if errors.Is(err, eventutil.ErrRoomNoExists) {
+		if errors.Is(err, eventutil.ErrRoomNoExists{}) {
 			return util.JSONResponse{
 				Code: 200,
-				JSON: jsonerror.NotFound(eventutil.ErrRoomNoExists.Error()),
+				JSON: spec.NotFound(err.Error()),
 			}
 		}
 		logrus.WithError(err).WithFields(logrus.Fields{
@@ -250,3 +495,93 @@ func AdminDownloadState(req *http.Request, device *api.Device, rsAPI roomserverA
 		JSON: struct{}{},
 	}
 }
+
+// GetEventReports returns reported events for a given user/room.
+func GetEventReports(
+	req *http.Request,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
+	from, limit uint64,
+	backwards bool,
+	userID, roomID string,
+) util.JSONResponse {
+
+	eventReports, count, err := rsAPI.QueryAdminEventReports(req.Context(), from, limit, backwards, userID, roomID)
+	if err != nil {
+		logrus.WithError(err).Error("failed to query event reports")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	resp := map[string]any{
+		"event_reports": eventReports,
+		"total":         count,
+	}
+
+	// Add a next_token if there are still reports
+	if int64(from+limit) < count {
+		resp["next_token"] = int(from) + len(eventReports)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: resp,
+	}
+}
+
+func GetEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
+	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			// Given this is an admin endpoint, let them know what didn't work.
+			JSON: spec.InvalidParam(err.Error()),
+		}
+	}
+
+	report, err := rsAPI.QueryAdminEventReport(req.Context(), parsedReportID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown(err.Error()),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: report,
+	}
+}
+
+func DeleteEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
+	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			// Given this is an admin endpoint, let them know what didn't work.
+			JSON: spec.InvalidParam(err.Error()),
+		}
+	}
+
+	err = rsAPI.PerformAdminDeleteEventReport(req.Context(), parsedReportID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown(err.Error()),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func parseUint64OrDefault(input string, defaultValue uint64) uint64 {
+	v, err := strconv.ParseUint(input, 10, 64)
+	if err != nil {
+		return defaultValue
+	}
+	return v
+}

clientapi/routing/admin_whois.go

@@ -17,8 +17,8 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/util"
 )
@@ -51,7 +51,7 @@ func GetAdminWhois(
 	if !allowed {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("userID does not match the current user"),
+			JSON: spec.Forbidden("userID does not match the current user"),
 		}
 	}
 
@@ -61,7 +61,10 @@ func GetAdminWhois(
 	}, &queryRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("GetAdminWhois failed to query user devices")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	devices := make(map[string]deviceInfo)

clientapi/routing/aliases.go

@@ -19,12 +19,10 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
-
 	"github.com/matrix-org/util"
 )
 
@@ -57,19 +55,29 @@ func GetAliases(
 		visibility = content.HistoryVisibility
 	}
 	if visibility != spec.WorldReadable {
+		deviceUserID, err := spec.NewUserID(device.UserID, true)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
+			}
+		}
 		queryReq := api.QueryMembershipForUserRequest{
 			RoomID: roomID,
-			UserID: device.UserID,
+			UserID: *deviceUserID,
 		}
 		var queryRes api.QueryMembershipForUserResponse
 		if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		if !queryRes.IsInRoom {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("You aren't a member of this room."),
+				JSON: spec.Forbidden("You aren't a member of this room."),
 			}
 		}
 	}

clientapi/routing/capabilities.go

@@ -17,6 +17,7 @@ package routing
 import (
 	"net/http"
 
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -24,7 +25,7 @@ import (
 
 // GetCapabilities returns information about the server's supported feature set
 // and other relevant capabilities to an authenticated user.
-func GetCapabilities() util.JSONResponse {
+func GetCapabilities(rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	versionsMap := map[gomatrixserverlib.RoomVersion]string{}
 	for v, desc := range version.SupportedRoomVersions() {
 		if desc.Stable() {
@@ -40,7 +41,7 @@ func GetCapabilities() util.JSONResponse {
 				"enabled": true,
 			},
 			"m.room_versions": map[string]interface{}{
-				"default":   version.DefaultRoomVersion(),
+				"default":   rsAPI.DefaultRoomVersion(),
 				"available": versionsMap,
 			},
 		},

clientapi/routing/createroom.go

@@ -22,18 +22,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/getsentry/sentry-go"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/types"
 	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -42,32 +37,19 @@ import (
 
 // https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom
 type createRoomRequest struct {
-	Invite                    []string                      `json:"invite"`
-	Name                      string                        `json:"name"`
-	Visibility                string                        `json:"visibility"`
-	Topic                     string                        `json:"topic"`
-	Preset                    string                        `json:"preset"`
-	CreationContent           json.RawMessage               `json:"creation_content"`
-	InitialState              []fledglingEvent              `json:"initial_state"`
-	RoomAliasName             string                        `json:"room_alias_name"`
-	RoomVersion               gomatrixserverlib.RoomVersion `json:"room_version"`
-	PowerLevelContentOverride json.RawMessage               `json:"power_level_content_override"`
-	IsDirect                  bool                          `json:"is_direct"`
+	Invite                    []string                           `json:"invite"`
+	Name                      string                             `json:"name"`
+	Visibility                string                             `json:"visibility"`
+	Topic                     string                             `json:"topic"`
+	Preset                    string                             `json:"preset"`
+	CreationContent           json.RawMessage                    `json:"creation_content"`
+	InitialState              []gomatrixserverlib.FledglingEvent `json:"initial_state"`
+	RoomAliasName             string                             `json:"room_alias_name"`
+	RoomVersion               gomatrixserverlib.RoomVersion      `json:"room_version"`
+	PowerLevelContentOverride json.RawMessage                    `json:"power_level_content_override"`
+	IsDirect                  bool                               `json:"is_direct"`
 }
 
-const (
-	presetPrivateChat        = "private_chat"
-	presetTrustedPrivateChat = "trusted_private_chat"
-	presetPublicChat         = "public_chat"
-)
-
-const (
-	historyVisibilityShared = "shared"
-	// TODO: These should be implemented once history visibility is implemented
-	// historyVisibilityWorldReadable = "world_readable"
-	// historyVisibilityInvited       = "invited"
-)
-
 func (r createRoomRequest) Validate() *util.JSONResponse {
 	whitespace := "\t\n\x0b\x0c\r " // https://docs.python.org/2/library/string.html#string.whitespace
 	// https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/handlers/room.py#L81
@@ -75,28 +57,23 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 	if strings.ContainsAny(r.RoomAliasName, whitespace+":") {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("room_alias_name cannot contain whitespace or ':'"),
+			JSON: spec.BadJSON("room_alias_name cannot contain whitespace or ':'"),
 		}
 	}
 	for _, userID := range r.Invite {
-		// TODO: We should put user ID parsing code into gomatrixserverlib and use that instead
-		//       (see https://github.com/matrix-org/gomatrixserverlib/blob/3394e7c7003312043208aa73727d2256eea3d1f6/eventcontent.go#L347 )
-		//       It should be a struct (with pointers into a single string to avoid copying) and
-		//       we should update all refs to use UserID types rather than strings.
-		// https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/types.py#L92
-		if _, _, err := gomatrixserverlib.SplitID('@', userID); err != nil {
+		if _, err := spec.NewUserID(userID, true); err != nil {
 			return &util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.BadJSON("user id must be in the form @localpart:domain"),
+				JSON: spec.BadJSON("user id must be in the form @localpart:domain"),
 			}
 		}
 	}
 	switch r.Preset {
-	case presetPrivateChat, presetTrustedPrivateChat, presetPublicChat, "":
+	case spec.PresetPrivateChat, spec.PresetTrustedPrivateChat, spec.PresetPublicChat, "":
 	default:
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("preset must be any of 'private_chat', 'trusted_private_chat', 'public_chat'"),
+			JSON: spec.BadJSON("preset must be any of 'private_chat', 'trusted_private_chat', 'public_chat'"),
 		}
 	}
 
@@ -108,7 +85,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 	if err != nil {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("malformed creation_content"),
+			JSON: spec.BadJSON("malformed creation_content"),
 		}
 	}
 
@@ -117,7 +94,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 	if err != nil {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("malformed creation_content"),
+			JSON: spec.BadJSON("malformed creation_content"),
 		}
 	}
 
@@ -130,13 +107,6 @@ type createRoomResponse struct {
 	RoomAlias string `json:"room_alias,omitempty"` // in synapse not spec
 }
 
-// fledglingEvent is a helper representation of an event used when creating many events in succession.
-type fledglingEvent struct {
-	Type     string      `json:"type"`
-	StateKey string      `json:"state_key"`
-	Content  interface{} `json:"content"`
-}
-
 // CreateRoom implements /createRoom
 func CreateRoom(
 	req *http.Request, device *api.Device,
@@ -144,478 +114,124 @@ func CreateRoom(
 	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
-	var r createRoomRequest
-	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	var createRequest createRoomRequest
+	resErr := httputil.UnmarshalJSONRequest(req, &createRequest)
 	if resErr != nil {
 		return *resErr
 	}
-	if resErr = r.Validate(); resErr != nil {
+	if resErr = createRequest.Validate(); resErr != nil {
 		return *resErr
 	}
 	evTime, err := httputil.ParseTSParam(req)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+			JSON: spec.InvalidParam(err.Error()),
 		}
 	}
-	return createRoom(req.Context(), r, device, cfg, profileAPI, rsAPI, asAPI, evTime)
+	return createRoom(req.Context(), createRequest, device, cfg, profileAPI, rsAPI, asAPI, evTime)
 }
 
 // createRoom implements /createRoom
-// nolint: gocyclo
 func createRoom(
 	ctx context.Context,
-	r createRoomRequest, device *api.Device,
+	createRequest createRoomRequest, device *api.Device,
 	cfg *config.ClientAPI,
 	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 	evTime time.Time,
 ) util.JSONResponse {
-	_, userDomain, err := gomatrixserverlib.SplitID('@', device.UserID)
+	userID, err := spec.NewUserID(device.UserID, true)
 	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+		util.GetLogger(ctx).WithError(err).Error("invalid userID")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
-	if !cfg.Matrix.IsLocalServerName(userDomain) {
+	if !cfg.Matrix.IsLocalServerName(userID.Domain()) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden(fmt.Sprintf("User domain %q not configured locally", userDomain)),
+			JSON: spec.Forbidden(fmt.Sprintf("User domain %q not configured locally", userID.Domain())),
 		}
 	}
 
-	// TODO (#267): Check room ID doesn't clash with an existing one, and we
-	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
-	roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), userDomain)
-
 	logger := util.GetLogger(ctx)
-	userID := device.UserID
+
+	// TODO: Check room ID doesn't clash with an existing one, and we
+	//       probably shouldn't be using pseudo-random strings, maybe GUIDs?
+	roomID, err := spec.NewRoomID(fmt.Sprintf("!%s:%s", util.RandomString(16), userID.Domain()))
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("invalid roomID")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
 
 	// Clobber keys: creator, room_version
 
-	roomVersion := roomserverVersion.DefaultRoomVersion()
-	if r.RoomVersion != "" {
-		candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion)
+	roomVersion := rsAPI.DefaultRoomVersion()
+	if createRequest.RoomVersion != "" {
+		candidateVersion := gomatrixserverlib.RoomVersion(createRequest.RoomVersion)
 		_, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion)
 		if roomVersionError != nil {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.UnsupportedRoomVersion(roomVersionError.Error()),
+				JSON: spec.UnsupportedRoomVersion(roomVersionError.Error()),
 			}
 		}
 		roomVersion = candidateVersion
 	}
 
-	// TODO: visibility/presets/raw initial state
-	// TODO: Create room alias association
-	// Make sure this doesn't fall into an application service's namespace though!
-
 	logger.WithFields(log.Fields{
-		"userID":      userID,
-		"roomID":      roomID,
+		"userID":      userID.String(),
+		"roomID":      roomID.String(),
 		"roomVersion": roomVersion,
 	}).Info("Creating new room")
 
-	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI)
+	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID.String(), asAPI, profileAPI)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed")
-		return jsonerror.InternalServerError()
-	}
-
-	createContent := map[string]interface{}{}
-	if len(r.CreationContent) > 0 {
-		if err = json.Unmarshal(r.CreationContent, &createContent); err != nil {
-			util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for creation_content failed")
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.BadJSON("invalid create content"),
-			}
-		}
-	}
-	createContent["creator"] = userID
-	createContent["room_version"] = roomVersion
-	powerLevelContent := eventutil.InitialPowerLevelsContent(userID)
-	joinRuleContent := gomatrixserverlib.JoinRuleContent{
-		JoinRule: spec.Invite,
-	}
-	historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{
-		HistoryVisibility: historyVisibilityShared,
-	}
-
-	if r.PowerLevelContentOverride != nil {
-		// Merge powerLevelContentOverride fields by unmarshalling it atop the defaults
-		err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent)
-		if err != nil {
-			util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for power_level_content_override failed")
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.BadJSON("malformed power_level_content_override"),
-			}
-		}
-	}
-
-	var guestsCanJoin bool
-	switch r.Preset {
-	case presetPrivateChat:
-		joinRuleContent.JoinRule = spec.Invite
-		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
-		guestsCanJoin = true
-	case presetTrustedPrivateChat:
-		joinRuleContent.JoinRule = spec.Invite
-		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
-		for _, invitee := range r.Invite {
-			powerLevelContent.Users[invitee] = 100
-		}
-		guestsCanJoin = true
-	case presetPublicChat:
-		joinRuleContent.JoinRule = spec.Public
-		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
-	}
-
-	createEvent := fledglingEvent{
-		Type:    spec.MRoomCreate,
-		Content: createContent,
-	}
-	powerLevelEvent := fledglingEvent{
-		Type:    spec.MRoomPowerLevels,
-		Content: powerLevelContent,
-	}
-	joinRuleEvent := fledglingEvent{
-		Type:    spec.MRoomJoinRules,
-		Content: joinRuleContent,
-	}
-	historyVisibilityEvent := fledglingEvent{
-		Type:    spec.MRoomHistoryVisibility,
-		Content: historyVisibilityContent,
-	}
-	membershipEvent := fledglingEvent{
-		Type:     spec.MRoomMember,
-		StateKey: userID,
-		Content: gomatrixserverlib.MemberContent{
-			Membership:  spec.Join,
-			DisplayName: profile.DisplayName,
-			AvatarURL:   profile.AvatarURL,
-		},
-	}
-
-	var nameEvent *fledglingEvent
-	var topicEvent *fledglingEvent
-	var guestAccessEvent *fledglingEvent
-	var aliasEvent *fledglingEvent
-
-	if r.Name != "" {
-		nameEvent = &fledglingEvent{
-			Type: spec.MRoomName,
-			Content: eventutil.NameContent{
-				Name: r.Name,
-			},
-		}
-	}
-
-	if r.Topic != "" {
-		topicEvent = &fledglingEvent{
-			Type: spec.MRoomTopic,
-			Content: eventutil.TopicContent{
-				Topic: r.Topic,
-			},
-		}
-	}
-
-	if guestsCanJoin {
-		guestAccessEvent = &fledglingEvent{
-			Type: spec.MRoomGuestAccess,
-			Content: eventutil.GuestAccessContent{
-				GuestAccess: "can_join",
-			},
-		}
-	}
-
-	var roomAlias string
-	if r.RoomAliasName != "" {
-		roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, userDomain)
-		// check it's free TODO: This races but is better than nothing
-		hasAliasReq := roomserverAPI.GetRoomIDForAliasRequest{
-			Alias:              roomAlias,
-			IncludeAppservices: false,
-		}
-
-		var aliasResp roomserverAPI.GetRoomIDForAliasResponse
-		err = rsAPI.GetRoomIDForAlias(ctx, &hasAliasReq, &aliasResp)
-		if err != nil {
-			util.GetLogger(ctx).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed")
-			return jsonerror.InternalServerError()
-		}
-		if aliasResp.RoomID != "" {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.RoomInUse("Room ID already exists."),
-			}
-		}
-
-		aliasEvent = &fledglingEvent{
-			Type: spec.MRoomCanonicalAlias,
-			Content: eventutil.CanonicalAlias{
-				Alias: roomAlias,
-			},
-		}
-	}
-
-	var initialStateEvents []fledglingEvent
-	for i := range r.InitialState {
-		if r.InitialState[i].StateKey != "" {
-			initialStateEvents = append(initialStateEvents, r.InitialState[i])
-			continue
-		}
-
-		switch r.InitialState[i].Type {
-		case spec.MRoomCreate:
-			continue
-
-		case spec.MRoomPowerLevels:
-			powerLevelEvent = r.InitialState[i]
-
-		case spec.MRoomJoinRules:
-			joinRuleEvent = r.InitialState[i]
-
-		case spec.MRoomHistoryVisibility:
-			historyVisibilityEvent = r.InitialState[i]
-
-		case spec.MRoomGuestAccess:
-			guestAccessEvent = &r.InitialState[i]
-
-		case spec.MRoomName:
-			nameEvent = &r.InitialState[i]
-
-		case spec.MRoomTopic:
-			topicEvent = &r.InitialState[i]
-
-		default:
-			initialStateEvents = append(initialStateEvents, r.InitialState[i])
-		}
-	}
-
-	// send events into the room in order of:
-	//  1- m.room.create
-	//  2- room creator join member
-	//  3- m.room.power_levels
-	//  4- m.room.join_rules
-	//  5- m.room.history_visibility
-	//  6- m.room.canonical_alias (opt)
-	//  7- m.room.guest_access (opt)
-	//  8- other initial state items
-	//  9- m.room.name (opt)
-	//  10- m.room.topic (opt)
-	//  11- invite events (opt) - with is_direct flag if applicable TODO
-	//  12- 3pid invite events (opt) TODO
-	// This differs from Synapse slightly. Synapse would vary the ordering of 3-7
-	// depending on if those events were in "initial_state" or not. This made it
-	// harder to reason about, hence sticking to a strict static ordering.
-	// TODO: Synapse has txn/token ID on each event. Do we need to do this here?
-	eventsToMake := []fledglingEvent{
-		createEvent, membershipEvent, powerLevelEvent, joinRuleEvent, historyVisibilityEvent,
-	}
-	if guestAccessEvent != nil {
-		eventsToMake = append(eventsToMake, *guestAccessEvent)
-	}
-	eventsToMake = append(eventsToMake, initialStateEvents...)
-	if nameEvent != nil {
-		eventsToMake = append(eventsToMake, *nameEvent)
-	}
-	if topicEvent != nil {
-		eventsToMake = append(eventsToMake, *topicEvent)
-	}
-	if aliasEvent != nil {
-		// TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room.
-		// This means we might fail creating the alias but say the canonical alias is something that doesn't exist.
-		eventsToMake = append(eventsToMake, *aliasEvent)
-	}
-
-	// TODO: invite events
-	// TODO: 3pid invite events
-
-	verImpl, err := gomatrixserverlib.GetRoomVersion(roomVersion)
-	if err != nil {
 		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("unknown room version"),
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
 		}
 	}
 
-	var builtEvents []*types.HeaderedEvent
-	authEvents := gomatrixserverlib.NewAuthEvents(nil)
-	for i, e := range eventsToMake {
-		depth := i + 1 // depth starts at 1
+	userDisplayName := profile.DisplayName
+	userAvatarURL := profile.AvatarURL
 
-		builder := verImpl.NewEventBuilderFromProtoEvent(&gomatrixserverlib.ProtoEvent{
-			Sender:   userID,
-			RoomID:   roomID,
-			Type:     e.Type,
-			StateKey: &e.StateKey,
-			Depth:    int64(depth),
-		})
-		err = builder.SetContent(e.Content)
-		if err != nil {
-			util.GetLogger(ctx).WithError(err).Error("builder.SetContent failed")
-			return jsonerror.InternalServerError()
-		}
-		if i > 0 {
-			builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
-		}
-		var ev gomatrixserverlib.PDU
-		if err = builder.AddAuthEvents(&authEvents); err != nil {
-			util.GetLogger(ctx).WithError(err).Error("AddAuthEvents failed")
-			return jsonerror.InternalServerError()
-		}
-		ev, err = builder.Build(evTime, userDomain, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey)
-		if err != nil {
-			util.GetLogger(ctx).WithError(err).Error("buildEvent failed")
-			return jsonerror.InternalServerError()
-		}
+	keyID := cfg.Matrix.KeyID
+	privateKey := cfg.Matrix.PrivateKey
 
-		if err = gomatrixserverlib.Allowed(ev, &authEvents); err != nil {
-			util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.Allowed failed")
-			return jsonerror.InternalServerError()
-		}
+	req := roomserverAPI.PerformCreateRoomRequest{
+		InvitedUsers:              createRequest.Invite,
+		RoomName:                  createRequest.Name,
+		Visibility:                createRequest.Visibility,
+		Topic:                     createRequest.Topic,
+		StatePreset:               createRequest.Preset,
+		CreationContent:           createRequest.CreationContent,
+		InitialState:              createRequest.InitialState,
+		RoomAliasName:             createRequest.RoomAliasName,
+		RoomVersion:               roomVersion,
+		PowerLevelContentOverride: createRequest.PowerLevelContentOverride,
+		IsDirect:                  createRequest.IsDirect,
 
-		// Add the event to the list of auth events
-		builtEvents = append(builtEvents, &types.HeaderedEvent{PDU: ev})
-		err = authEvents.AddEvent(ev)
-		if err != nil {
-			util.GetLogger(ctx).WithError(err).Error("authEvents.AddEvent failed")
-			return jsonerror.InternalServerError()
-		}
+		UserDisplayName: userDisplayName,
+		UserAvatarURL:   userAvatarURL,
+		KeyID:           keyID,
+		PrivateKey:      privateKey,
+		EventTime:       evTime,
 	}
 
-	inputs := make([]roomserverAPI.InputRoomEvent, 0, len(builtEvents))
-	for _, event := range builtEvents {
-		inputs = append(inputs, roomserverAPI.InputRoomEvent{
-			Kind:         roomserverAPI.KindNew,
-			Event:        event,
-			Origin:       userDomain,
-			SendAsServer: roomserverAPI.DoNotSendToOtherServers,
-		})
-	}
-	if err = roomserverAPI.SendInputRoomEvents(ctx, rsAPI, device.UserDomain(), inputs, false); err != nil {
-		util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed")
-		return jsonerror.InternalServerError()
-	}
-
-	// TODO(#269): Reserve room alias while we create the room. This stops us
-	// from creating the room but still failing due to the alias having already
-	// been taken.
-	if roomAlias != "" {
-		aliasReq := roomserverAPI.SetRoomAliasRequest{
-			Alias:  roomAlias,
-			RoomID: roomID,
-			UserID: userID,
-		}
-
-		var aliasResp roomserverAPI.SetRoomAliasResponse
-		err = rsAPI.SetRoomAlias(ctx, &aliasReq, &aliasResp)
-		if err != nil {
-			util.GetLogger(ctx).WithError(err).Error("aliasAPI.SetRoomAlias failed")
-			return jsonerror.InternalServerError()
-		}
-
-		if aliasResp.AliasExists {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.RoomInUse("Room alias already exists."),
-			}
-		}
-	}
-
-	// If this is a direct message then we should invite the participants.
-	if len(r.Invite) > 0 {
-		// Build some stripped state for the invite.
-		var globalStrippedState []fclient.InviteV2StrippedState
-		for _, event := range builtEvents {
-			// Chosen events from the spec:
-			// https://spec.matrix.org/v1.3/client-server-api/#stripped-state
-			switch event.Type() {
-			case spec.MRoomCreate:
-				fallthrough
-			case spec.MRoomName:
-				fallthrough
-			case spec.MRoomAvatar:
-				fallthrough
-			case spec.MRoomTopic:
-				fallthrough
-			case spec.MRoomCanonicalAlias:
-				fallthrough
-			case spec.MRoomEncryption:
-				fallthrough
-			case spec.MRoomMember:
-				fallthrough
-			case spec.MRoomJoinRules:
-				ev := event.PDU
-				globalStrippedState = append(
-					globalStrippedState,
-					fclient.NewInviteV2StrippedState(ev),
-				)
-			}
-		}
-
-		// Process the invites.
-		var inviteEvent *types.HeaderedEvent
-		for _, invitee := range r.Invite {
-			// Build the invite event.
-			inviteEvent, err = buildMembershipEvent(
-				ctx, invitee, "", profileAPI, device, spec.Invite,
-				roomID, r.IsDirect, cfg, evTime, rsAPI, asAPI,
-			)
-			if err != nil {
-				util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
-				continue
-			}
-			inviteStrippedState := append(
-				globalStrippedState,
-				fclient.NewInviteV2StrippedState(inviteEvent.PDU),
-			)
-			// Send the invite event to the roomserver.
-			event := inviteEvent
-			err = rsAPI.PerformInvite(ctx, &roomserverAPI.PerformInviteRequest{
-				Event:           event,
-				InviteRoomState: inviteStrippedState,
-				RoomVersion:     event.Version(),
-				SendAsServer:    string(userDomain),
-			})
-			switch e := err.(type) {
-			case roomserverAPI.ErrInvalidID:
-				return util.JSONResponse{
-					Code: http.StatusBadRequest,
-					JSON: jsonerror.Unknown(e.Error()),
-				}
-			case roomserverAPI.ErrNotAllowed:
-				return util.JSONResponse{
-					Code: http.StatusForbidden,
-					JSON: jsonerror.Forbidden(e.Error()),
-				}
-			case nil:
-			default:
-				util.GetLogger(ctx).WithError(err).Error("PerformInvite failed")
-				sentry.CaptureException(err)
-				return util.JSONResponse{
-					Code: http.StatusInternalServerError,
-					JSON: jsonerror.InternalServerError(),
-				}
-			}
-		}
-	}
-
-	if r.Visibility == spec.Public {
-		// expose this room in the published room list
-		if err = rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{
-			RoomID:     roomID,
-			Visibility: spec.Public,
-		}); err != nil {
-			util.GetLogger(ctx).WithError(err).Error("failed to publish room")
-			return jsonerror.InternalServerError()
-		}
+	roomAlias, createRes := rsAPI.PerformCreateRoom(ctx, *userID, *roomID, &req)
+	if createRes != nil {
+		return *createRes
 	}
 
 	response := createRoomResponse{
-		RoomID:    roomID,
+		RoomID:    roomID.String(),
 		RoomAlias: roomAlias,
 	}
 

clientapi/routing/deactivate.go

@@ -5,9 +5,9 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -24,7 +24,7 @@ func Deactivate(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
+			JSON: spec.BadJSON("The request body could not be read: " + err.Error()),
 		}
 	}
 
@@ -36,7 +36,10 @@ func Deactivate(
 	localpart, serverName, err := gomatrixserverlib.SplitID('@', login.Username())
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	var res api.PerformAccountDeactivationResponse
@@ -46,7 +49,10 @@ func Deactivate(
 	}, &res)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformAccountDeactivation failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/device.go

@@ -22,9 +22,9 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/tidwall/gjson"
 )
@@ -60,7 +60,10 @@ func GetDeviceByID(
 	}, &queryRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryDevices failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	var targetDevice *api.Device
 	for _, device := range queryRes.Devices {
@@ -72,7 +75,7 @@ func GetDeviceByID(
 	if targetDevice == nil {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Unknown device"),
+			JSON: spec.NotFound("Unknown device"),
 		}
 	}
 
@@ -97,7 +100,10 @@ func GetDevicesByLocalpart(
 	}, &queryRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryDevices failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	res := devicesJSON{}
@@ -139,12 +145,15 @@ func UpdateDeviceByID(
 	}, &performRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("PerformDeviceUpdate failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if !performRes.DeviceExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.Forbidden("device does not exist"),
+			JSON: spec.Forbidden("device does not exist"),
 		}
 	}
 
@@ -174,7 +183,7 @@ func DeleteDeviceById(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
+			JSON: spec.BadJSON("The request body could not be read: " + err.Error()),
 		}
 	}
 
@@ -184,7 +193,7 @@ func DeleteDeviceById(
 		if dev != deviceID {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("session and device mismatch"),
+				JSON: spec.Forbidden("session and device mismatch"),
 			}
 		}
 	}
@@ -206,7 +215,10 @@ func DeleteDeviceById(
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	// make sure that the access token being used matches the login creds used for user interactive auth, else
@@ -214,7 +226,7 @@ func DeleteDeviceById(
 	if login.Username() != localpart && login.Username() != device.UserID {
 		return util.JSONResponse{
 			Code: 403,
-			JSON: jsonerror.Forbidden("Cannot delete another user's device"),
+			JSON: spec.Forbidden("Cannot delete another user's device"),
 		}
 	}
 
@@ -224,7 +236,10 @@ func DeleteDeviceById(
 		DeviceIDs: []string{deviceID},
 	}, &res); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformDeviceDeletion failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	deleteOK = true
@@ -245,7 +260,7 @@ func DeleteDevices(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
+			JSON: spec.BadJSON("The request body could not be read: " + err.Error()),
 		}
 	}
 	defer req.Body.Close() // nolint:errcheck
@@ -259,14 +274,17 @@ func DeleteDevices(
 	if login.Username() != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("unable to delete devices for other user"),
+			JSON: spec.Forbidden("unable to delete devices for other user"),
 		}
 	}
 
 	payload := devicesDeleteJSON{}
 	if err = json.Unmarshal(bodyBytes, &payload); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("unable to unmarshal device deletion request")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	var res api.PerformDeviceDeletionResponse
@@ -275,7 +293,10 @@ func DeleteDevices(
 		DeviceIDs: payload.Devices,
 	}, &res); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformDeviceDeletion failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/directory.go

@@ -24,7 +24,6 @@ import (
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -56,7 +55,7 @@ func DirectoryRoom(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
+			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 
@@ -70,7 +69,10 @@ func DirectoryRoom(
 	queryRes := &roomserverAPI.GetRoomIDForAliasResponse{}
 	if err = rsAPI.GetRoomIDForAlias(req.Context(), queryReq, queryRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.GetRoomIDForAlias failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	res.RoomID = queryRes.RoomID
@@ -84,7 +86,10 @@ func DirectoryRoom(
 				// TODO: Return 502 if the remote server errored.
 				// TODO: Return 504 if the remote server timed out.
 				util.GetLogger(req.Context()).WithError(fedErr).Error("federation.LookupRoomAlias failed")
-				return jsonerror.InternalServerError()
+				return util.JSONResponse{
+					Code: http.StatusInternalServerError,
+					JSON: spec.InternalServerError{},
+				}
 			}
 			res.RoomID = fedRes.RoomID
 			res.fillServers(fedRes.Servers)
@@ -93,7 +98,7 @@ func DirectoryRoom(
 		if res.RoomID == "" {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
-				JSON: jsonerror.NotFound(
+				JSON: spec.NotFound(
 					fmt.Sprintf("Room alias %s not found", roomAlias),
 				),
 			}
@@ -103,7 +108,10 @@ func DirectoryRoom(
 		var joinedHostsRes federationAPI.QueryJoinedHostServerNamesInRoomResponse
 		if err = fedSenderAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &joinedHostsReq, &joinedHostsRes); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("fedSenderAPI.QueryJoinedHostServerNamesInRoom failed")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		res.fillServers(joinedHostsRes.ServerNames)
 	}
@@ -126,14 +134,14 @@ func SetLocalAlias(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
+			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 
 	if !cfg.Matrix.IsLocalServerName(domain) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Alias must be on local homeserver"),
+			JSON: spec.Forbidden("Alias must be on local homeserver"),
 		}
 	}
 
@@ -146,7 +154,7 @@ func SetLocalAlias(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("User ID must be in the form '@localpart:domain'"),
+			JSON: spec.BadJSON("User ID must be in the form '@localpart:domain'"),
 		}
 	}
 	for _, appservice := range cfg.Derived.ApplicationServices {
@@ -158,7 +166,7 @@ func SetLocalAlias(
 					if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) {
 						return util.JSONResponse{
 							Code: http.StatusBadRequest,
-							JSON: jsonerror.ASExclusive("Alias is reserved by an application service"),
+							JSON: spec.ASExclusive("Alias is reserved by an application service"),
 						}
 					}
 				}
@@ -173,21 +181,50 @@ func SetLocalAlias(
 		return *resErr
 	}
 
-	queryReq := roomserverAPI.SetRoomAliasRequest{
-		UserID: device.UserID,
-		RoomID: r.RoomID,
-		Alias:  alias,
-	}
-	var queryRes roomserverAPI.SetRoomAliasResponse
-	if err := rsAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
-		return jsonerror.InternalServerError()
+	roomID, err := spec.NewRoomID(r.RoomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("invalid room ID"),
+		}
 	}
 
-	if queryRes.AliasExists {
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *roomID, *userID)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("QuerySenderIDForUser failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	} else if senderID == nil {
+		util.GetLogger(req.Context()).WithField("roomID", *roomID).WithField("userID", *userID).Error("Sender ID not found")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	aliasAlreadyExists, err := rsAPI.SetRoomAlias(req.Context(), *senderID, *roomID, alias)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	if aliasAlreadyExists {
 		return util.JSONResponse{
 			Code: http.StatusConflict,
-			JSON: jsonerror.Unknown("The alias " + alias + " already exists."),
+			JSON: spec.Unknown("The alias " + alias + " already exists."),
 		}
 	}
 
@@ -204,27 +241,91 @@ func RemoveLocalAlias(
 	alias string,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 ) util.JSONResponse {
-	queryReq := roomserverAPI.RemoveRoomAliasRequest{
-		Alias:  alias,
-		UserID: device.UserID,
-	}
-	var queryRes roomserverAPI.RemoveRoomAliasResponse
-	if err := rsAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if !queryRes.Found {
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
 		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("The alias does not exist."),
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{Err: "UserID for device is invalid"},
 		}
 	}
 
-	if !queryRes.Removed {
+	roomIDReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: alias}
+	roomIDRes := roomserverAPI.GetRoomIDForAliasResponse{}
+	err = rsAPI.GetRoomIDForAlias(req.Context(), &roomIDReq, &roomIDRes)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The alias does not exist."),
+		}
+	}
+
+	validRoomID, err := spec.NewRoomID(roomIDRes.RoomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The alias does not exist."),
+		}
+	}
+
+	// This seems like the kind of auth check that should be done in the roomserver, but
+	// if this check fails (user is not in the room), then there will be no SenderID for the user
+	// for pseudo-ID rooms - it will just return "". However, we can't use lack of a sender ID
+	// as meaning they are not in the room, since lacking a sender ID could be caused by other bugs.
+	// TODO: maybe have QuerySenderIDForUser return richer errors?
+	var queryResp roomserverAPI.QueryMembershipForUserResponse
+	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+		RoomID: validRoomID.String(),
+		UserID: *userID,
+	}, &queryResp)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.QueryMembershipForUser failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+	if !queryResp.IsInRoom {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("You do not have permission to remove this alias."),
+			JSON: spec.Forbidden("You do not have permission to remove this alias."),
+		}
+	}
+
+	deviceSenderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *userID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The alias does not exist."),
+		}
+	}
+	// TODO: how to handle this case? missing user/room keys seem to be a whole new class of errors
+	if deviceSenderID == nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	aliasFound, aliasRemoved, err := rsAPI.RemoveRoomAlias(req.Context(), *deviceSenderID, alias)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	if !aliasFound {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The alias does not exist."),
+		}
+	}
+
+	if !aliasRemoved {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You do not have permission to remove this alias."),
 		}
 	}
 
@@ -249,7 +350,10 @@ func GetVisibility(
 	}, &res)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryPublishedRooms failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	var v roomVisibility
@@ -271,7 +375,30 @@ func SetVisibility(
 	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, dev *userapi.Device,
 	roomID string,
 ) util.JSONResponse {
-	resErr := checkMemberInRoom(req.Context(), rsAPI, dev.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(dev.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("userID for this device is invalid"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("roomID is invalid")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
+	if err != nil || senderID == nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.Unknown("failed to find senderID for this user"),
+		}
+	}
+
+	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}
@@ -284,18 +411,21 @@ func SetVisibility(
 		}},
 	}
 	var queryEventsRes roomserverAPI.QueryLatestEventsAndStateResponse
-	err := rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
+	err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
 	if err != nil || len(queryEventsRes.StateEvents) == 0 {
 		util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	// NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event
 	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].PDU)
-	if power.UserLevel(dev.UserID) < power.EventLevel(spec.MRoomCanonicalAlias, true) {
+	if power.UserLevel(*senderID) < power.EventLevel(spec.MRoomCanonicalAlias, true) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),
+			JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
 		}
 	}
 
@@ -309,7 +439,10 @@ func SetVisibility(
 		Visibility: v.Visibility,
 	}); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("failed to publish room")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -325,7 +458,7 @@ func SetVisibilityAS(
 	if dev.AccountType != userapi.AccountTypeAppService {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Only appservice may use this endpoint"),
+			JSON: spec.Forbidden("Only appservice may use this endpoint"),
 		}
 	}
 	var v roomVisibility
@@ -345,7 +478,10 @@ func SetVisibilityAS(
 		AppserviceID: dev.AppserviceID,
 	}); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("failed to publish room")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/directory_public.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 )
@@ -68,7 +67,7 @@ func GetPostPublicRooms(
 	if request.IncludeAllNetworks && request.NetworkID != "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidParam("include_all_networks and third_party_instance_id can not be used together"),
+			JSON: spec.InvalidParam("include_all_networks and third_party_instance_id can not be used together"),
 		}
 	}
 
@@ -82,7 +81,10 @@ func GetPostPublicRooms(
 		)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("failed to get public rooms")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		return util.JSONResponse{
 			Code: http.StatusOK,
@@ -93,7 +95,10 @@ func GetPostPublicRooms(
 	response, err := publicRooms(req.Context(), request, rsAPI, extRoomsProvider)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Errorf("failed to work out public rooms")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -173,7 +178,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 	if httpReq.Method != "GET" && httpReq.Method != "POST" {
 		return &util.JSONResponse{
 			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
+			JSON: spec.NotFound("Bad method"),
 		}
 	}
 	if httpReq.Method == "GET" {
@@ -184,7 +189,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 			util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
 			return &util.JSONResponse{
 				Code: 400,
-				JSON: jsonerror.BadJSON("limit param is not a number"),
+				JSON: spec.BadJSON("limit param is not a number"),
 			}
 		}
 		request.Limit = int64(limit)

clientapi/routing/joined_rooms.go

@@ -19,9 +19,9 @@ import (
 
 	"github.com/matrix-org/util"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 type getJoinedRoomsResponse struct {
@@ -33,20 +33,36 @@ func GetJoinedRooms(
 	device *userapi.Device,
 	rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
-	var res api.QueryRoomsForUserResponse
-	err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{
-		UserID:         device.UserID,
-		WantMembership: "join",
-	}, &res)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("Invalid device user ID")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	rooms, err := rsAPI.QueryRoomsForUser(req.Context(), *deviceUserID, "join")
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
 	}
-	if res.RoomIDs == nil {
-		res.RoomIDs = []string{}
+
+	var roomIDStrs []string
+	if rooms == nil {
+		roomIDStrs = []string{}
+	} else {
+		roomIDStrs = make([]string, len(rooms))
+		for i, roomID := range rooms {
+			roomIDStrs[i] = roomID.String()
+		}
 	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: getJoinedRoomsResponse{res.RoomIDs},
+		JSON: getJoinedRoomsResponse{roomIDStrs},
 	}
 }

clientapi/routing/joinroom.go

@@ -16,13 +16,11 @@ package routing
 
 import (
 	"encoding/json"
-	"errors"
 	"net/http"
 	"time"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
@@ -75,7 +73,7 @@ func JoinRoomByIDOrAlias(
 		util.GetLogger(req.Context()).Error("Unable to query user profile, no profile found.")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("Unable to query user profile, no profile found."),
+			JSON: spec.Unknown("Unable to query user profile, no profile found."),
 		}
 	default:
 	}
@@ -99,12 +97,12 @@ func JoinRoomByIDOrAlias(
 		case roomserverAPI.ErrInvalidID:
 			response = util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.Unknown(e.Error()),
+				JSON: spec.Unknown(e.Error()),
 			}
 		case roomserverAPI.ErrNotAllowed:
-			jsonErr := jsonerror.Forbidden(e.Error())
+			jsonErr := spec.Forbidden(e.Error())
 			if device.AccountType == api.AccountTypeGuest {
-				jsonErr = jsonerror.GuestAccessForbidden(e.Error())
+				jsonErr = spec.GuestAccessForbidden(e.Error())
 			}
 			response = util.JSONResponse{
 				Code: http.StatusForbidden,
@@ -115,16 +113,15 @@ func JoinRoomByIDOrAlias(
 				Code: e.Code,
 				JSON: json.RawMessage(e.Message),
 			}
+		case eventutil.ErrRoomNoExists:
+			response = util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: spec.NotFound(e.Error()),
+			}
 		default:
 			response = util.JSONResponse{
 				Code: http.StatusInternalServerError,
-				JSON: jsonerror.InternalServerError(),
-			}
-			if errors.Is(err, eventutil.ErrRoomNoExists) {
-				response = util.JSONResponse{
-					Code: http.StatusNotFound,
-					JSON: jsonerror.NotFound(e.Error()),
-				}
+				JSON: spec.InternalServerError{},
 			}
 		}
 		done <- response
@@ -137,7 +134,7 @@ func JoinRoomByIDOrAlias(
 	case <-timer.C:
 		return util.JSONResponse{
 			Code: http.StatusAccepted,
-			JSON: jsonerror.Unknown("The room join will continue in the background."),
+			JSON: spec.Unknown("The room join will continue in the background."),
 		}
 	case result := <-done:
 		// Stop and drain the timer

clientapi/routing/joinroom_test.go

@@ -7,10 +7,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/roomserver"
@@ -20,6 +22,10 @@ import (
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestJoinRoomByIDOrAlias(t *testing.T) {
 	alice := test.NewUser(t)
 	bob := test.NewUser(t)
@@ -34,9 +40,9 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		natsInstance := jetstream.NATSInstance{}
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
-		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
 		// Create the users in the userapi
 		for _, u := range []*test.User{alice, bob, charlie} {
@@ -63,7 +69,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 			IsDirect:      true,
 			Topic:         "testing",
 			Visibility:    "public",
-			Preset:        presetPublicChat,
+			Preset:        spec.PresetPublicChat,
 			RoomAliasName: "alias",
 			Invite:        []string{bob.ID},
 		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
@@ -78,7 +84,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 			IsDirect:   true,
 			Topic:      "testing",
 			Visibility: "public",
-			Preset:     presetPublicChat,
+			Preset:     spec.PresetPublicChat,
 			Invite:     []string{charlie.ID},
 		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
 		crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse)

clientapi/routing/key_backup.go

@@ -20,8 +20,8 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -64,7 +64,7 @@ func CreateKeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, de
 	if len(kb.AuthData) == 0 {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("missing auth_data"),
+			JSON: spec.BadJSON("missing auth_data"),
 		}
 	}
 	version, err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
@@ -98,7 +98,7 @@ func KeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, device *
 	if !queryResp.Exists {
 		return util.JSONResponse{
 			Code: 404,
-			JSON: jsonerror.NotFound("version not found"),
+			JSON: spec.NotFound("version not found"),
 		}
 	}
 	return util.JSONResponse{
@@ -128,7 +128,7 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.ClientUse
 		Algorithm: kb.Algorithm,
 	})
 	switch e := err.(type) {
-	case *jsonerror.ErrRoomKeysVersion:
+	case spec.ErrRoomKeysVersion:
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: e,
@@ -141,7 +141,7 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.ClientUse
 	if !performKeyBackupResp.Exists {
 		return util.JSONResponse{
 			Code: 404,
-			JSON: jsonerror.NotFound("backup version not found"),
+			JSON: spec.NotFound("backup version not found"),
 		}
 	}
 	return util.JSONResponse{
@@ -162,7 +162,7 @@ func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, de
 	if !exists {
 		return util.JSONResponse{
 			Code: 404,
-			JSON: jsonerror.NotFound("backup version not found"),
+			JSON: spec.NotFound("backup version not found"),
 		}
 	}
 	return util.JSONResponse{
@@ -182,7 +182,7 @@ func UploadBackupKeys(
 	})
 
 	switch e := err.(type) {
-	case *jsonerror.ErrRoomKeysVersion:
+	case spec.ErrRoomKeysVersion:
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: e,
@@ -194,7 +194,7 @@ func UploadBackupKeys(
 	if !performKeyBackupResp.Exists {
 		return util.JSONResponse{
 			Code: 404,
-			JSON: jsonerror.NotFound("backup version not found"),
+			JSON: spec.NotFound("backup version not found"),
 		}
 	}
 	return util.JSONResponse{
@@ -223,7 +223,7 @@ func GetBackupKeys(
 	if !queryResp.Exists {
 		return util.JSONResponse{
 			Code: 404,
-			JSON: jsonerror.NotFound("version not found"),
+			JSON: spec.NotFound("version not found"),
 		}
 	}
 	if sessionID != "" {
@@ -274,6 +274,6 @@ func GetBackupKeys(
 	}
 	return util.JSONResponse{
 		Code: 404,
-		JSON: jsonerror.NotFound("keys not found"),
+		JSON: spec.NotFound("keys not found"),
 	}
 }

clientapi/routing/key_crosssigning.go

@@ -20,9 +20,9 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -32,7 +32,7 @@ type crossSigningRequest struct {
 }
 
 func UploadCrossSigningDeviceKeys(
-	req *http.Request, userInteractiveAuth *auth.UserInteractive,
+	req *http.Request,
 	keyserverAPI api.ClientKeyAPI, device *api.Device,
 	accountAPI api.ClientUserAPI, cfg *config.ClientAPI,
 ) util.JSONResponse {
@@ -62,8 +62,8 @@ func UploadCrossSigningDeviceKeys(
 		}
 	}
 	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: accountAPI.QueryAccountByPassword,
-		Config:               cfg,
+		UserAPI: accountAPI,
+		Config:  cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {
 		return *authErr
@@ -71,31 +71,29 @@ func UploadCrossSigningDeviceKeys(
 	sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
 
 	uploadReq.UserID = device.UserID
-	if err := keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes); err != nil {
-		return jsonerror.InternalAPIError(req.Context(), err)
-	}
+	keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes)
 
 	if err := uploadRes.Error; err != nil {
 		switch {
 		case err.IsInvalidSignature:
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidSignature(err.Error()),
+				JSON: spec.InvalidSignature(err.Error()),
 			}
 		case err.IsMissingParam:
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.MissingParam(err.Error()),
+				JSON: spec.MissingParam(err.Error()),
 			}
 		case err.IsInvalidParam:
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidParam(err.Error()),
+				JSON: spec.InvalidParam(err.Error()),
 			}
 		default:
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.Unknown(err.Error()),
+				JSON: spec.Unknown(err.Error()),
 			}
 		}
 	}
@@ -115,31 +113,29 @@ func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.Clie
 	}
 
 	uploadReq.UserID = device.UserID
-	if err := keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes); err != nil {
-		return jsonerror.InternalAPIError(req.Context(), err)
-	}
+	keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes)
 
 	if err := uploadRes.Error; err != nil {
 		switch {
 		case err.IsInvalidSignature:
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidSignature(err.Error()),
+				JSON: spec.InvalidSignature(err.Error()),
 			}
 		case err.IsMissingParam:
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.MissingParam(err.Error()),
+				JSON: spec.MissingParam(err.Error()),
 			}
 		case err.IsInvalidParam:
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidParam(err.Error()),
+				JSON: spec.InvalidParam(err.Error()),
 			}
 		default:
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.Unknown(err.Error()),
+				JSON: spec.Unknown(err.Error()),
 			}
 		}
 	}

clientapi/routing/keys.go

@@ -22,8 +22,8 @@ import (
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 type uploadKeysRequest struct {
@@ -67,7 +67,10 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device)
 	}
 	if uploadRes.Error != nil {
 		util.GetLogger(req.Context()).WithError(uploadRes.Error).Error("Failed to PerformUploadKeys")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if len(uploadRes.KeyErrors) > 0 {
 		util.GetLogger(req.Context()).WithField("key_errors", uploadRes.KeyErrors).Error("Failed to upload one or more keys")
@@ -90,7 +93,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device)
 
 type queryKeysRequest struct {
 	Timeout    int                 `json:"timeout"`
-	Token      string              `json:"token"`
 	DeviceKeys map[string][]string `json:"device_keys"`
 }
 
@@ -112,14 +114,11 @@ func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) u
 		return *resErr
 	}
 	queryRes := api.QueryKeysResponse{}
-	if err := keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{
+	keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{
 		UserID:        device.UserID,
 		UserToDevices: r.DeviceKeys,
 		Timeout:       r.GetTimeout(),
-		// TODO: Token?
-	}, &queryRes); err != nil {
-		return util.ErrorResponse(err)
-	}
+	}, &queryRes)
 	return util.JSONResponse{
 		Code: 200,
 		JSON: map[string]interface{}{
@@ -152,15 +151,16 @@ func ClaimKeys(req *http.Request, keyAPI api.ClientKeyAPI) util.JSONResponse {
 		return *resErr
 	}
 	claimRes := api.PerformClaimKeysResponse{}
-	if err := keyAPI.PerformClaimKeys(req.Context(), &api.PerformClaimKeysRequest{
+	keyAPI.PerformClaimKeys(req.Context(), &api.PerformClaimKeysRequest{
 		OneTimeKeys: r.OneTimeKeys,
 		Timeout:     r.GetTimeout(),
-	}, &claimRes); err != nil {
-		return jsonerror.InternalAPIError(req.Context(), err)
-	}
+	}, &claimRes)
 	if claimRes.Error != nil {
 		util.GetLogger(req.Context()).WithError(claimRes.Error).Error("failed to PerformClaimKeys")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	return util.JSONResponse{
 		Code: 200,

clientapi/routing/leaveroom.go

@@ -17,9 +17,9 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -29,10 +29,18 @@ func LeaveRoomByID(
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	roomID string,
 ) util.JSONResponse {
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.Unknown("device userID is invalid"),
+		}
+	}
+
 	// Prepare to ask the roomserver to perform the room join.
 	leaveReq := roomserverAPI.PerformLeaveRequest{
 		RoomID: roomID,
-		UserID: device.UserID,
+		Leaver: *userID,
 	}
 	leaveRes := roomserverAPI.PerformLeaveResponse{}
 
@@ -41,12 +49,12 @@ func LeaveRoomByID(
 		if leaveRes.Code != 0 {
 			return util.JSONResponse{
 				Code: leaveRes.Code,
-				JSON: jsonerror.LeaveServerNoticeError(),
+				JSON: spec.LeaveServerNoticeError(),
 			}
 		}
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown(err.Error()),
+			JSON: spec.Unknown(err.Error()),
 		}
 	}
 

clientapi/routing/login.go

@@ -19,10 +19,11 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -40,28 +41,25 @@ type flow struct {
 	Type string `json:"type"`
 }
 
-func passwordLogin() flows {
-	f := flows{}
-	s := flow{
-		Type: "m.login.password",
-	}
-	f.Flows = append(f.Flows, s)
-	return f
-}
-
 // Login implements GET and POST /login
 func Login(
 	req *http.Request, userAPI userapi.ClientUserAPI,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	if req.Method == http.MethodGet {
-		// TODO: support other forms of login other than password, depending on config options
+		loginFlows := []flow{{Type: authtypes.LoginTypePassword}}
+		if len(cfg.Derived.ApplicationServices) > 0 {
+			loginFlows = append(loginFlows, flow{Type: authtypes.LoginTypeApplicationService})
+		}
+		// TODO: support other forms of login, depending on config options
 		return util.JSONResponse{
 			Code: http.StatusOK,
-			JSON: passwordLogin(),
+			JSON: flows{
+				Flows: loginFlows,
+			},
 		}
 	} else if req.Method == http.MethodPost {
-		login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg)
+		login, cleanup, authErr := auth.LoginFromJSONReader(req, userAPI, userAPI, cfg)
 		if authErr != nil {
 			return *authErr
 		}
@@ -72,7 +70,7 @@ func Login(
 	}
 	return util.JSONResponse{
 		Code: http.StatusMethodNotAllowed,
-		JSON: jsonerror.NotFound("Bad method"),
+		JSON: spec.NotFound("Bad method"),
 	}
 }
 
@@ -83,13 +81,19 @@ func completeAuth(
 	token, err := auth.GenerateAccessToken()
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("auth.GenerateAccessToken failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	localpart, serverName, err := userutil.ParseUsernameParam(login.Username(), cfg)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("auth.ParseUsernameParam failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	var performRes userapi.PerformDeviceCreationResponse
@@ -105,7 +109,7 @@ func completeAuth(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("failed to create device: " + err.Error()),
+			JSON: spec.Unknown("failed to create device: " + err.Error()),
 		}
 	}
 

clientapi/routing/login_test.go

@@ -47,8 +47,9 @@ func TestLogin(t *testing.T) {
 		routers := httputil.NewRouters()
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for /login
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
 		Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics)
@@ -113,6 +114,44 @@ func TestLogin(t *testing.T) {
 
 		ctx := context.Background()
 
+		// Inject a dummy application service, so we have a "m.login.application_service"
+		// in the login flows
+		as := &config.ApplicationService{}
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+		t.Run("Supported log-in flows are returned", func(t *testing.T) {
+			req := test.NewRequest(t, http.MethodGet, "/_matrix/client/v3/login")
+			rec := httptest.NewRecorder()
+			routers.Client.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("failed to get log-in flows: %s", rec.Body.String())
+			}
+
+			t.Logf("response: %s", rec.Body.String())
+			resp := flows{}
+			if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+
+			appServiceFound := false
+			passwordFound := false
+			for _, flow := range resp.Flows {
+				if flow.Type == "m.login.password" {
+					passwordFound = true
+				} else if flow.Type == "m.login.application_service" {
+					appServiceFound = true
+				} else {
+					t.Fatalf("got unknown login flow: %s", flow.Type)
+				}
+			}
+			if !appServiceFound {
+				t.Fatal("m.login.application_service missing from login flows")
+			}
+			if !passwordFound {
+				t.Fatal("m.login.password missing from login flows")
+			}
+		})
+
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
 				req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{

clientapi/routing/logout.go

@@ -17,8 +17,8 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -33,7 +33,10 @@ func Logout(
 	}, &performRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("PerformDeviceDeletion failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -53,7 +56,10 @@ func LogoutAll(
 	}, &performRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("PerformDeviceDeletion failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/membership.go

@@ -16,18 +16,15 @@ package routing
 
 import (
 	"context"
+	"crypto/ed25519"
 	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/getsentry/sentry-go"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
-
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -35,6 +32,9 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/util"
 )
@@ -52,11 +52,33 @@ func SendBan(
 	if body.UserID == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("missing user_id"),
+			JSON: spec.BadJSON("missing user_id"),
 		}
 	}
 
-	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to ban this user, bad userID"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
+	if err != nil || senderID == nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to ban this user, unknown senderID"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
@@ -65,11 +87,11 @@ func SendBan(
 	if errRes != nil {
 		return *errRes
 	}
-	allowedToBan := pl.UserLevel(device.UserID) >= pl.Ban
+	allowedToBan := pl.UserLevel(*senderID) >= pl.Ban
 	if !allowedToBan {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("You don't have permission to ban this user, power level too low."),
+			JSON: spec.Forbidden("You don't have permission to ban this user, power level too low."),
 		}
 	}
 
@@ -86,7 +108,10 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic
 	)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	serverName := device.UserDomain()
@@ -101,7 +126,10 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic
 		false,
 	); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -122,31 +150,61 @@ func SendKick(
 	if body.UserID == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("missing user_id"),
+			JSON: spec.BadJSON("missing user_id"),
 		}
 	}
 
-	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
+	if err != nil || senderID == nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
 
+	bodyUserID, err := spec.NewUserID(body.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("body userID is invalid"),
+		}
+	}
+
 	pl, errRes := getPowerlevels(req, rsAPI, roomID)
 	if errRes != nil {
 		return *errRes
 	}
-	allowedToKick := pl.UserLevel(device.UserID) >= pl.Kick
+	allowedToKick := pl.UserLevel(*senderID) >= pl.Kick || bodyUserID.String() == deviceUserID.String()
 	if !allowedToKick {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("You don't have permission to kick this user, power level too low."),
+			JSON: spec.Forbidden("You don't have permission to kick this user, power level too low."),
 		}
 	}
 
 	var queryRes roomserverAPI.QueryMembershipForUserResponse
-	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
-		UserID: body.UserID,
+		UserID: *bodyUserID,
 	}, &queryRes)
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -155,7 +213,7 @@ func SendKick(
 	if queryRes.Membership != spec.Join && queryRes.Membership != spec.Invite {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Unknown("cannot /kick banned or left users"),
+			JSON: spec.Unknown("cannot /kick banned or left users"),
 		}
 	}
 	// TODO: should we be using SendLeave instead?
@@ -174,19 +232,34 @@ func SendUnban(
 	if body.UserID == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("missing user_id"),
+			JSON: spec.BadJSON("missing user_id"),
 		}
 	}
 
-	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
 
+	bodyUserID, err := spec.NewUserID(body.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("body userID is invalid"),
+		}
+	}
 	var queryRes roomserverAPI.QueryMembershipForUserResponse
-	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
-		UserID: body.UserID,
+		UserID: *bodyUserID,
 	}, &queryRes)
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -196,7 +269,7 @@ func SendUnban(
 	if queryRes.Membership != spec.Ban {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown("can only /unban users that are banned"),
+			JSON: spec.Unknown("can only /unban users that are banned"),
 		}
 	}
 	// TODO: should we be using SendLeave instead?
@@ -233,43 +306,78 @@ func SendInvite(
 	if body.UserID == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("missing user_id"),
+			JSON: spec.BadJSON("missing user_id"),
 		}
 	}
 
-	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
 
 	// We already received the return value, so no need to check for an error here.
-	response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime)
+	response, _ := sendInvite(req.Context(), device, roomID, body.UserID, body.Reason, cfg, rsAPI, evTime)
 	return response
 }
 
 // sendInvite sends an invitation to a user. Returns a JSONResponse and an error
 func sendInvite(
 	ctx context.Context,
-	profileAPI userapi.ClientUserAPI,
 	device *userapi.Device,
 	roomID, userID, reason string,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
+	evTime time.Time,
 ) (util.JSONResponse, error) {
-	event, err := buildMembershipEvent(
-		ctx, userID, reason, profileAPI, device, spec.Invite,
-		roomID, false, cfg, evTime, rsAPI, asAPI,
-	)
+	validRoomID, err := spec.NewRoomID(roomID)
 	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
-		return jsonerror.InternalServerError(), err
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("RoomID is invalid"),
+		}, err
+	}
+	inviter, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, err
+	}
+	invitee, err := spec.NewUserID(userID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("UserID is invalid"),
+		}, err
 	}
 
+	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, err
+	}
 	err = rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{
-		Event:           event,
+		InviteInput: roomserverAPI.InviteInput{
+			RoomID:     *validRoomID,
+			Inviter:    *inviter,
+			Invitee:    *invitee,
+			Reason:     reason,
+			IsDirect:   false,
+			KeyID:      identity.KeyID,
+			PrivateKey: identity.PrivateKey,
+			EventTime:  evTime,
+		},
 		InviteRoomState: nil, // ask the roomserver to draw up invite room state for us
-		RoomVersion:     event.Version(),
 		SendAsServer:    string(device.UserDomain()),
 	})
 
@@ -277,12 +385,12 @@ func sendInvite(
 	case roomserverAPI.ErrInvalidID:
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown(e.Error()),
+			JSON: spec.Unknown(e.Error()),
 		}, e
 	case roomserverAPI.ErrNotAllowed:
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden(e.Error()),
+			JSON: spec.Forbidden(e.Error()),
 		}, e
 	case nil:
 	default:
@@ -290,7 +398,7 @@ func sendInvite(
 		sentry.CaptureException(err)
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.InternalServerError(),
+			JSON: spec.InternalServerError{},
 		}, err
 	}
 
@@ -300,6 +408,42 @@ func sendInvite(
 	}, nil
 }
 
+func buildMembershipEventDirect(
+	ctx context.Context,
+	targetSenderID spec.SenderID, reason string, userDisplayName, userAvatarURL string,
+	sender spec.SenderID, senderDomain spec.ServerName,
+	membership, roomID string, isDirect bool,
+	keyID gomatrixserverlib.KeyID, privateKey ed25519.PrivateKey, evTime time.Time,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
+) (*types.HeaderedEvent, error) {
+	targetSenderString := string(targetSenderID)
+	proto := gomatrixserverlib.ProtoEvent{
+		SenderID: string(sender),
+		RoomID:   roomID,
+		Type:     "m.room.member",
+		StateKey: &targetSenderString,
+	}
+
+	content := gomatrixserverlib.MemberContent{
+		Membership:  membership,
+		DisplayName: userDisplayName,
+		AvatarURL:   userAvatarURL,
+		Reason:      reason,
+		IsDirect:    isDirect,
+	}
+
+	if err := proto.SetContent(content); err != nil {
+		return nil, err
+	}
+
+	identity := &fclient.SigningIdentity{
+		ServerName: senderDomain,
+		KeyID:      keyID,
+		PrivateKey: privateKey,
+	}
+	return eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, rsAPI, nil)
+}
+
 func buildMembershipEvent(
 	ctx context.Context,
 	targetUserID, reason string, profileAPI userapi.ClientUserAPI,
@@ -313,31 +457,39 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
-	proto := gomatrixserverlib.ProtoEvent{
-		Sender:   device.UserID,
-		RoomID:   roomID,
-		Type:     "m.room.member",
-		StateKey: &targetUserID,
-	}
-
-	content := gomatrixserverlib.MemberContent{
-		Membership:  membership,
-		DisplayName: profile.DisplayName,
-		AvatarURL:   profile.AvatarURL,
-		Reason:      reason,
-		IsDirect:    isDirect,
-	}
-
-	if err = proto.SetContent(content); err != nil {
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
 		return nil, err
 	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return nil, err
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID)
+	if err != nil {
+		return nil, err
+	} else if senderID == nil {
+		return nil, fmt.Errorf("no sender ID for %s in %s", *userID, *validRoomID)
+	}
 
-	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+	targetID, err := spec.NewUserID(targetUserID, true)
+	if err != nil {
+		return nil, err
+	}
+	targetSenderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *targetID)
+	if err != nil {
+		return nil, err
+	} else if targetSenderID == nil {
+		return nil, fmt.Errorf("no sender ID for %s in %s", *targetID, *validRoomID)
+	}
+
+	identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *userID)
 	if err != nil {
 		return nil, err
 	}
 
-	return eventutil.QueryAndBuildEvent(ctx, &proto, cfg.Matrix, identity, evTime, rsAPI, nil)
+	return buildMembershipEventDirect(ctx, *targetSenderID, reason, profile.DisplayName, profile.AvatarURL,
+		*senderID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI)
 }
 
 // loadProfile lookups the profile of a given user from the database and returns
@@ -377,7 +529,7 @@ func extractRequestData(req *http.Request) (body *threepid.MembershipRequest, ev
 	if err != nil {
 		resErr = &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+			JSON: spec.InvalidParam(err.Error()),
 		}
 		return
 	}
@@ -399,36 +551,43 @@ func checkAndProcessThreepid(
 		req.Context(), device, body, cfg, rsAPI, profileAPI,
 		roomID, evTime,
 	)
-	if err == threepid.ErrMissingParameter {
+	switch e := err.(type) {
+	case nil:
+	case threepid.ErrMissingParameter:
+		util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAndProcessInvite failed")
 		return inviteStored, &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
+			JSON: spec.BadJSON(err.Error()),
 		}
-	} else if err == threepid.ErrNotTrusted {
+	case threepid.ErrNotTrusted:
+		util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAndProcessInvite failed")
 		return inviteStored, &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.NotTrusted(body.IDServer),
+			JSON: spec.NotTrusted(body.IDServer),
 		}
-	} else if err == eventutil.ErrRoomNoExists {
+	case eventutil.ErrRoomNoExists:
+		util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAndProcessInvite failed")
 		return inviteStored, &util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(err.Error()),
+			JSON: spec.NotFound(err.Error()),
 		}
-	} else if e, ok := err.(gomatrixserverlib.BadJSONError); ok {
+	case gomatrixserverlib.BadJSONError:
+		util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAndProcessInvite failed")
 		return inviteStored, &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(e.Error()),
+			JSON: spec.BadJSON(e.Error()),
 		}
-	}
-	if err != nil {
+	default:
 		util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAndProcessInvite failed")
-		er := jsonerror.InternalServerError()
-		return inviteStored, &er
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	return
 }
 
-func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID, roomID string) *util.JSONResponse {
+func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID spec.UserID, roomID string) *util.JSONResponse {
 	var membershipRes roomserverAPI.QueryMembershipForUserResponse
 	err := rsAPI.QueryMembershipForUser(ctx, &roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
@@ -436,13 +595,15 @@ func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserver
 	}, &membershipRes)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("QueryMembershipForUser: could not query membership for user")
-		e := jsonerror.InternalServerError()
-		return &e
+		return &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if !membershipRes.IsInRoom {
 		return &util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("user does not belong to room"),
+			JSON: spec.Forbidden("user does not belong to room"),
 		}
 	}
 	return nil
@@ -454,26 +615,38 @@ func SendForget(
 ) util.JSONResponse {
 	ctx := req.Context()
 	logger := util.GetLogger(ctx).WithField("roomID", roomID).WithField("userID", device.UserID)
+
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"),
+		}
+	}
+
 	var membershipRes roomserverAPI.QueryMembershipForUserResponse
 	membershipReq := roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
-		UserID: device.UserID,
+		UserID: *deviceUserID,
 	}
-	err := rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes)
+	err = rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes)
 	if err != nil {
 		logger.WithError(err).Error("QueryMembershipForUser: could not query membership for user")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if !membershipRes.RoomExists {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("room does not exist"),
+			JSON: spec.Forbidden("room does not exist"),
 		}
 	}
 	if membershipRes.IsInRoom {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown(fmt.Sprintf("User %s is in room %s", device.UserID, roomID)),
+			JSON: spec.Unknown(fmt.Sprintf("User %s is in room %s", device.UserID, roomID)),
 		}
 	}
 
@@ -484,7 +657,10 @@ func SendForget(
 	response := roomserverAPI.PerformForgetResponse{}
 	if err := rsAPI.PerformForget(ctx, &request, &response); err != nil {
 		logger.WithError(err).Error("PerformForget: unable to forget room")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -500,14 +676,14 @@ func getPowerlevels(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
 	if plEvent == nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("You don't have permission to perform this action, no power_levels event in this room."),
+			JSON: spec.Forbidden("You don't have permission to perform this action, no power_levels event in this room."),
 		}
 	}
 	pl, err := plEvent.PowerLevels()
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("You don't have permission to perform this action, the power_levels event for this room is malformed so auth checks cannot be performed."),
+			JSON: spec.Forbidden("You don't have permission to perform this action, the power_levels event for this room is malformed so auth checks cannot be performed."),
 		}
 	}
 	return pl, nil

clientapi/routing/memberships.go

@@ -0,0 +1,139 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+)
+
+// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
+type getJoinedMembersResponse struct {
+	Joined map[string]joinedMember `json:"joined"`
+}
+
+type joinedMember struct {
+	DisplayName string `json:"display_name"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// The database stores 'displayname' without an underscore.
+// Deserialize into this and then change to the actual API response
+type databaseJoinedMember struct {
+	DisplayName string `json:"displayname"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// GetJoinedMembers implements
+//
+//	GET /rooms/{roomId}/joined_members
+func GetJoinedMembers(
+	req *http.Request, device *userapi.Device, roomID string,
+	rsAPI api.ClientRoomserverAPI,
+) util.JSONResponse {
+	// Validate the userID
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("Device UserID is invalid"),
+		}
+	}
+
+	// Validate the roomID
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("RoomID is invalid"),
+		}
+	}
+
+	// Get the current memberships for the requesting user to determine
+	// if they are allowed to query this endpoint.
+	queryReq := api.QueryMembershipForUserRequest{
+		RoomID: validRoomID.String(),
+		UserID: *userID,
+	}
+
+	var queryRes api.QueryMembershipForUserResponse
+	if queryErr := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); queryErr != nil {
+		util.GetLogger(req.Context()).WithError(queryErr).Error("rsAPI.QueryMembershipsForRoom failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	if !queryRes.HasBeenInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	if !queryRes.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	// Get the current membership events
+	var membershipsForRoomResp api.QueryMembershipsForRoomResponse
+	if err = rsAPI.QueryMembershipsForRoom(req.Context(), &api.QueryMembershipsForRoomRequest{
+		JoinedOnly: true,
+		RoomID:     validRoomID.String(),
+	}, &membershipsForRoomResp); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryEventsByID failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	var res getJoinedMembersResponse
+	res.Joined = make(map[string]joinedMember)
+	for _, ev := range membershipsForRoomResp.JoinEvents {
+		var content databaseJoinedMember
+		if err := json.Unmarshal(ev.Content, &content); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(ev.Sender))
+		if err != nil || userID == nil {
+			util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		res.Joined[userID.String()] = joinedMember(content)
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}

clientapi/routing/notification.go

@@ -18,9 +18,9 @@ import (
 	"net/http"
 	"strconv"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -35,7 +35,10 @@ func GetNotifications(
 		limit, err = strconv.ParseInt(limitStr, 10, 64)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("ParseInt(limit) failed")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 	}
 
@@ -43,7 +46,10 @@ func GetNotifications(
 	localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	err = userAPI.QueryNotifications(req.Context(), &userapi.QueryNotificationsRequest{
 		Localpart:  localpart,
@@ -54,7 +60,10 @@ func GetNotifications(
 	}, &queryRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryNotifications failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	util.GetLogger(req.Context()).WithField("from", req.URL.Query().Get("from")).WithField("limit", limit).WithField("only", req.URL.Query().Get("only")).WithField("next", queryRes.NextToken).Infof("QueryNotifications: len %d", len(queryRes.Notifications))
 	return util.JSONResponse{

clientapi/routing/openid.go

@@ -17,9 +17,9 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -43,7 +43,7 @@ func CreateOpenIDToken(
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Cannot request tokens for other users"),
+			JSON: spec.Forbidden("Cannot request tokens for other users"),
 		}
 	}
 
@@ -55,7 +55,10 @@ func CreateOpenIDToken(
 	err := userAPI.PerformOpenIDTokenCreation(req.Context(), &request, &response)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("userAPI.CreateOpenIDToken failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/password.go

@@ -6,11 +6,11 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )
@@ -73,8 +73,8 @@ func Password(
 
 	// Check if the existing password is correct.
 	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: userAPI.QueryAccountByPassword,
-		Config:               cfg,
+		UserAPI: userAPI,
+		Config:  cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {
 		return *authErr
@@ -90,7 +90,10 @@ func Password(
 	localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	// Ask the user API to perform the password change.
@@ -102,11 +105,17 @@ func Password(
 	passwordRes := &api.PerformPasswordUpdateResponse{}
 	if err := userAPI.PerformPasswordUpdate(req.Context(), passwordReq, passwordRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("PerformPasswordUpdate failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if !passwordRes.PasswordUpdated {
 		util.GetLogger(req.Context()).Error("Expected password to have been updated but wasn't")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	// If the request asks us to log out all other devices then
@@ -120,7 +129,10 @@ func Password(
 		logoutRes := &api.PerformDeviceDeletionResponse{}
 		if err := userAPI.PerformDeviceDeletion(req.Context(), logoutReq, logoutRes); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("PerformDeviceDeletion failed")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 
 		pushersReq := &api.PerformPusherDeletionRequest{
@@ -130,7 +142,10 @@ func Password(
 		}
 		if err := userAPI.PerformPusherDeletion(req.Context(), pushersReq, &struct{}{}); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("PerformPusherDeletion failed")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 	}
 

clientapi/routing/peekroom.go

@@ -18,7 +18,6 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
@@ -61,12 +60,12 @@ func PeekRoomByIDOrAlias(
 	case roomserverAPI.ErrInvalidID:
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown(e.Error()),
+			JSON: spec.Unknown(e.Error()),
 		}
 	case roomserverAPI.ErrNotAllowed:
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden(e.Error()),
+			JSON: spec.Forbidden(e.Error()),
 		}
 	case *gomatrix.HTTPError:
 		return util.JSONResponse{
@@ -76,7 +75,10 @@ func PeekRoomByIDOrAlias(
 	case nil:
 	default:
 		logrus.WithError(err).WithField("roomID", roomIDOrAlias).Errorf("Failed to peek room")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	// if this user is already joined to the room, we let them peek anyway
@@ -107,12 +109,15 @@ func UnpeekRoomByID(
 	case roomserverAPI.ErrInvalidID:
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown(e.Error()),
+			JSON: spec.Unknown(e.Error()),
 		}
 	case nil:
 	default:
 		logrus.WithError(err).WithField("roomID", roomID).Errorf("Failed to un-peek room")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/presence.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -54,7 +53,7 @@ func SetPresence(
 	if device.UserID != userID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Unable to set presence for other user."),
+			JSON: spec.Forbidden("Unable to set presence for other user."),
 		}
 	}
 	var presence presenceReq
@@ -67,7 +66,7 @@ func SetPresence(
 	if !ok {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown(fmt.Sprintf("Unknown presence '%s'.", presence.Presence)),
+			JSON: spec.Unknown(fmt.Sprintf("Unknown presence '%s'.", presence.Presence)),
 		}
 	}
 	err := producer.SendPresence(req.Context(), userID, presenceStatus, presence.StatusMsg)
@@ -75,7 +74,7 @@ func SetPresence(
 		log.WithError(err).Errorf("failed to update presence")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.InternalServerError(),
+			JSON: spec.InternalServerError{},
 		}
 	}
 
@@ -100,7 +99,7 @@ func GetPresence(
 		log.WithError(err).Errorf("unable to get presence")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.InternalServerError(),
+			JSON: spec.InternalServerError{},
 		}
 	}
 
@@ -119,7 +118,7 @@ func GetPresence(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.InternalServerError(),
+			JSON: spec.InternalServerError{},
 		}
 	}
 

clientapi/routing/profile.go

@@ -16,6 +16,7 @@ package routing
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"time"
 
@@ -26,13 +27,11 @@ import (
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/util"
 )
@@ -49,12 +48,15 @@ func GetProfile(
 		if err == appserviceAPI.ErrProfileNotExists {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
-				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
+				JSON: spec.NotFound("The user does not exist or does not have a profile"),
 			}
 		}
 
 		util.GetLogger(req.Context()).WithError(err).Error("getProfile failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -95,7 +97,7 @@ func SetAvatarURL(
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("userID does not match the current user"),
+			JSON: spec.Forbidden("userID does not match the current user"),
 		}
 	}
 
@@ -103,23 +105,20 @@ func SetAvatarURL(
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
-	if r.AvatarURL == "" {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("'avatar_url' must be supplied."),
-		}
-	}
 
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	if !cfg.Matrix.IsLocalServerName(domain) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("userID does not belong to a locally configured domain"),
+			JSON: spec.Forbidden("userID does not belong to a locally configured domain"),
 		}
 	}
 
@@ -127,14 +126,17 @@ func SetAvatarURL(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+			JSON: spec.InvalidParam(err.Error()),
 		}
 	}
 
 	profile, changed, err := profileAPI.SetAvatarURL(req.Context(), localpart, domain, r.AvatarURL)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetAvatarURL failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	// No need to build new membership events, since nothing changed
 	if !changed {
@@ -144,7 +146,7 @@ func SetAvatarURL(
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, evTime)
 	if err != nil {
 		return response
 	}
@@ -184,7 +186,7 @@ func SetDisplayName(
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("userID does not match the current user"),
+			JSON: spec.Forbidden("userID does not match the current user"),
 		}
 	}
 
@@ -192,23 +194,20 @@ func SetDisplayName(
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
-	if r.DisplayName == "" {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("'displayname' must be supplied."),
-		}
-	}
 
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	if !cfg.Matrix.IsLocalServerName(domain) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("userID does not belong to a locally configured domain"),
+			JSON: spec.Forbidden("userID does not belong to a locally configured domain"),
 		}
 	}
 
@@ -216,14 +215,17 @@ func SetDisplayName(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+			JSON: spec.InvalidParam(err.Error()),
 		}
 	}
 
 	profile, changed, err := profileAPI.SetDisplayName(req.Context(), localpart, domain, r.DisplayName)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetDisplayName failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	// No need to build new membership events, since nothing changed
 	if !changed {
@@ -233,7 +235,7 @@ func SetDisplayName(
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, evTime)
 	if err != nil {
 		return response
 	}
@@ -247,42 +249,63 @@ func SetDisplayName(
 func updateProfile(
 	ctx context.Context, rsAPI api.ClientRoomserverAPI, device *userapi.Device,
 	profile *authtypes.Profile,
-	userID string, cfg *config.ClientAPI, evTime time.Time,
+	userID string, evTime time.Time,
 ) (util.JSONResponse, error) {
-	var res api.QueryRoomsForUserResponse
-	err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
-		UserID:         device.UserID,
-		WantMembership: "join",
-	}, &res)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}, err
+	}
+
+	rooms, err := rsAPI.QueryRoomsForUser(ctx, *deviceUserID, "join")
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("QueryRoomsForUser failed")
-		return jsonerror.InternalServerError(), err
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, err
+	}
+
+	roomIDStrs := make([]string, len(rooms))
+	for i, room := range rooms {
+		roomIDStrs[i] = room.String()
 	}
 
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError(), err
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, err
 	}
 
 	events, err := buildMembershipEvents(
-		ctx, device, res.RoomIDs, *profile, userID, cfg, evTime, rsAPI,
+		ctx, roomIDStrs, *profile, userID, evTime, rsAPI,
 	)
 	switch e := err.(type) {
 	case nil:
 	case gomatrixserverlib.BadJSONError:
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(e.Error()),
+			JSON: spec.BadJSON(e.Error()),
 		}, e
 	default:
 		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvents failed")
-		return jsonerror.InternalServerError(), e
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, e
 	}
 
-	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil {
+	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, false); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
-		return jsonerror.InternalServerError(), err
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, err
 	}
 	return util.JSONResponse{}, nil
 }
@@ -331,19 +354,33 @@ func getProfile(
 
 func buildMembershipEvents(
 	ctx context.Context,
-	device *userapi.Device,
 	roomIDs []string,
-	newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
+	newProfile authtypes.Profile, userID string,
 	evTime time.Time, rsAPI api.ClientRoomserverAPI,
 ) ([]*types.HeaderedEvent, error) {
 	evs := []*types.HeaderedEvent{}
 
+	fullUserID, err := spec.NewUserID(userID, true)
+	if err != nil {
+		return nil, err
+	}
 	for _, roomID := range roomIDs {
+		validRoomID, err := spec.NewRoomID(roomID)
+		if err != nil {
+			return nil, err
+		}
+		senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID)
+		if err != nil {
+			return nil, err
+		} else if senderID == nil {
+			return nil, fmt.Errorf("sender ID not found for %s in %s", *fullUserID, *validRoomID)
+		}
+		senderIDString := string(*senderID)
 		proto := gomatrixserverlib.ProtoEvent{
-			Sender:   userID,
+			SenderID: senderIDString,
 			RoomID:   roomID,
 			Type:     "m.room.member",
-			StateKey: &userID,
+			StateKey: &senderIDString,
 		}
 
 		content := gomatrixserverlib.MemberContent{
@@ -353,16 +390,21 @@ func buildMembershipEvents(
 		content.DisplayName = newProfile.DisplayName
 		content.AvatarURL = newProfile.AvatarURL
 
-		if err := proto.SetContent(content); err != nil {
+		if err = proto.SetContent(content); err != nil {
 			return nil, err
 		}
 
-		identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+		user, err := spec.NewUserID(userID, true)
 		if err != nil {
 			return nil, err
 		}
 
-		event, err := eventutil.QueryAndBuildEvent(ctx, &proto, cfg.Matrix, identity, evTime, rsAPI, nil)
+		identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *user)
+		if err != nil {
+			return nil, err
+		}
+
+		event, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, evTime, rsAPI, nil)
 		if err != nil {
 			return nil, err
 		}

clientapi/routing/pusher.go

@@ -19,9 +19,9 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -34,7 +34,10 @@ func GetPushers(
 	localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	err = userAPI.QueryPushers(req.Context(), &userapi.QueryPushersRequest{
 		Localpart:  localpart,
@@ -42,7 +45,10 @@ func GetPushers(
 	}, &queryRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryPushers failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	for i := range queryRes.Pushers {
 		queryRes.Pushers[i].SessionID = 0
@@ -63,7 +69,10 @@ func SetPusher(
 	localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	body := userapi.PerformPusherSetRequest{}
 	if resErr := httputil.UnmarshalJSONRequest(req, &body); resErr != nil {
@@ -99,7 +108,10 @@ func SetPusher(
 	err = userAPI.PerformPusherSet(req.Context(), &body, &struct{}{})
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("PerformPusherSet failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -111,6 +123,6 @@ func SetPusher(
 func invalidParam(msg string) util.JSONResponse {
 	return util.JSONResponse{
 		Code: http.StatusBadRequest,
-		JSON: jsonerror.InvalidParam(msg),
+		JSON: spec.InvalidParam(msg),
 	}
 }

clientapi/routing/pushrules.go

@@ -7,27 +7,30 @@ import (
 	"net/http"
 	"reflect"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/pushrules"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
 func errorResponse(ctx context.Context, err error, msg string, args ...interface{}) util.JSONResponse {
-	if eerr, ok := err.(*jsonerror.MatrixError); ok {
+	if eerr, ok := err.(spec.MatrixError); ok {
 		var status int
 		switch eerr.ErrCode {
-		case "M_INVALID_ARGUMENT_VALUE":
+		case spec.ErrorInvalidParam:
 			status = http.StatusBadRequest
-		case "M_NOT_FOUND":
+		case spec.ErrorNotFound:
 			status = http.StatusNotFound
 		default:
 			status = http.StatusInternalServerError
 		}
-		return util.MatrixErrorResponse(status, eerr.ErrCode, eerr.Err)
+		return util.MatrixErrorResponse(status, string(eerr.ErrCode), eerr.Err)
 	}
 	util.GetLogger(ctx).WithError(err).Errorf(msg, args...)
-	return jsonerror.InternalServerError()
+	return util.JSONResponse{
+		Code: http.StatusInternalServerError,
+		JSON: spec.InternalServerError{},
+	}
 }
 
 func GetAllPushRules(ctx context.Context, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
@@ -48,7 +51,7 @@ func GetPushRulesByScope(ctx context.Context, scope string, device *userapi.Devi
 	}
 	ruleSet := pushRuleSetByScope(ruleSets, pushrules.Scope(scope))
 	if ruleSet == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rule set"), "pushRuleSetByScope failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rule set"), "pushRuleSetByScope failed")
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -63,12 +66,12 @@ func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi
 	}
 	ruleSet := pushRuleSetByScope(ruleSets, pushrules.Scope(scope))
 	if ruleSet == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rule set"), "pushRuleSetByScope failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rule set"), "pushRuleSetByScope failed")
 	}
 	rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind))
 	// Even if rulesPtr is not nil, there may not be any rules for this kind
 	if rulesPtr == nil || (rulesPtr != nil && len(*rulesPtr) == 0) {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rules kind"), "pushRuleSetKindPointer failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rules kind"), "pushRuleSetKindPointer failed")
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -83,15 +86,15 @@ func GetPushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device
 	}
 	ruleSet := pushRuleSetByScope(ruleSets, pushrules.Scope(scope))
 	if ruleSet == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rule set"), "pushRuleSetByScope failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rule set"), "pushRuleSetByScope failed")
 	}
 	rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind))
 	if rulesPtr == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rules kind"), "pushRuleSetKindPointer failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rules kind"), "pushRuleSetKindPointer failed")
 	}
 	i := pushRuleIndexByID(*rulesPtr, ruleID)
 	if i < 0 {
-		return errorResponse(ctx, jsonerror.NotFound("push rule ID not found"), "pushRuleIndexByID failed")
+		return errorResponse(ctx, spec.NotFound("push rule ID not found"), "pushRuleIndexByID failed")
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -104,14 +107,14 @@ func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID,
 	if err := json.NewDecoder(body).Decode(&newRule); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
+			JSON: spec.BadJSON(err.Error()),
 		}
 	}
 	newRule.RuleID = ruleID
 
 	errs := pushrules.ValidateRule(pushrules.Kind(kind), &newRule)
 	if len(errs) > 0 {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue(errs[0].Error()), "rule sanity check failed: %v", errs)
+		return errorResponse(ctx, spec.InvalidParam(errs[0].Error()), "rule sanity check failed: %v", errs)
 	}
 
 	ruleSets, err := userAPI.QueryPushRules(ctx, device.UserID)
@@ -120,12 +123,12 @@ func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID,
 	}
 	ruleSet := pushRuleSetByScope(ruleSets, pushrules.Scope(scope))
 	if ruleSet == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rule set"), "pushRuleSetByScope failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rule set"), "pushRuleSetByScope failed")
 	}
 	rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind))
 	if rulesPtr == nil {
 		// while this should be impossible (ValidateRule would already return an error), better keep it around
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rules kind"), "pushRuleSetKindPointer failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rules kind"), "pushRuleSetKindPointer failed")
 	}
 	i := pushRuleIndexByID(*rulesPtr, ruleID)
 	if i >= 0 && afterRuleID == "" && beforeRuleID == "" {
@@ -172,15 +175,15 @@ func DeletePushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, dev
 	}
 	ruleSet := pushRuleSetByScope(ruleSets, pushrules.Scope(scope))
 	if ruleSet == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rule set"), "pushRuleSetByScope failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rule set"), "pushRuleSetByScope failed")
 	}
 	rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind))
 	if rulesPtr == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rules kind"), "pushRuleSetKindPointer failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rules kind"), "pushRuleSetKindPointer failed")
 	}
 	i := pushRuleIndexByID(*rulesPtr, ruleID)
 	if i < 0 {
-		return errorResponse(ctx, jsonerror.NotFound("push rule ID not found"), "pushRuleIndexByID failed")
+		return errorResponse(ctx, spec.NotFound("push rule ID not found"), "pushRuleIndexByID failed")
 	}
 
 	*rulesPtr = append((*rulesPtr)[:i], (*rulesPtr)[i+1:]...)
@@ -203,15 +206,15 @@ func GetPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
 	}
 	ruleSet := pushRuleSetByScope(ruleSets, pushrules.Scope(scope))
 	if ruleSet == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rule set"), "pushRuleSetByScope failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rule set"), "pushRuleSetByScope failed")
 	}
 	rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind))
 	if rulesPtr == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rules kind"), "pushRuleSetKindPointer failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rules kind"), "pushRuleSetKindPointer failed")
 	}
 	i := pushRuleIndexByID(*rulesPtr, ruleID)
 	if i < 0 {
-		return errorResponse(ctx, jsonerror.NotFound("push rule ID not found"), "pushRuleIndexByID failed")
+		return errorResponse(ctx, spec.NotFound("push rule ID not found"), "pushRuleIndexByID failed")
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -226,7 +229,7 @@ func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
 	if err := json.NewDecoder(body).Decode(&newPartialRule); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
+			JSON: spec.BadJSON(err.Error()),
 		}
 	}
 	if newPartialRule.Actions == nil {
@@ -249,15 +252,15 @@ func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
 	}
 	ruleSet := pushRuleSetByScope(ruleSets, pushrules.Scope(scope))
 	if ruleSet == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rule set"), "pushRuleSetByScope failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rule set"), "pushRuleSetByScope failed")
 	}
 	rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind))
 	if rulesPtr == nil {
-		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rules kind"), "pushRuleSetKindPointer failed")
+		return errorResponse(ctx, spec.InvalidParam("invalid push rules kind"), "pushRuleSetKindPointer failed")
 	}
 	i := pushRuleIndexByID(*rulesPtr, ruleID)
 	if i < 0 {
-		return errorResponse(ctx, jsonerror.NotFound("push rule ID not found"), "pushRuleIndexByID failed")
+		return errorResponse(ctx, spec.NotFound("push rule ID not found"), "pushRuleIndexByID failed")
 	}
 
 	if !reflect.DeepEqual(attrGet((*rulesPtr)[i]), attrGet(&newPartialRule)) {
@@ -313,7 +316,7 @@ func pushRuleAttrGetter(attr string) (func(*pushrules.Rule) interface{}, error)
 	case "enabled":
 		return func(rule *pushrules.Rule) interface{} { return rule.Enabled }, nil
 	default:
-		return nil, jsonerror.InvalidArgumentValue("invalid push rule attribute")
+		return nil, spec.InvalidParam("invalid push rule attribute")
 	}
 }
 
@@ -324,7 +327,7 @@ func pushRuleAttrSetter(attr string) (func(dest, src *pushrules.Rule), error) {
 	case "enabled":
 		return func(dest, src *pushrules.Rule) { dest.Enabled = src.Enabled }, nil
 	default:
-		return nil, jsonerror.InvalidArgumentValue("invalid push rule attribute")
+		return nil, spec.InvalidParam("invalid push rule attribute")
 	}
 }
 
@@ -338,10 +341,10 @@ func findPushRuleInsertionIndex(rules []*pushrules.Rule, afterID, beforeID strin
 			}
 		}
 		if i == len(rules) {
-			return 0, jsonerror.NotFound("after: rule ID not found")
+			return 0, spec.NotFound("after: rule ID not found")
 		}
 		if rules[i].Default {
-			return 0, jsonerror.NotFound("after: rule ID must not be a default rule")
+			return 0, spec.NotFound("after: rule ID must not be a default rule")
 		}
 		// We stopped on the "after" match to differentiate
 		// not-found from is-last-entry. Now we move to the earliest
@@ -356,10 +359,10 @@ func findPushRuleInsertionIndex(rules []*pushrules.Rule, afterID, beforeID strin
 			}
 		}
 		if i == len(rules) {
-			return 0, jsonerror.NotFound("before: rule ID not found")
+			return 0, spec.NotFound("before: rule ID not found")
 		}
 		if rules[i].Default {
-			return 0, jsonerror.NotFound("before: rule ID must not be a default rule")
+			return 0, spec.NotFound("before: rule ID must not be a default rule")
 		}
 	}
 

clientapi/routing/receipt.go

@@ -20,17 +20,15 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 
-	"github.com/matrix-org/dendrite/userapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )
 
-func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
+func SetReceipt(req *http.Request, userAPI userapi.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
 	timestamp := spec.AsTimestamp(time.Now())
 	logrus.WithFields(logrus.Fields{
 		"roomID":      roomID,
@@ -49,16 +47,19 @@ func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *prod
 	case "m.fully_read":
 		data, err := json.Marshal(fullyReadEvent{EventID: eventID})
 		if err != nil {
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 
-		dataReq := api.InputAccountDataRequest{
+		dataReq := userapi.InputAccountDataRequest{
 			UserID:      device.UserID,
 			DataType:    "m.fully_read",
 			RoomID:      roomID,
 			AccountData: data,
 		}
-		dataRes := api.InputAccountDataResponse{}
+		dataRes := userapi.InputAccountDataResponse{}
 		if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
 			return util.ErrorResponse(err)

clientapi/routing/redaction.go

@@ -16,6 +16,7 @@ package routing
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"time"
 
@@ -24,7 +25,6 @@ import (
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
@@ -34,7 +34,8 @@ import (
 )
 
 type redactionContent struct {
-	Reason string `json:"reason"`
+	Reason  string `json:"reason"`
+	Redacts string `json:"redacts"`
 }
 
 type redactionResponse struct {
@@ -47,11 +48,43 @@ func SendRedaction(
 	txnID *string,
 	txnCache *transactions.Cache,
 ) util.JSONResponse {
-	resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, userIDErr := spec.NewUserID(device.UserID, true)
+	if userIDErr != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("userID doesn't have power level to redact"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, queryErr := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
+	if queryErr != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("userID doesn't have power level to redact"),
+		}
+	}
+
+	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}
 
+	// if user is member of room, and sender ID is nil, then this user doesn't have a pseudo ID for some reason,
+	// which is unexpected.
+	if senderID == nil {
+		util.GetLogger(req.Context()).WithField("userID", *deviceUserID).WithField("roomID", roomID).Error("missing sender ID for user, despite having membership")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
 	if txnID != nil {
 		// Try to fetch response from transactionsCache
 		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok {
@@ -63,20 +96,20 @@ func SendRedaction(
 	if ev == nil {
 		return util.JSONResponse{
 			Code: 400,
-			JSON: jsonerror.NotFound("unknown event ID"), // TODO: is it ok to leak existence?
+			JSON: spec.NotFound("unknown event ID"), // TODO: is it ok to leak existence?
 		}
 	}
-	if ev.RoomID() != roomID {
+	if ev.RoomID().String() != roomID {
 		return util.JSONResponse{
 			Code: 400,
-			JSON: jsonerror.NotFound("cannot redact event in another room"),
+			JSON: spec.NotFound("cannot redact event in another room"),
 		}
 	}
 
 	// "Users may redact their own events, and any user with a power level greater than or equal
 	// to the redact power level of the room may redact events there"
 	// https://matrix.org/docs/spec/client_server/r0.6.1#put-matrix-client-r0-rooms-roomid-redact-eventid-txnid
-	allowedToRedact := ev.Sender() == device.UserID
+	allowedToRedact := ev.SenderID() == *senderID
 	if !allowedToRedact {
 		plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
 			EventType: spec.MRoomPowerLevels,
@@ -85,24 +118,24 @@ func SendRedaction(
 		if plEvent == nil {
 			return util.JSONResponse{
 				Code: 403,
-				JSON: jsonerror.Forbidden("You don't have permission to redact this event, no power_levels event in this room."),
+				JSON: spec.Forbidden("You don't have permission to redact this event, no power_levels event in this room."),
 			}
 		}
-		pl, err := plEvent.PowerLevels()
-		if err != nil {
+		pl, plErr := plEvent.PowerLevels()
+		if plErr != nil {
 			return util.JSONResponse{
 				Code: 403,
-				JSON: jsonerror.Forbidden(
+				JSON: spec.Forbidden(
 					"You don't have permission to redact this event, the power_levels event for this room is malformed so auth checks cannot be performed.",
 				),
 			}
 		}
-		allowedToRedact = pl.UserLevel(device.UserID) >= pl.Redact
+		allowedToRedact = pl.UserLevel(*senderID) >= pl.Redact
 	}
 	if !allowedToRedact {
 		return util.JSONResponse{
 			Code: 403,
-			JSON: jsonerror.Forbidden("You don't have permission to redact this event, power level too low."),
+			JSON: spec.Forbidden("You don't have permission to redact this event, power level too low."),
 		}
 	}
 
@@ -114,34 +147,48 @@ func SendRedaction(
 
 	// create the new event and set all the fields we can
 	proto := gomatrixserverlib.ProtoEvent{
-		Sender:  device.UserID,
-		RoomID:  roomID,
-		Type:    spec.MRoomRedaction,
-		Redacts: eventID,
-	}
-	err := proto.SetContent(r)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("proto.SetContent failed")
-		return jsonerror.InternalServerError()
+		SenderID: string(*senderID),
+		RoomID:   roomID,
+		Type:     spec.MRoomRedaction,
+		Redacts:  eventID,
 	}
 
-	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+	// Room version 11 expects the "redacts" field on the
+	// content field, so add it here as well
+	r.Redacts = eventID
+
+	err = proto.SetContent(r)
 	if err != nil {
-		return jsonerror.InternalServerError()
+		util.GetLogger(req.Context()).WithError(err).Error("proto.SetContent failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	identity, err := rsAPI.SigningIdentityFor(req.Context(), *validRoomID, *deviceUserID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
-	e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, cfg.Matrix, identity, time.Now(), rsAPI, &queryRes)
-	if err == eventutil.ErrRoomNoExists {
+	e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, &identity, time.Now(), rsAPI, &queryRes)
+	if errors.Is(err, eventutil.ErrRoomNoExists{}) {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Room does not exist"),
+			JSON: spec.NotFound("Room does not exist"),
 		}
 	}
 	domain := device.UserDomain()
 	if err = roomserverAPI.SendEvents(context.Background(), rsAPI, roomserverAPI.KindNew, []*types.HeaderedEvent{e}, device.UserDomain(), domain, domain, nil, false); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	res := util.JSONResponse{

clientapi/routing/register.go

@@ -46,7 +46,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
@@ -237,7 +236,7 @@ type authDict struct {
 	// TODO: Lots of custom keys depending on the type
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
+// https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api
 type userInteractiveResponse struct {
 	Flows     []authtypes.Flow       `json:"flows"`
 	Completed []authtypes.LoginType  `json:"completed"`
@@ -257,7 +256,7 @@ func newUserInteractiveResponse(
 	}
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 type registerResponse struct {
 	UserID      string `json:"user_id"`
 	AccessToken string `json:"access_token,omitempty"`
@@ -428,7 +427,7 @@ func validateApplicationService(
 	if matchedApplicationService == nil {
 		return "", &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.UnknownToken("Supplied access_token does not match any known application service"),
+			JSON: spec.UnknownToken("Supplied access_token does not match any known application service"),
 		}
 	}
 
@@ -439,7 +438,7 @@ func validateApplicationService(
 		// If we didn't find any matches, return M_EXCLUSIVE
 		return "", &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.ASExclusive(fmt.Sprintf(
+			JSON: spec.ASExclusive(fmt.Sprintf(
 				"Supplied username %s did not match any namespaces for application service ID: %s", username, matchedApplicationService.ID)),
 		}
 	}
@@ -448,7 +447,7 @@ func validateApplicationService(
 	if UsernameMatchesMultipleExclusiveNamespaces(cfg, userID) {
 		return "", &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.ASExclusive(fmt.Sprintf(
+			JSON: spec.ASExclusive(fmt.Sprintf(
 				"Supplied username %s matches multiple exclusive application service namespaces. Only 1 match allowed", username)),
 		}
 	}
@@ -463,7 +462,7 @@ func validateApplicationService(
 }
 
 // Register processes a /register request.
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 func Register(
 	req *http.Request,
 	userAPI userapi.ClientUserAPI,
@@ -474,7 +473,7 @@ func Register(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.NotJSON("Unable to read request body"),
+			JSON: spec.NotJSON("Unable to read request body"),
 		}
 	}
 
@@ -518,7 +517,7 @@ func Register(
 	if _, err = strconv.ParseInt(r.Username, 10, 64); err == nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Numeric user IDs are reserved"),
+			JSON: spec.InvalidUsername("Numeric user IDs are reserved"),
 		}
 	}
 	// Auto generate a numeric username if r.Username is empty
@@ -529,7 +528,10 @@ func Register(
 		nres := &userapi.QueryNumericLocalpartResponse{}
 		if err = userAPI.QueryNumericLocalpart(req.Context(), nreq, nres); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryNumericLocalpart failed")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		r.Username = strconv.FormatInt(nres.ID, 10)
 	}
@@ -552,7 +554,7 @@ func Register(
 		// type is not known or specified)
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("A known registration type (e.g. m.login.application_service) must be specified if an access_token is provided"),
+			JSON: spec.MissingParam("A known registration type (e.g. m.login.application_service) must be specified if an access_token is provided"),
 		}
 	default:
 		// Spec-compliant case (neither the access_token nor the login type are
@@ -590,7 +592,7 @@ func handleGuestRegistration(
 	if !registrationEnabled || !guestsEnabled {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden(
+			JSON: spec.Forbidden(
 				fmt.Sprintf("Guest registration is disabled on %q", r.ServerName),
 			),
 		}
@@ -604,7 +606,7 @@ func handleGuestRegistration(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("failed to create account: " + err.Error()),
+			JSON: spec.Unknown("failed to create account: " + err.Error()),
 		}
 	}
 	token, err := tokens.GenerateLoginToken(tokens.TokenOptions{
@@ -616,7 +618,7 @@ func handleGuestRegistration(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("Failed to generate access token"),
+			JSON: spec.Unknown("Failed to generate access token"),
 		}
 	}
 	//we don't allow guests to specify their own device_id
@@ -628,11 +630,12 @@ func handleGuestRegistration(
 		AccessToken:       token,
 		IPAddr:            req.RemoteAddr,
 		UserAgent:         req.UserAgent(),
+		FromRegistration:  true,
 	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("failed to create device: " + err.Error()),
+			JSON: spec.Unknown("failed to create device: " + err.Error()),
 		}
 	}
 	return util.JSONResponse{
@@ -645,6 +648,16 @@ func handleGuestRegistration(
 	}
 }
 
+// localpartMatchesExclusiveNamespaces will check if a given username matches any
+// application service's exclusive users namespace
+func localpartMatchesExclusiveNamespaces(
+	cfg *config.ClientAPI,
+	localpart string,
+) bool {
+	userID := userutil.MakeUserID(localpart, cfg.Matrix.ServerName)
+	return cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(userID)
+}
+
 // handleRegistrationFlow will direct and complete registration flow stages
 // that the client has requested.
 // nolint: gocyclo
@@ -682,7 +695,7 @@ func handleRegistrationFlow(
 	if !registrationEnabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden(
+			JSON: spec.Forbidden(
 				fmt.Sprintf("Registration is disabled on %q", r.ServerName),
 			),
 		}
@@ -693,10 +706,10 @@ func handleRegistrationFlow(
 	// If an access token is provided, ignore this check this is an appservice
 	// request and we will validate in validateApplicationService
 	if len(cfg.Derived.ApplicationServices) != 0 &&
-		UsernameMatchesExclusiveNamespaces(cfg, r.Username) {
+		localpartMatchesExclusiveNamespaces(cfg, r.Username) {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.ASExclusive("This username is reserved by an application service."),
+			JSON: spec.ASExclusive("This username is reserved by an application service."),
 		}
 	}
 
@@ -706,15 +719,15 @@ func handleRegistrationFlow(
 		err := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
 		switch err {
 		case ErrCaptchaDisabled:
-			return util.JSONResponse{Code: http.StatusForbidden, JSON: jsonerror.Unknown(err.Error())}
+			return util.JSONResponse{Code: http.StatusForbidden, JSON: spec.Unknown(err.Error())}
 		case ErrMissingResponse:
-			return util.JSONResponse{Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(err.Error())}
+			return util.JSONResponse{Code: http.StatusBadRequest, JSON: spec.BadJSON(err.Error())}
 		case ErrInvalidCaptcha:
-			return util.JSONResponse{Code: http.StatusUnauthorized, JSON: jsonerror.BadJSON(err.Error())}
+			return util.JSONResponse{Code: http.StatusUnauthorized, JSON: spec.BadJSON(err.Error())}
 		case nil:
 		default:
 			util.GetLogger(req.Context()).WithError(err).Error("failed to validate recaptcha")
-			return util.JSONResponse{Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError()}
+			return util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}}
 		}
 
 		// Add Recaptcha to the list of completed registration stages
@@ -732,7 +745,7 @@ func handleRegistrationFlow(
 	default:
 		return util.JSONResponse{
 			Code: http.StatusNotImplemented,
-			JSON: jsonerror.Unknown("unknown/unimplemented auth type"),
+			JSON: spec.Unknown("unknown/unimplemented auth type"),
 		}
 	}
 
@@ -764,13 +777,13 @@ func handleApplicationServiceRegistration(
 	if tokenErr != nil {
 		return util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.MissingToken(tokenErr.Error()),
+			JSON: spec.MissingToken(tokenErr.Error()),
 		}
 	}
 
 	// Check application service register user request is valid.
 	// The application service's ID is returned if so.
-	appserviceID, err := validateApplicationService(
+	appserviceID, err := internal.ValidateApplicationServiceRequest(
 		cfg, r.Username, accessToken,
 	)
 	if err != nil {
@@ -834,14 +847,14 @@ func completeRegistration(
 	if username == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Missing username"),
+			JSON: spec.MissingParam("Missing username"),
 		}
 	}
 	// Blank passwords are only allowed by registered application services
 	if password == "" && appserviceID == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Missing password"),
+			JSON: spec.MissingParam("Missing password"),
 		}
 	}
 	var accRes userapi.PerformAccountCreationResponse
@@ -857,12 +870,12 @@ func completeRegistration(
 		if _, ok := err.(*userapi.ErrorConflict); ok { // user already exists
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.UserInUse("Desired user ID is already taken."),
+				JSON: spec.UserInUse("Desired user ID is already taken."),
 			}
 		}
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("failed to create account: " + err.Error()),
+			JSON: spec.Unknown("failed to create account: " + err.Error()),
 		}
 	}
 
@@ -884,7 +897,7 @@ func completeRegistration(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("Failed to generate access token"),
+			JSON: spec.Unknown("Failed to generate access token"),
 		}
 	}
 
@@ -893,7 +906,7 @@ func completeRegistration(
 		if err != nil {
 			return util.JSONResponse{
 				Code: http.StatusInternalServerError,
-				JSON: jsonerror.Unknown("failed to set display name: " + err.Error()),
+				JSON: spec.Unknown("failed to set display name: " + err.Error()),
 			}
 		}
 	}
@@ -907,11 +920,12 @@ func completeRegistration(
 		DeviceID:          deviceID,
 		IPAddr:            ipAddr,
 		UserAgent:         userAgent,
+		FromRegistration:  true,
 	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("failed to create device: " + err.Error()),
+			JSON: spec.Unknown("failed to create device: " + err.Error()),
 		}
 	}
 
@@ -1006,7 +1020,7 @@ func RegisterAvailable(
 		if v.ServerName == domain && !v.AllowRegistration {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden(
+				JSON: spec.Forbidden(
 					fmt.Sprintf("Registration is not allowed on %q", string(v.ServerName)),
 				),
 			}
@@ -1023,7 +1037,7 @@ func RegisterAvailable(
 		if appservice.OwnsNamespaceCoveringUserId(userID) {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.UserInUse("Desired user ID is reserved by an application service."),
+				JSON: spec.UserInUse("Desired user ID is reserved by an application service."),
 			}
 		}
 	}
@@ -1036,14 +1050,14 @@ func RegisterAvailable(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("failed to check availability:" + err.Error()),
+			JSON: spec.Unknown("failed to check availability:" + err.Error()),
 		}
 	}
 
 	if !res.Available {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.UserInUse("Desired User ID is already taken."),
+			JSON: spec.UserInUse("Desired User ID is already taken."),
 		}
 	}
 
@@ -1060,7 +1074,7 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
 	if err != nil {
 		return util.JSONResponse{
 			Code: 400,
-			JSON: jsonerror.BadJSON(fmt.Sprintf("malformed json: %s", err)),
+			JSON: spec.BadJSON(fmt.Sprintf("malformed json: %s", err)),
 		}
 	}
 	valid, err := sr.IsValidMacLogin(ssrr.Nonce, ssrr.User, ssrr.Password, ssrr.Admin, ssrr.MacBytes)
@@ -1070,7 +1084,7 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
 	if !valid {
 		return util.JSONResponse{
 			Code: 403,
-			JSON: jsonerror.Forbidden("bad mac"),
+			JSON: spec.Forbidden("bad mac"),
 		}
 	}
 	// downcase capitals

clientapi/routing/register_test.go

@@ -28,7 +28,6 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -39,6 +38,7 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/patrickmn/go-cache"
 	"github.com/stretchr/testify/assert"
@@ -298,53 +298,65 @@ func Test_register(t *testing.T) {
 		guestsDisabled       bool
 		enableRecaptcha      bool
 		captchaBody          string
-		wantResponse         util.JSONResponse
+		// in case of an error, the expected response
+		wantErrorResponse util.JSONResponse
+		// in case of success, the expected username assigned
+		wantUsername string
 	}{
 		{
 			name:           "disallow guests",
 			kind:           "guest",
 			guestsDisabled: true,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden(`Guest registration is disabled on "test"`),
+				JSON: spec.Forbidden(`Guest registration is disabled on "test"`),
 			},
 		},
 		{
-			name: "allow guests",
-			kind: "guest",
+			name:         "allow guests",
+			kind:         "guest",
+			wantUsername: "1",
 		},
 		{
 			name:      "unknown login type",
 			loginType: "im.not.known",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusNotImplemented,
-				JSON: jsonerror.Unknown("unknown/unimplemented auth type"),
+				JSON: spec.Unknown("unknown/unimplemented auth type"),
 			},
 		},
 		{
 			name:                 "disabled registration",
 			registrationDisabled: true,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden(`Registration is disabled on "test"`),
+				JSON: spec.Forbidden(`Registration is disabled on "test"`),
 			},
 		},
 		{
-			name:       "successful registration, numeric ID",
-			username:   "",
-			password:   "someRandomPassword",
-			forceEmpty: true,
+			name:         "successful registration, numeric ID",
+			username:     "",
+			password:     "someRandomPassword",
+			forceEmpty:   true,
+			wantUsername: "2",
 		},
 		{
 			name:     "successful registration",
 			username: "success",
 		},
+		{
+			name:         "successful registration, sequential numeric ID",
+			username:     "",
+			password:     "someRandomPassword",
+			forceEmpty:   true,
+			wantUsername: "3",
+		},
 		{
 			name:     "failing registration - user already exists",
 			username: "success",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.UserInUse("Desired user ID is already taken."),
+				JSON: spec.UserInUse("Desired user ID is already taken."),
 			},
 		},
 		{
@@ -352,33 +364,33 @@ func Test_register(t *testing.T) {
 			username: "LOWERCASED", // this is going to be lower-cased
 		},
 		{
-			name:         "invalid username",
-			username:     "#totalyNotValid",
-			wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
+			name:              "invalid username",
+			username:          "#totalyNotValid",
+			wantErrorResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
 		},
 		{
 			name:     "numeric username is forbidden",
 			username: "1337",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidUsername("Numeric user IDs are reserved"),
+				JSON: spec.InvalidUsername("Numeric user IDs are reserved"),
 			},
 		},
 		{
 			name:      "disabled recaptcha login",
 			loginType: authtypes.LoginTypeRecaptcha,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Unknown(ErrCaptchaDisabled.Error()),
+				JSON: spec.Unknown(ErrCaptchaDisabled.Error()),
 			},
 		},
 		{
 			name:            "enabled recaptcha, no response defined",
 			enableRecaptcha: true,
 			loginType:       authtypes.LoginTypeRecaptcha,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.BadJSON(ErrMissingResponse.Error()),
+				JSON: spec.BadJSON(ErrMissingResponse.Error()),
 			},
 		},
 		{
@@ -386,9 +398,9 @@ func Test_register(t *testing.T) {
 			enableRecaptcha: true,
 			loginType:       authtypes.LoginTypeRecaptcha,
 			captchaBody:     `notvalid`,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusUnauthorized,
-				JSON: jsonerror.BadJSON(ErrInvalidCaptcha.Error()),
+				JSON: spec.BadJSON(ErrInvalidCaptcha.Error()),
 			},
 		},
 		{
@@ -398,11 +410,11 @@ func Test_register(t *testing.T) {
 			captchaBody:     `success`,
 		},
 		{
-			name:            "captcha invalid from remote",
-			enableRecaptcha: true,
-			loginType:       authtypes.LoginTypeRecaptcha,
-			captchaBody:     `i should fail for other reasons`,
-			wantResponse:    util.JSONResponse{Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError()},
+			name:              "captcha invalid from remote",
+			enableRecaptcha:   true,
+			loginType:         authtypes.LoginTypeRecaptcha,
+			captchaBody:       `i should fail for other reasons`,
+			wantErrorResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}},
 		},
 	}
 
@@ -415,7 +427,8 @@ func Test_register(t *testing.T) {
 
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
@@ -484,9 +497,9 @@ func Test_register(t *testing.T) {
 					if !reflect.DeepEqual(r.Flows, cfg.Derived.Registration.Flows) {
 						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows)
 					}
-				case *jsonerror.MatrixError:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse)
+				case spec.MatrixError:
+					if !reflect.DeepEqual(tc.wantErrorResponse, resp) {
+						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantErrorResponse)
 					}
 					return
 				case registerResponse:
@@ -504,6 +517,13 @@ func Test_register(t *testing.T) {
 					if r.DeviceID == "" {
 						t.Fatalf("missing deviceID in response")
 					}
+					// if an expected username is provided, assert that it is a match
+					if tc.wantUsername != "" {
+						wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test"))
+						if wantUserID != r.UserID {
+							t.Fatalf("unexpected userID: %s, want %s", r.UserID, wantUserID)
+						}
+					}
 					return
 				default:
 					t.Logf("Got response: %T", resp.JSON)
@@ -540,39 +560,29 @@ func Test_register(t *testing.T) {
 
 				resp = Register(req, userAPI, &cfg.ClientAPI)
 
-				switch resp.JSON.(type) {
-				case *jsonerror.MatrixError:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
+				switch rr := resp.JSON.(type) {
+				case spec.InternalServerError, spec.MatrixError, util.JSONResponse:
+					if !reflect.DeepEqual(tc.wantErrorResponse, resp) {
+						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantErrorResponse)
 					}
 					return
-				case util.JSONResponse:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
+				case registerResponse:
+					// validate the response
+					if tc.wantUsername != "" {
+						// if an expected username is provided, assert that it is a match
+						wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test"))
+						if wantUserID != rr.UserID {
+							t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID)
+						}
 					}
-					return
-				}
-
-				rr, ok := resp.JSON.(registerResponse)
-				if !ok {
-					t.Fatalf("expected a registerresponse, got %T", resp.JSON)
-				}
-
-				// validate the response
-				if tc.forceEmpty {
-					// when not supplying a username, one will be generated. Given this _SHOULD_ be
-					// the second user, set the username accordingly
-					reg.Username = "2"
-				}
-				wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test"))
-				if wantUserID != rr.UserID {
-					t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID)
-				}
-				if rr.DeviceID != *reg.DeviceID {
-					t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID)
-				}
-				if rr.AccessToken == "" {
-					t.Fatalf("missing accessToken in response")
+					if rr.DeviceID != *reg.DeviceID {
+						t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID)
+					}
+					if rr.AccessToken == "" {
+						t.Fatalf("missing accessToken in response")
+					}
+				default:
+					t.Fatalf("expected one of internalservererror, matrixerror, jsonresponse, registerresponse, got %T", resp.JSON)
 				}
 			})
 		}
@@ -589,7 +599,8 @@ func TestRegisterUserWithDisplayName(t *testing.T) {
 		natsInstance := jetstream.NATSInstance{}
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		deviceName, deviceID := "deviceName", "deviceID"
 		expectedDisplayName := "DisplayName"
 		response := completeRegistration(
@@ -629,7 +640,8 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		expectedDisplayName := "rabbit"
 		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)

clientapi/routing/report_event.go

@@ -0,0 +1,93 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userAPI "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+)
+
+type reportEventRequest struct {
+	Reason string `json:"reason"`
+	Score  int64  `json:"score"`
+}
+
+func ReportEvent(
+	req *http.Request,
+	device *userAPI.Device,
+	roomID, eventID string,
+	rsAPI api.ClientRoomserverAPI,
+) util.JSONResponse {
+	defer req.Body.Close() // nolint: errcheck
+
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.NotFound("You don't have permission to report this event, bad userID"),
+		}
+	}
+	// The requesting user must be a member of the room
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
+	if errRes != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound, // Spec demands this...
+			JSON: spec.NotFound("The event was not found or you are not joined to the room."),
+		}
+	}
+
+	// Parse the request
+	report := reportEventRequest{}
+	if resErr := httputil.UnmarshalJSONRequest(req, &report); resErr != nil {
+		return *resErr
+	}
+
+	queryRes := &api.QueryEventsByIDResponse{}
+	if err = rsAPI.QueryEventsByID(req.Context(), &api.QueryEventsByIDRequest{
+		RoomID:   roomID,
+		EventIDs: []string{eventID},
+	}, queryRes); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{Err: err.Error()},
+		}
+	}
+
+	// No event was found or it was already redacted
+	if len(queryRes.Events) == 0 || queryRes.Events[0].Redacted() {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The event was not found or you are not joined to the room."),
+		}
+	}
+
+	_, err = rsAPI.InsertReportedEvent(req.Context(), roomID, eventID, device.UserID, report.Reason, report.Score)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{Err: err.Error()},
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/room_hierarchy.go

@@ -0,0 +1,180 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+	"strconv"
+	"sync"
+
+	"github.com/google/uuid"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+	log "github.com/sirupsen/logrus"
+)
+
+// For storing pagination information for room hierarchies
+type RoomHierarchyPaginationCache struct {
+	cache map[string]roomserverAPI.RoomHierarchyWalker
+	mu    sync.Mutex
+}
+
+// Create a new, empty, pagination cache.
+func NewRoomHierarchyPaginationCache() RoomHierarchyPaginationCache {
+	return RoomHierarchyPaginationCache{
+		cache: map[string]roomserverAPI.RoomHierarchyWalker{},
+	}
+}
+
+// Get a cached page, or nil if there is no associated page in the cache.
+func (c *RoomHierarchyPaginationCache) Get(token string) *roomserverAPI.RoomHierarchyWalker {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	line, ok := c.cache[token]
+	if ok {
+		return &line
+	} else {
+		return nil
+	}
+}
+
+// Add a cache line to the pagination cache.
+func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.RoomHierarchyWalker) string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	token := uuid.NewString()
+	c.cache[token] = line
+	return token
+}
+
+// Query the hierarchy of a room/space
+//
+// Implements /_matrix/client/v1/rooms/{roomID}/hierarchy
+func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr string, rsAPI roomserverAPI.ClientRoomserverAPI, paginationCache *RoomHierarchyPaginationCache) util.JSONResponse {
+	parsedRoomID, err := spec.NewRoomID(roomIDStr)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.InvalidParam("room is unknown/forbidden"),
+		}
+	}
+	roomID := *parsedRoomID
+
+	suggestedOnly := false // Defaults to false (spec-defined)
+	switch req.URL.Query().Get("suggested_only") {
+	case "true":
+		suggestedOnly = true
+	case "false":
+	case "": // Empty string is returned when query param is not set
+	default:
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"),
+		}
+	}
+
+	limit := 1000 // Default to 1000
+	limitStr := req.URL.Query().Get("limit")
+	if limitStr != "" {
+		var maybeLimit int
+		maybeLimit, err = strconv.Atoi(limitStr)
+		if err != nil || maybeLimit < 0 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("query parameter 'limit', if set, must be a positive integer"),
+			}
+		}
+		limit = maybeLimit
+		if limit > 1000 {
+			limit = 1000 // Maximum limit of 1000
+		}
+	}
+
+	maxDepth := -1 // '-1' representing no maximum depth
+	maxDepthStr := req.URL.Query().Get("max_depth")
+	if maxDepthStr != "" {
+		var maybeMaxDepth int
+		maybeMaxDepth, err = strconv.Atoi(maxDepthStr)
+		if err != nil || maybeMaxDepth < 0 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("query parameter 'max_depth', if set, must be a positive integer"),
+			}
+		}
+		maxDepth = maybeMaxDepth
+	}
+
+	from := req.URL.Query().Get("from")
+
+	var walker roomserverAPI.RoomHierarchyWalker
+	if from == "" { // No pagination token provided, so start new hierarchy walker
+		walker = roomserverAPI.NewRoomHierarchyWalker(types.NewDeviceNotServerName(*device), roomID, suggestedOnly, maxDepth)
+	} else { // Attempt to resume cached walker
+		cachedWalker := paginationCache.Get(from)
+
+		if cachedWalker == nil || cachedWalker.SuggestedOnly != suggestedOnly || cachedWalker.MaxDepth != maxDepth {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("pagination not found for provided token ('from') with given 'max_depth', 'suggested_only' and room ID"),
+			}
+		}
+
+		walker = *cachedWalker
+	}
+
+	discoveredRooms, _, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit)
+
+	if err != nil {
+		switch err.(type) {
+		case roomserverAPI.ErrRoomUnknownOrNotAllowed:
+			util.GetLogger(req.Context()).WithError(err).Debugln("room unknown/forbidden when handling CS room hierarchy request")
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.Forbidden("room is unknown/forbidden"),
+			}
+		default:
+			log.WithError(err).Errorf("failed to fetch next page of room hierarchy (CS API)")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+	}
+
+	nextBatch := ""
+	// nextWalker will be nil if there's no more rooms left to walk
+	if nextWalker != nil {
+		nextBatch = paginationCache.AddLine(*nextWalker)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: RoomHierarchyClientResponse{
+			Rooms:     discoveredRooms,
+			NextBatch: nextBatch,
+		},
+	}
+
+}
+
+// Success response for /_matrix/client/v1/rooms/{roomID}/hierarchy
+type RoomHierarchyClientResponse struct {
+	Rooms     []fclient.RoomHierarchyRoom `json:"rooms"`
+	NextBatch string                      `json:"next_batch,omitempty"`
+}

clientapi/routing/room_tagging.go

@@ -19,10 +19,10 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -39,14 +39,17 @@ func GetTags(
 	if device.UserID != userID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Cannot retrieve another user's tags"),
+			JSON: spec.Forbidden("Cannot retrieve another user's tags"),
 		}
 	}
 
 	tagContent, err := obtainSavedTags(req, userID, roomID, userAPI)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -71,7 +74,7 @@ func PutTag(
 	if device.UserID != userID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Cannot modify another user's tags"),
+			JSON: spec.Forbidden("Cannot modify another user's tags"),
 		}
 	}
 
@@ -83,7 +86,10 @@ func PutTag(
 	tagContent, err := obtainSavedTags(req, userID, roomID, userAPI)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	if tagContent.Tags == nil {
@@ -93,7 +99,10 @@ func PutTag(
 
 	if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -118,14 +127,17 @@ func DeleteTag(
 	if device.UserID != userID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Cannot modify another user's tags"),
+			JSON: spec.Forbidden("Cannot modify another user's tags"),
 		}
 	}
 
 	tagContent, err := obtainSavedTags(req, userID, roomID, userAPI)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	// Check whether the tag to be deleted exists
@@ -141,7 +153,10 @@ func DeleteTag(
 
 	if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/routing.go

@@ -20,20 +20,21 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/setup/base"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/singleflight"
+
+	"github.com/matrix-org/dendrite/setup/base"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	clientutil "github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -43,6 +44,19 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 )
 
+type WellKnownClientHomeserver struct {
+	BaseUrl string `json:"base_url"`
+}
+
+type WellKnownSlidingSyncProxy struct {
+	Url string `json:"url"`
+}
+
+type WellKnownClientResponse struct {
+	Homeserver       WellKnownClientHomeserver  `json:"m.homeserver"`
+	SlidingSyncProxy *WellKnownSlidingSyncProxy `json:"org.matrix.msc3575.proxy,omitempty"`
+}
+
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
 //
@@ -85,22 +99,32 @@ func Setup(
 		unstableFeatures["org.matrix."+msc] = true
 	}
 
+	// singleflight protects /join endpoints from being invoked
+	// multiple times from the same user and room, otherwise
+	// a state reset can occur. This also avoids unneeded
+	// state calculations.
+	// TODO: actually fix this in the roomserver, as there are
+	// 		 possibly other ways that can result in a stat reset.
+	sf := singleflight.Group{}
+
 	if cfg.Matrix.WellKnownClientName != "" {
 		logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName)
+		if cfg.Matrix.WellKnownSlidingSyncProxy != "" {
+			logrus.Infof("Setting org.matrix.msc3575.proxy url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownSlidingSyncProxy)
+		}
 		wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse {
+			response := WellKnownClientResponse{
+				Homeserver: WellKnownClientHomeserver{cfg.Matrix.WellKnownClientName},
+			}
+			if cfg.Matrix.WellKnownSlidingSyncProxy != "" {
+				response.SlidingSyncProxy = &WellKnownSlidingSyncProxy{
+					Url: cfg.Matrix.WellKnownSlidingSyncProxy,
+				}
+			}
+
 			return util.JSONResponse{
 				Code: http.StatusOK,
-				JSON: struct {
-					HomeserverName struct {
-						BaseUrl string `json:"base_url"`
-					} `json:"m.homeserver"`
-				}{
-					HomeserverName: struct {
-						BaseUrl string `json:"base_url"`
-					}{
-						BaseUrl: cfg.Matrix.WellKnownClientName,
-					},
-				},
+				JSON: response,
 			}
 		})).Methods(http.MethodGet, http.MethodOptions)
 	}
@@ -148,11 +172,41 @@ func Setup(
 				}
 				return util.JSONResponse{
 					Code: http.StatusMethodNotAllowed,
-					JSON: jsonerror.NotFound("unknown method"),
+					JSON: spec.NotFound("unknown method"),
 				}
 			}),
 		).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 	}
+	dendriteAdminRouter.Handle("/admin/registrationTokens/new",
+		httputil.MakeAdminAPI("admin_registration_tokens_new", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminCreateNewRegistrationToken(req, cfg, userAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	dendriteAdminRouter.Handle("/admin/registrationTokens",
+		httputil.MakeAdminAPI("admin_list_registration_tokens", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminListRegistrationTokens(req, cfg, userAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	dendriteAdminRouter.Handle("/admin/registrationTokens/{token}",
+		httputil.MakeAdminAPI("admin_get_registration_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			switch req.Method {
+			case http.MethodGet:
+				return AdminGetRegistrationToken(req, cfg, userAPI)
+			case http.MethodPut:
+				return AdminUpdateRegistrationToken(req, cfg, userAPI)
+			case http.MethodDelete:
+				return AdminDeleteRegistrationToken(req, cfg, userAPI)
+			default:
+				return util.MatrixErrorResponse(
+					404,
+					string(spec.ErrorNotFound),
+					"unknown method",
+				)
+			}
+		}),
+	).Methods(http.MethodGet, http.MethodPut, http.MethodDelete, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
 		httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -201,7 +255,7 @@ func Setup(
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
 		serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
 		if err != nil {
-			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+			logrus.WithError(err).Fatal("unable to get account for sending server notices")
 		}
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
@@ -249,6 +303,8 @@ func Setup(
 	// Note that 'apiversion' is chosen because it must not collide with a variable used in any of the routing!
 	v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
 
+	v1mux := publicAPIMux.PathPrefix("/v1/").Subrouter()
+
 	unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter()
 
 	v3mux.Handle("/createRoom",
@@ -265,9 +321,17 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return JoinRoomByIDOrAlias(
-				req, device, rsAPI, userAPI, vars["roomIDOrAlias"],
-			)
+			// Only execute a join for roomIDOrAlias and UserID once. If there is a join in progress
+			// it waits for it to complete and returns that result for subsequent requests.
+			resp, _, _ := sf.Do(vars["roomIDOrAlias"]+device.UserID, func() (any, error) {
+				return JoinRoomByIDOrAlias(
+					req, device, rsAPI, userAPI, vars["roomIDOrAlias"],
+				), nil
+			})
+			// once all joins are processed, drop them from the cache. Further requests
+			// will be processed as usual.
+			sf.Forget(vars["roomIDOrAlias"] + device.UserID)
+			return resp.(util.JSONResponse)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -301,9 +365,17 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return JoinRoomByIDOrAlias(
-				req, device, rsAPI, userAPI, vars["roomID"],
-			)
+			// Only execute a join for roomID and UserID once. If there is a join in progress
+			// it waits for it to complete and returns that result for subsequent requests.
+			resp, _, _ := sf.Do(vars["roomID"]+device.UserID, func() (any, error) {
+				return JoinRoomByIDOrAlias(
+					req, device, rsAPI, userAPI, vars["roomID"],
+				), nil
+			})
+			// once all joins are processed, drop them from the cache. Further requests
+			// will be processed as usual.
+			sf.Forget(vars["roomID"] + device.UserID)
+			return resp.(util.JSONResponse)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/leave",
@@ -450,6 +522,19 @@ func Setup(
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPut, http.MethodOptions)
 
+	// Defined outside of handler to persist between calls
+	// TODO: clear based on some criteria
+	roomHierarchyPaginationCache := NewRoomHierarchyPaginationCache()
+	v1mux.Handle("/rooms/{roomID}/hierarchy",
+		httputil.MakeAuthAPI("spaces", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return QueryRoomHierarchy(req, device, vars["roomID"], rsAPI, &roomHierarchyPaginationCache)
+		}, httputil.WithAllowGuests()),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
 		if r := rateLimits.Limit(req, nil); r != nil {
 			return *r
@@ -659,7 +744,7 @@ func Setup(
 		httputil.MakeAuthAPI("push_rules", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidArgumentValue("missing trailing slash"),
+				JSON: spec.InvalidParam("missing trailing slash"),
 			}
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -674,7 +759,7 @@ func Setup(
 		httputil.MakeAuthAPI("push_rules", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidArgumentValue("scope, kind and rule ID must be specified"),
+				JSON: spec.InvalidParam("scope, kind and rule ID must be specified"),
 			}
 		}),
 	).Methods(http.MethodPut)
@@ -693,7 +778,7 @@ func Setup(
 		httputil.MakeAuthAPI("push_rules", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidArgumentValue("missing trailing slash after scope"),
+				JSON: spec.InvalidParam("missing trailing slash after scope"),
 			}
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -702,7 +787,7 @@ func Setup(
 		httputil.MakeAuthAPI("push_rules", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidArgumentValue("kind and rule ID must be specified"),
+				JSON: spec.InvalidParam("kind and rule ID must be specified"),
 			}
 		}),
 	).Methods(http.MethodPut)
@@ -721,7 +806,7 @@ func Setup(
 		httputil.MakeAuthAPI("push_rules", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidArgumentValue("missing trailing slash after kind"),
+				JSON: spec.InvalidParam("missing trailing slash after kind"),
 			}
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -730,7 +815,7 @@ func Setup(
 		httputil.MakeAuthAPI("push_rules", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidArgumentValue("rule ID must be specified"),
+				JSON: spec.InvalidParam("rule ID must be specified"),
 			}
 		}),
 	).Methods(http.MethodPut)
@@ -939,7 +1024,7 @@ func Setup(
 			// TODO: Allow people to peek into rooms.
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.GuestAccessForbidden("Guest access not implemented"),
+				JSON: spec.GuestAccessForbidden("Guest access not implemented"),
 			}
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -1186,7 +1271,7 @@ func Setup(
 			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
-			return GetCapabilities()
+			return GetCapabilities(rsAPI)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -1244,7 +1329,7 @@ func Setup(
 		if version == "" {
 			return util.JSONResponse{
 				Code: 400,
-				JSON: jsonerror.InvalidArgumentValue("version must be specified"),
+				JSON: spec.InvalidParam("version must be specified"),
 			}
 		}
 		var reqBody keyBackupSessionRequest
@@ -1265,7 +1350,7 @@ func Setup(
 		if version == "" {
 			return util.JSONResponse{
 				Code: 400,
-				JSON: jsonerror.InvalidArgumentValue("version must be specified"),
+				JSON: spec.InvalidParam("version must be specified"),
 			}
 		}
 		roomID := vars["roomID"]
@@ -1297,7 +1382,7 @@ func Setup(
 		if version == "" {
 			return util.JSONResponse{
 				Code: 400,
-				JSON: jsonerror.InvalidArgumentValue("version must be specified"),
+				JSON: spec.InvalidParam("version must be specified"),
 			}
 		}
 		var reqBody userapi.KeyBackupSession
@@ -1363,7 +1448,7 @@ func Setup(
 	// Cross-signing device keys
 
 	postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg)
+		return UploadCrossSigningDeviceKeys(req, userAPI, device, userAPI, cfg)
 	})
 
 	postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -1428,4 +1513,58 @@ func Setup(
 			return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomID}/joined_members",
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetJoinedMembers(req, device, vars["roomID"], rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomID}/report/{eventID}",
+		httputil.MakeAuthAPI("report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return ReportEvent(req, device, vars["roomID"], vars["eventID"], rsAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports",
+		httputil.MakeAdminAPI("admin_report_events", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			from := parseUint64OrDefault(req.URL.Query().Get("from"), 0)
+			limit := parseUint64OrDefault(req.URL.Query().Get("limit"), 100)
+			dir := req.URL.Query().Get("dir")
+			userID := req.URL.Query().Get("user_id")
+			roomID := req.URL.Query().Get("room_id")
+
+			// Go backwards if direction is empty or "b"
+			backwards := dir == "" || dir == "b"
+			return GetEventReports(req, rsAPI, from, limit, backwards, userID, roomID)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}",
+		httputil.MakeAdminAPI("admin_report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetEventReport(req, rsAPI, vars["reportID"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}",
+		httputil.MakeAdminAPI("admin_report_event_delete", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return DeleteEventReport(req, rsAPI, vars["reportID"])
+		}),
+	).Methods(http.MethodDelete, http.MethodOptions)
 }

clientapi/routing/sendevent.go

@@ -23,20 +23,19 @@ import (
 	"sync"
 	"time"
 
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
-	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
-
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/synctypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 )
 
 // http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid
@@ -69,6 +68,8 @@ var sendEventDuration = prometheus.NewHistogramVec(
 //	/rooms/{roomID}/send/{eventType}
 //	/rooms/{roomID}/send/{eventType}/{txnID}
 //	/rooms/{roomID}/state/{eventType}/{stateKey}
+//
+// nolint: gocyclo
 func SendEvent(
 	req *http.Request,
 	device *userapi.Device,
@@ -81,7 +82,7 @@ func SendEvent(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
+			JSON: spec.UnsupportedRoomVersion(err.Error()),
 		}
 	}
 
@@ -92,6 +93,30 @@ func SendEvent(
 		}
 	}
 
+	// Translate user ID state keys to room keys in pseudo ID rooms
+	if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && stateKey != nil {
+		parsedRoomID, innerErr := spec.NewRoomID(roomID)
+		if innerErr != nil {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("invalid room ID"),
+			}
+		}
+
+		newStateKey, innerErr := synctypes.FromClientStateKey(*parsedRoomID, *stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+			return rsAPI.QuerySenderIDForUser(req.Context(), roomID, userID)
+		})
+		if innerErr != nil {
+			// TODO: work out better logic for failure cases (e.g. sender ID not found)
+			util.GetLogger(req.Context()).WithError(innerErr).Error("synctypes.FromClientStateKey failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+		stateKey = newStateKey
+	}
+
 	// create a mutex for the specific user in the specific room
 	// this avoids a situation where events that are received in quick succession are sent to the roomserver in a jumbled order
 	userID := device.UserID
@@ -122,15 +147,26 @@ func SendEvent(
 		delete(r, "join_authorised_via_users_server")
 	}
 
+	// for power level events we need to replace the userID with the pseudoID
+	if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && eventType == spec.MRoomPowerLevels {
+		err = updatePowerLevels(req, r, roomID, rsAPI)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{Err: err.Error()},
+			}
+		}
+	}
+
 	evTime, err := httputil.ParseTSParam(req)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+			JSON: spec.InvalidParam(err.Error()),
 		}
 	}
 
-	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, cfg, rsAPI, evTime)
+	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, rsAPI, evTime)
 	if resErr != nil {
 		return *resErr
 	}
@@ -145,12 +181,15 @@ func SendEvent(
 		if !aliasReq.Valid() {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidParam("Request contains invalid aliases."),
+				JSON: spec.InvalidParam("Request contains invalid aliases."),
 			}
 		}
 		aliasRes := &api.GetAliasesForRoomIDResponse{}
 		if err = rsAPI.GetAliasesForRoomID(req.Context(), &api.GetAliasesForRoomIDRequest{RoomID: roomID}, aliasRes); err != nil {
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		var found int
 		requestAliases := append(aliasReq.AltAliases, aliasReq.Alias)
@@ -165,7 +204,7 @@ func SendEvent(
 		if aliasReq.Alias != "" && found < len(requestAliases) {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.BadAlias("No matching alias found."),
+				JSON: spec.BadAlias("No matching alias found."),
 			}
 		}
 	}
@@ -185,7 +224,7 @@ func SendEvent(
 		req.Context(), rsAPI,
 		api.KindNew,
 		[]*types.HeaderedEvent{
-			&types.HeaderedEvent{PDU: e},
+			{PDU: e},
 		},
 		device.UserDomain(),
 		domain,
@@ -194,7 +233,10 @@ func SendEvent(
 		false,
 	); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	timeToSubmitEvent := time.Since(startedSubmittingEvent)
 	util.GetLogger(req.Context()).WithFields(logrus.Fields{
@@ -220,6 +262,35 @@ func SendEvent(
 	return res
 }
 
+func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID string, rsAPI api.ClientRoomserverAPI) error {
+	users, ok := r["users"]
+	if !ok {
+		return nil
+	}
+	userMap := users.(map[string]interface{})
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return err
+	}
+	for user, level := range userMap {
+		uID, err := spec.NewUserID(user, true)
+		if err != nil {
+			continue // we're modifying the map in place, so we're going to have invalid userIDs after the first iteration
+		}
+		senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *uID)
+		if err != nil {
+			return err
+		} else if senderID == nil {
+			util.GetLogger(req.Context()).Warnf("sender ID not found for %s in %s", uID, *validRoomID)
+			continue
+		}
+		userMap[string(*senderID)] = level
+		delete(userMap, user)
+	}
+	r["users"] = userMap
+	return nil
+}
+
 // stateEqual compares the new and the existing state event content. If they are equal, returns a *util.JSONResponse
 // with the existing event_id, making this an idempotent request.
 func stateEqual(ctx context.Context, rsAPI api.ClientRoomserverAPI, eventType, stateKey, roomID string, newContent map[string]interface{}) *util.JSONResponse {
@@ -256,60 +327,94 @@ func generateSendEvent(
 	r map[string]interface{},
 	device *userapi.Device,
 	roomID, eventType string, stateKey *string,
-	cfg *config.ClientAPI,
 	rsAPI api.ClientRoomserverAPI,
 	evTime time.Time,
 ) (gomatrixserverlib.PDU, *util.JSONResponse) {
 	// parse the incoming http request
-	userID := device.UserID
+	fullUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("Bad userID"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.NotFound("internal server error"),
+		}
+	} else if senderID == nil {
+		// TODO: is it always the case that lack of a sender ID means they're not joined?
+		//       And should this logic be deferred to the roomserver somehow?
+		return nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("not joined to room"),
+		}
+	}
 
 	// create the new event and set all the fields we can
 	proto := gomatrixserverlib.ProtoEvent{
-		Sender:   userID,
+		SenderID: string(*senderID),
 		RoomID:   roomID,
 		Type:     eventType,
 		StateKey: stateKey,
 	}
-	err := proto.SetContent(r)
+	err = proto.SetContent(r)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("proto.SetContent failed")
-		resErr := jsonerror.InternalServerError()
-		return nil, &resErr
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
-	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+	identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *fullUserID)
 	if err != nil {
-		resErr := jsonerror.InternalServerError()
-		return nil, &resErr
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := eventutil.QueryAndBuildEvent(ctx, &proto, cfg.Matrix, identity, evTime, rsAPI, &queryRes)
-	if err == eventutil.ErrRoomNoExists {
+	e, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, evTime, rsAPI, &queryRes)
+	switch specificErr := err.(type) {
+	case nil:
+	case eventutil.ErrRoomNoExists:
 		return nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Room does not exist"),
+			JSON: spec.NotFound("Room does not exist"),
 		}
-	} else if e, ok := err.(gomatrixserverlib.BadJSONError); ok {
+	case gomatrixserverlib.BadJSONError:
 		return nil, &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(e.Error()),
+			JSON: spec.BadJSON(specificErr.Error()),
 		}
-	} else if e, ok := err.(gomatrixserverlib.EventValidationError); ok {
-		if e.Code == gomatrixserverlib.EventValidationTooLarge {
+	case gomatrixserverlib.EventValidationError:
+		if specificErr.Code == gomatrixserverlib.EventValidationTooLarge {
 			return nil, &util.JSONResponse{
 				Code: http.StatusRequestEntityTooLarge,
-				JSON: jsonerror.BadJSON(e.Error()),
+				JSON: spec.BadJSON(specificErr.Error()),
 			}
 		}
 		return nil, &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(e.Error()),
+			JSON: spec.BadJSON(specificErr.Error()),
 		}
-	} else if err != nil {
+	default:
 		util.GetLogger(ctx).WithError(err).Error("eventutil.BuildEvent failed")
-		resErr := jsonerror.InternalServerError()
-		return nil, &resErr
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	// check to see if this user can perform this operation
@@ -318,10 +423,12 @@ func generateSendEvent(
 		stateEvents[i] = queryRes.StateEvents[i].PDU
 	}
 	provider := gomatrixserverlib.NewAuthEvents(gomatrixserverlib.ToPDUs(stateEvents))
-	if err = gomatrixserverlib.Allowed(e.PDU, &provider); err != nil {
+	if err = gomatrixserverlib.Allowed(e.PDU, &provider, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return rsAPI.QueryUserIDForSender(ctx, *validRoomID, senderID)
+	}); err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
+			JSON: spec.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
 		}
 	}
 
@@ -332,13 +439,13 @@ func generateSendEvent(
 			util.GetLogger(ctx).WithError(err).Error("Cannot unmarshal the event content.")
 			return nil, &util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.BadJSON("Cannot unmarshal the event content."),
+				JSON: spec.BadJSON("Cannot unmarshal the event content."),
 			}
 		}
-		if content["replacement_room"] == e.RoomID() {
+		if content["replacement_room"] == e.RoomID().String() {
 			return nil, &util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidParam("Cannot send tombstone event that points to the same room."),
+				JSON: spec.InvalidParam("Cannot send tombstone event that points to the same room."),
 			}
 		}
 	}

clientapi/routing/sendevent_test.go

@@ -0,0 +1,275 @@
+package routing
+
+import (
+	"context"
+	"crypto/ed25519"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"gotest.tools/v3/assert"
+)
+
+// Mock roomserver API for testing
+//
+// Currently pretty specialised for the pseudo ID test, so will need
+// editing if future (other) sendevent tests are using this.
+type sendEventTestRoomserverAPI struct {
+	rsapi.ClientRoomserverAPI
+	t           *testing.T
+	roomIDStr   string
+	roomVersion gomatrixserverlib.RoomVersion
+	roomState   []*types.HeaderedEvent
+
+	// userID -> room key
+	senderMapping map[string]ed25519.PrivateKey
+
+	savedInputRoomEvents []rsapi.InputRoomEvent
+}
+
+func (s *sendEventTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) {
+	if roomID == s.roomIDStr {
+		return s.roomVersion, nil
+	} else {
+		s.t.Logf("room version queried for %s", roomID)
+		return "", fmt.Errorf("unknown room")
+	}
+}
+
+func (s *sendEventTestRoomserverAPI) QueryCurrentState(ctx context.Context, req *rsapi.QueryCurrentStateRequest, res *rsapi.QueryCurrentStateResponse) error {
+	res.StateEvents = map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{}
+	for _, stateKeyTuple := range req.StateTuples {
+		for _, stateEv := range s.roomState {
+			if stateEv.Type() == stateKeyTuple.EventType && stateEv.StateKey() != nil && *stateEv.StateKey() == stateKeyTuple.StateKey {
+				res.StateEvents[stateKeyTuple] = stateEv
+			}
+		}
+	}
+	return nil
+}
+
+func (s *sendEventTestRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error {
+	if req.RoomID == s.roomIDStr {
+		res.RoomExists = true
+		res.RoomVersion = s.roomVersion
+
+		res.StateEvents = make([]*types.HeaderedEvent, len(s.roomState))
+		copy(res.StateEvents, s.roomState)
+
+		res.LatestEvents = []string{}
+		res.Depth = 1
+		return nil
+	} else {
+		s.t.Logf("room event/state queried for %s", req.RoomID)
+		return fmt.Errorf("unknown room")
+	}
+
+}
+
+func (s *sendEventTestRoomserverAPI) QuerySenderIDForUser(
+	ctx context.Context,
+	roomID spec.RoomID,
+	userID spec.UserID,
+) (*spec.SenderID, error) {
+	if roomID.String() == s.roomIDStr {
+		if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs {
+			roomKey, ok := s.senderMapping[userID.String()]
+			if ok {
+				sender := spec.SenderIDFromPseudoIDKey(roomKey)
+				return &sender, nil
+			} else {
+				return nil, nil
+			}
+		} else {
+			senderID := spec.SenderIDFromUserID(userID)
+			return &senderID, nil
+		}
+	}
+
+	return nil, fmt.Errorf("room not found")
+}
+
+func (s *sendEventTestRoomserverAPI) QueryUserIDForSender(
+	ctx context.Context,
+	roomID spec.RoomID,
+	senderID spec.SenderID,
+) (*spec.UserID, error) {
+	if roomID.String() == s.roomIDStr {
+		if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs {
+			for uID, roomKey := range s.senderMapping {
+				if string(spec.SenderIDFromPseudoIDKey(roomKey)) == string(senderID) {
+					parsedUserID, err := spec.NewUserID(uID, true)
+					if err != nil {
+						s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err)
+					}
+					return parsedUserID, nil
+				}
+			}
+		} else {
+			userID := senderID.ToUserID()
+			if userID == nil {
+				return nil, fmt.Errorf("bad sender ID")
+			}
+			return userID, nil
+		}
+	}
+
+	return nil, fmt.Errorf("room not found")
+}
+
+func (s *sendEventTestRoomserverAPI) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, sender spec.UserID) (fclient.SigningIdentity, error) {
+	if s.roomIDStr == roomID.String() {
+		if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs {
+			roomKey, ok := s.senderMapping[sender.String()]
+			if !ok {
+				s.t.Logf("SigningIdentityFor used with unknown user ID: %v", sender.String())
+				return fclient.SigningIdentity{}, fmt.Errorf("could not get signing identity for %v", sender.String())
+			}
+			return fclient.SigningIdentity{PrivateKey: roomKey}, nil
+		} else {
+			return fclient.SigningIdentity{PrivateKey: ed25519.NewKeyFromSeed(make([]byte, 32))}, nil
+		}
+	}
+
+	return fclient.SigningIdentity{}, fmt.Errorf("room not found")
+}
+
+func (s *sendEventTestRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
+	s.savedInputRoomEvents = req.InputRoomEvents
+}
+
+// Test that user ID state keys are translated correctly
+func Test_SendEvent_PseudoIDStateKeys(t *testing.T) {
+	nonpseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10
+	pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs
+
+	senderKeySeed := make([]byte, 32)
+	senderUserID := "@testuser:domain"
+	senderPrivKey := ed25519.NewKeyFromSeed(senderKeySeed)
+	senderPseudoID := string(spec.SenderIDFromPseudoIDKey(senderPrivKey))
+
+	eventType := "com.example.test"
+	roomIDStr := "!id:domain"
+
+	device := &uapi.Device{
+		UserID: senderUserID,
+	}
+
+	t.Run("user ID state key are not translated to room key in non-pseudo ID room", func(t *testing.T) {
+		eventsJSON := []string{
+			fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderUserID, senderUserID, nonpseudoIDRoomVersion),
+			fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderUserID, roomIDStr, senderUserID),
+		}
+
+		roomState, err := createEvents(eventsJSON, nonpseudoIDRoomVersion)
+		if err != nil {
+			t.Fatalf("failed to prepare state events: %s", err.Error())
+		}
+
+		rsAPI := &sendEventTestRoomserverAPI{
+			t:           t,
+			roomIDStr:   roomIDStr,
+			roomVersion: nonpseudoIDRoomVersion,
+			roomState:   roomState,
+		}
+
+		req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}")))
+		if err != nil {
+			t.Fatalf("failed to make new request: %s", err.Error())
+		}
+
+		cfg := &config.ClientAPI{}
+
+		resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil)
+
+		if resp.Code != http.StatusOK {
+			t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp)
+		}
+
+		assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1)
+
+		ev := rsAPI.savedInputRoomEvents[0]
+		stateKey := ev.Event.StateKey()
+		if stateKey == nil {
+			t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderUserID)
+		}
+		if *stateKey != senderUserID {
+			t.Fatalf("expected submitted InputRoomEvent to have user ID state key\nfound: %v\nexpected: %v", *stateKey, senderUserID)
+		}
+	})
+
+	t.Run("user ID state key are translated to room key in pseudo ID room", func(t *testing.T) {
+		eventsJSON := []string{
+			fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderPseudoID, senderPseudoID, pseudoIDRoomVersion),
+			fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderPseudoID, roomIDStr, senderPseudoID),
+		}
+
+		roomState, err := createEvents(eventsJSON, pseudoIDRoomVersion)
+		if err != nil {
+			t.Fatalf("failed to prepare state events: %s", err.Error())
+		}
+
+		rsAPI := &sendEventTestRoomserverAPI{
+			t:           t,
+			roomIDStr:   roomIDStr,
+			roomVersion: pseudoIDRoomVersion,
+			senderMapping: map[string]ed25519.PrivateKey{
+				senderUserID: senderPrivKey,
+			},
+			roomState: roomState,
+		}
+
+		req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}")))
+		if err != nil {
+			t.Fatalf("failed to make new request: %s", err.Error())
+		}
+
+		cfg := &config.ClientAPI{}
+
+		resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil)
+
+		if resp.Code != http.StatusOK {
+			t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp)
+		}
+
+		assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1)
+
+		ev := rsAPI.savedInputRoomEvents[0]
+		stateKey := ev.Event.StateKey()
+		if stateKey == nil {
+			t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderPseudoID)
+		}
+		if *stateKey != senderPseudoID {
+			t.Fatalf("expected submitted InputRoomEvent to have pseudo ID state key\nfound: %v\nexpected: %v", *stateKey, senderPseudoID)
+		}
+	})
+}
+
+func createEvents(eventsJSON []string, roomVer gomatrixserverlib.RoomVersion) ([]*types.HeaderedEvent, error) {
+	events := make([]*types.HeaderedEvent, len(eventsJSON))
+
+	roomVerImpl, err := gomatrixserverlib.GetRoomVersion(roomVer)
+	if err != nil {
+		return nil, fmt.Errorf("no roomver impl: %s", err.Error())
+	}
+
+	for i, eventJSON := range eventsJSON {
+		pdu, evErr := roomVerImpl.NewEventFromTrustedJSON([]byte(eventJSON), false)
+		if evErr != nil {
+			return nil, fmt.Errorf("failed to make event: %s", err.Error())
+		}
+		ev := types.HeaderedEvent{PDU: pdu}
+		events[i] = &ev
+	}
+
+	return events, nil
+}

clientapi/routing/sendtodevice.go

@@ -19,10 +19,10 @@ import (
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 // SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}
@@ -53,7 +53,10 @@ func SendToDevice(
 				req.Context(), device.UserID, userID, deviceID, eventType, message,
 			); err != nil {
 				util.GetLogger(req.Context()).WithError(err).Error("eduProducer.SendToDevice failed")
-				return jsonerror.InternalServerError()
+				return util.JSONResponse{
+					Code: http.StatusInternalServerError,
+					JSON: spec.InternalServerError{},
+				}
 			}
 		}
 	}

clientapi/routing/sendtyping.go

@@ -18,10 +18,10 @@ import (
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 type typingContentJSON struct {
@@ -39,12 +39,20 @@ func SendTyping(
 	if device.UserID != userID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Cannot set another user's typing state"),
+			JSON: spec.Forbidden("Cannot set another user's typing state"),
+		}
+	}
+
+	deviceUserID, err := spec.NewUserID(userID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
 		}
 	}
 
 	// Verify that the user is a member of this room
-	resErr := checkMemberInRoom(req.Context(), rsAPI, userID, roomID)
+	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}
@@ -58,7 +66,10 @@ func SendTyping(
 
 	if err := syncProducer.SendTyping(req.Context(), userID, roomID, r.Typing, r.Timeout); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("eduProducer.Send failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/server_notices.go

@@ -28,16 +28,15 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/dendrite/roomserver/version"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 // Unspecced server notice request
@@ -52,6 +51,7 @@ type sendServerNoticeRequest struct {
 	StateKey string `json:"state_key,omitempty"`
 }
 
+// nolint:gocyclo
 // SendServerNotice sends a message to a specific user. It can only be invoked by an admin.
 func SendServerNotice(
 	req *http.Request,
@@ -68,7 +68,7 @@ func SendServerNotice(
 	if device.AccountType != userapi.AccountTypeAdmin {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
+			JSON: spec.Forbidden("This API can only be used by admin users."),
 		}
 	}
 
@@ -90,38 +90,46 @@ func SendServerNotice(
 	if !r.valid() {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Invalid request"),
+			JSON: spec.BadJSON("Invalid request"),
+		}
+	}
+
+	userID, err := spec.NewUserID(r.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("invalid user ID"),
 		}
 	}
 
 	// get rooms for specified user
-	allUserRooms := []string{}
-	userRooms := api.QueryRoomsForUserResponse{}
+	allUserRooms := []spec.RoomID{}
 	// Get rooms the user is either joined, invited or has left.
 	for _, membership := range []string{"join", "invite", "leave"} {
-		if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
-			UserID:         r.UserID,
-			WantMembership: membership,
-		}, &userRooms); err != nil {
+		userRooms, queryErr := rsAPI.QueryRoomsForUser(ctx, *userID, membership)
+		if queryErr != nil {
 			return util.ErrorResponse(err)
 		}
-		allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
+		allUserRooms = append(allUserRooms, userRooms...)
 	}
 
 	// get rooms of the sender
-	senderUserID := fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName)
-	senderRooms := api.QueryRoomsForUserResponse{}
-	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
-		UserID:         senderUserID,
-		WantMembership: "join",
-	}, &senderRooms); err != nil {
+	senderUserID, err := spec.NewUserID(fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName), true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+	senderRooms, err := rsAPI.QueryRoomsForUser(ctx, *senderUserID, "join")
+	if err != nil {
 		return util.ErrorResponse(err)
 	}
 
 	// check if we have rooms in common
-	commonRooms := []string{}
+	commonRooms := []spec.RoomID{}
 	for _, userRoomID := range allUserRooms {
-		for _, senderRoomID := range senderRooms.RoomIDs {
+		for _, senderRoomID := range senderRooms {
 			if userRoomID == senderRoomID {
 				commonRooms = append(commonRooms, senderRoomID)
 			}
@@ -134,12 +142,12 @@ func SendServerNotice(
 
 	var (
 		roomID      string
-		roomVersion = version.DefaultRoomVersion()
+		roomVersion = rsAPI.DefaultRoomVersion()
 	)
 
 	// create a new room for the user
 	if len(commonRooms) == 0 {
-		powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID)
+		powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID.String())
 		powerLevelContent.Users[r.UserID] = -10 // taken from Synapse
 		pl, err := json.Marshal(powerLevelContent)
 		if err != nil {
@@ -155,7 +163,7 @@ func SendServerNotice(
 			Invite:                    []string{r.UserID},
 			Name:                      cfgNotices.RoomName,
 			Visibility:                "private",
-			Preset:                    presetPrivateChat,
+			Preset:                    spec.PresetPrivateChat,
 			CreationContent:           cc,
 			RoomVersion:               roomVersion,
 			PowerLevelContentOverride: pl,
@@ -175,7 +183,10 @@ func SendServerNotice(
 			}}
 			if err = saveTagData(req, r.UserID, roomID, userAPI, serverAlertTag); err != nil {
 				util.GetLogger(ctx).WithError(err).Error("saveTagData failed")
-				return jsonerror.InternalServerError()
+				return util.JSONResponse{
+					Code: http.StatusInternalServerError,
+					JSON: spec.InternalServerError{},
+				}
 			}
 
 		default:
@@ -184,16 +195,27 @@ func SendServerNotice(
 		}
 	} else {
 		// we've found a room in common, check the membership
-		roomID = commonRooms[0]
+		deviceUserID, err := spec.NewUserID(r.UserID, true)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
+			}
+		}
+
+		roomID = commonRooms[0].String()
 		membershipRes := api.QueryMembershipForUserResponse{}
-		err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: r.UserID, RoomID: roomID}, &membershipRes)
+		err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: *deviceUserID, RoomID: roomID}, &membershipRes)
 		if err != nil {
 			util.GetLogger(ctx).WithError(err).Error("unable to query membership for user")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		if !membershipRes.IsInRoom {
 			// re-invite the user
-			res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
+			res, err := sendInvite(ctx, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, time.Now())
 			if err != nil {
 				return res
 			}
@@ -206,7 +228,7 @@ func SendServerNotice(
 		"body":    r.Content.Body,
 		"msgtype": r.Content.MsgType,
 	}
-	e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, cfgClient, rsAPI, time.Now())
+	e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, rsAPI, time.Now())
 	if resErr != nil {
 		logrus.Errorf("failed to send message: %+v", resErr)
 		return *resErr
@@ -228,7 +250,7 @@ func SendServerNotice(
 		ctx, rsAPI,
 		api.KindNew,
 		[]*types.HeaderedEvent{
-			&types.HeaderedEvent{PDU: e},
+			{PDU: e},
 		},
 		device.UserDomain(),
 		cfgClient.Matrix.ServerName,
@@ -237,7 +259,10 @@ func SendServerNotice(
 		false,
 	); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	util.GetLogger(ctx).WithFields(logrus.Fields{
 		"event_id":     e.EventID(),
@@ -332,7 +357,7 @@ func getSenderDevice(
 	if len(deviceRes.Devices) > 0 {
 		// If there were changes to the profile, create a new membership event
 		if displayNameChanged || avatarChanged {
-			_, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, cfg, time.Now())
+			_, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, time.Now())
 			if err != nil {
 				return nil, err
 			}

clientapi/routing/state.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/syncapi/synctypes"
@@ -57,12 +56,15 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		StateToFetch: []gomatrixserverlib.StateKeyTuple{},
 	}, &stateRes); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if !stateRes.RoomExists {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("room does not exist"),
+			JSON: spec.Forbidden("room does not exist"),
 		}
 	}
 
@@ -74,7 +76,10 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 			content := map[string]string{}
 			if err := json.Unmarshal(ev.Content(), &content); err != nil {
 				util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for history visibility failed")
-				return jsonerror.InternalServerError()
+				return util.JSONResponse{
+					Code: http.StatusInternalServerError,
+					JSON: spec.InternalServerError{},
+				}
 			}
 			if visibility, ok := content["history_visibility"]; ok {
 				worldReadable = visibility == "world_readable"
@@ -94,20 +99,31 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 	if !worldReadable {
 		// The room isn't world-readable so try to work out based on the
 		// user's membership if we want the latest state or not.
-		err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{
+		userID, err := spec.NewUserID(device.UserID, true)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("UserID is invalid")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.Unknown("Device UserID is invalid"),
+			}
+		}
+		err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{
 			RoomID: roomID,
-			UserID: device.UserID,
+			UserID: *userID,
 		}, &membershipRes)
 		if err != nil {
 			util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		// If the user has never been in the room then stop at this point.
 		// We won't tell the user about a room they have never joined.
 		if !membershipRes.HasBeenInRoom {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
+				JSON: spec.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
 			}
 		}
 		// Otherwise, if the user has been in the room, whether or not we
@@ -134,7 +150,9 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		for _, ev := range stateRes.StateEvents {
 			stateEvents = append(
 				stateEvents,
-				synctypes.ToClientEvent(ev, synctypes.FormatAll),
+				synctypes.ToClientEventDefault(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+					return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+				}, ev),
 			)
 		}
 	} else {
@@ -148,12 +166,22 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		}, &stateAfterRes)
 		if err != nil {
 			util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		for _, ev := range stateAfterRes.StateEvents {
+			clientEvent, err := synctypes.ToClientEvent(ev, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+				return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+			})
+			if err != nil {
+				util.GetLogger(ctx).WithError(err).Error("Failed converting to ClientEvent")
+				continue
+			}
 			stateEvents = append(
 				stateEvents,
-				synctypes.ToClientEvent(ev, synctypes.FormatAll),
+				*clientEvent,
 			)
 		}
 	}
@@ -177,6 +205,37 @@ func OnIncomingStateTypeRequest(
 	var worldReadable bool
 	var wantLatestState bool
 
+	roomVer, err := rsAPI.QueryRoomVersionForRoom(ctx, roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
+		}
+	}
+
+	// Translate user ID state keys to room keys in pseudo ID rooms
+	if roomVer == gomatrixserverlib.RoomVersionPseudoIDs {
+		parsedRoomID, err := spec.NewRoomID(roomID)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: spec.InvalidParam("invalid room ID"),
+			}
+		}
+		newStateKey, err := synctypes.FromClientStateKey(*parsedRoomID, stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+			return rsAPI.QuerySenderIDForUser(ctx, roomID, userID)
+		})
+		if err != nil {
+			// TODO: work out better logic for failure cases (e.g. sender ID not found)
+			util.GetLogger(ctx).WithError(err).Error("synctypes.FromClientStateKey failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+		stateKey = *newStateKey
+	}
+
 	// Always fetch visibility so that we can work out whether to show
 	// the latest events or the last event from when the user was joined.
 	// Then include the requested event type and state key, assuming it
@@ -203,7 +262,10 @@ func OnIncomingStateTypeRequest(
 		StateToFetch: stateToFetch,
 	}, &stateRes); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	// Look at the room state and see if we have a history visibility event
@@ -214,7 +276,10 @@ func OnIncomingStateTypeRequest(
 			content := map[string]string{}
 			if err := json.Unmarshal(ev.Content(), &content); err != nil {
 				util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for history visibility failed")
-				return jsonerror.InternalServerError()
+				return util.JSONResponse{
+					Code: http.StatusInternalServerError,
+					JSON: spec.InternalServerError{},
+				}
 			}
 			if visibility, ok := content["history_visibility"]; ok {
 				worldReadable = visibility == "world_readable"
@@ -232,22 +297,33 @@ func OnIncomingStateTypeRequest(
 	// membershipRes will only be populated if the room is not world-readable.
 	var membershipRes api.QueryMembershipForUserResponse
 	if !worldReadable {
+		userID, err := spec.NewUserID(device.UserID, true)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("UserID is invalid")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.Unknown("Device UserID is invalid"),
+			}
+		}
 		// The room isn't world-readable so try to work out based on the
 		// user's membership if we want the latest state or not.
-		err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{
+		err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{
 			RoomID: roomID,
-			UserID: device.UserID,
+			UserID: *userID,
 		}, &membershipRes)
 		if err != nil {
 			util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		// If the user has never been in the room then stop at this point.
 		// We won't tell the user about a room they have never joined.
 		if !membershipRes.HasBeenInRoom || membershipRes.Membership == spec.Ban {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
+				JSON: spec.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
 			}
 		}
 		// Otherwise, if the user has been in the room, whether or not we
@@ -295,7 +371,10 @@ func OnIncomingStateTypeRequest(
 		}, &stateAfterRes)
 		if err != nil {
 			util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 		if len(stateAfterRes.StateEvents) > 0 {
 			event = stateAfterRes.StateEvents[0]
@@ -307,12 +386,14 @@ func OnIncomingStateTypeRequest(
 	if event == nil {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(fmt.Sprintf("Cannot find state event for %q", evType)),
+			JSON: spec.NotFound(fmt.Sprintf("Cannot find state event for %q", evType)),
 		}
 	}
 
 	stateEvent := stateEventInStateResp{
-		ClientEvent: synctypes.ToClientEvent(event, synctypes.FormatAll),
+		ClientEvent: synctypes.ToClientEventDefault(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+		}, event),
 	}
 
 	var res interface{}

clientapi/routing/state_test.go

@@ -0,0 +1,253 @@
+package routing
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"testing"
+
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+	"gotest.tools/v3/assert"
+)
+
+var ()
+
+type stateTestRoomserverAPI struct {
+	rsapi.RoomserverInternalAPI
+	t           *testing.T
+	roomState   map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent
+	roomIDStr   string
+	roomVersion gomatrixserverlib.RoomVersion
+	userIDStr   string
+	// userID -> senderID
+	senderMapping map[string]string
+}
+
+func (s stateTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) {
+	if roomID == s.roomIDStr {
+		return s.roomVersion, nil
+	} else {
+		s.t.Logf("room version queried for %s", roomID)
+		return "", fmt.Errorf("unknown room")
+	}
+}
+
+func (s stateTestRoomserverAPI) QueryLatestEventsAndState(
+	ctx context.Context,
+	req *rsapi.QueryLatestEventsAndStateRequest,
+	res *rsapi.QueryLatestEventsAndStateResponse,
+) error {
+	res.RoomExists = req.RoomID == s.roomIDStr
+	if !res.RoomExists {
+		return nil
+	}
+
+	res.StateEvents = []*types.HeaderedEvent{}
+	for _, stateKeyTuple := range req.StateToFetch {
+		val, ok := s.roomState[stateKeyTuple]
+		if ok && val != nil {
+			res.StateEvents = append(res.StateEvents, val)
+		}
+	}
+
+	return nil
+}
+
+func (s stateTestRoomserverAPI) QueryMembershipForUser(
+	ctx context.Context,
+	req *rsapi.QueryMembershipForUserRequest,
+	res *rsapi.QueryMembershipForUserResponse,
+) error {
+	if req.UserID.String() == s.userIDStr {
+		res.HasBeenInRoom = true
+		res.IsInRoom = true
+		res.RoomExists = true
+		res.Membership = spec.Join
+	}
+
+	return nil
+}
+
+func (s stateTestRoomserverAPI) QuerySenderIDForUser(
+	ctx context.Context,
+	roomID spec.RoomID,
+	userID spec.UserID,
+) (*spec.SenderID, error) {
+	sID, ok := s.senderMapping[userID.String()]
+	if ok {
+		sender := spec.SenderID(sID)
+		return &sender, nil
+	} else {
+		return nil, nil
+	}
+}
+
+func (s stateTestRoomserverAPI) QueryUserIDForSender(
+	ctx context.Context,
+	roomID spec.RoomID,
+	senderID spec.SenderID,
+) (*spec.UserID, error) {
+	for uID, sID := range s.senderMapping {
+		if sID == string(senderID) {
+			parsedUserID, err := spec.NewUserID(uID, true)
+			if err != nil {
+				s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err)
+			}
+			return parsedUserID, nil
+		}
+	}
+	return nil, nil
+}
+
+func (s stateTestRoomserverAPI) QueryStateAfterEvents(
+	ctx context.Context,
+	req *rsapi.QueryStateAfterEventsRequest,
+	res *rsapi.QueryStateAfterEventsResponse,
+) error {
+	return nil
+}
+
+func Test_OnIncomingStateTypeRequest(t *testing.T) {
+	var tempRoomServerCfg config.RoomServer
+	tempRoomServerCfg.Defaults(config.DefaultOpts{})
+	defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion
+	pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs
+	nonPseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10
+
+	userIDStr := "@testuser:domain"
+	eventType := "com.example.test"
+	stateKey := "testStateKey"
+	roomIDStr := "!id:domain"
+
+	device := &uapi.Device{
+		UserID: userIDStr,
+	}
+
+	t.Run("request simple state key", func(t *testing.T) {
+		ctx := context.Background()
+
+		rsAPI := stateTestRoomserverAPI{
+			roomVersion: defaultRoomVersion,
+			roomIDStr:   roomIDStr,
+			roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{
+				{
+					EventType: eventType,
+					StateKey:  stateKey,
+				}: mustCreateStatePDU(t, defaultRoomVersion, roomIDStr, eventType, stateKey, map[string]interface{}{
+					"foo": "bar",
+				}),
+			},
+			userIDStr: userIDStr,
+		}
+
+		jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateKey, false)
+
+		assert.DeepEqual(t, jsonResp, util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: spec.RawJSON(`{"foo":"bar"}`),
+		})
+	})
+
+	t.Run("user ID key translated to room key in pseudo ID rooms", func(t *testing.T) {
+		ctx := context.Background()
+
+		stateSenderUserID := "@sender:domain"
+		stateSenderRoomKey := "testsenderkey"
+
+		rsAPI := stateTestRoomserverAPI{
+			roomVersion: pseudoIDRoomVersion,
+			roomIDStr:   roomIDStr,
+			roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{
+				{
+					EventType: eventType,
+					StateKey:  stateSenderRoomKey,
+				}: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{
+					"foo": "bar",
+				}),
+				{
+					EventType: eventType,
+					StateKey:  stateSenderUserID,
+				}: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{
+					"not": "thisone",
+				}),
+			},
+			userIDStr: userIDStr,
+			senderMapping: map[string]string{
+				stateSenderUserID: stateSenderRoomKey,
+			},
+		}
+
+		jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false)
+
+		assert.DeepEqual(t, jsonResp, util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: spec.RawJSON(`{"foo":"bar"}`),
+		})
+	})
+
+	t.Run("user ID key not translated to room key in non-pseudo ID rooms", func(t *testing.T) {
+		ctx := context.Background()
+
+		stateSenderUserID := "@sender:domain"
+		stateSenderRoomKey := "testsenderkey"
+
+		rsAPI := stateTestRoomserverAPI{
+			roomVersion: nonPseudoIDRoomVersion,
+			roomIDStr:   roomIDStr,
+			roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{
+				{
+					EventType: eventType,
+					StateKey:  stateSenderRoomKey,
+				}: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{
+					"not": "thisone",
+				}),
+				{
+					EventType: eventType,
+					StateKey:  stateSenderUserID,
+				}: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{
+					"foo": "bar",
+				}),
+			},
+			userIDStr: userIDStr,
+			senderMapping: map[string]string{
+				stateSenderUserID: stateSenderUserID,
+			},
+		}
+
+		jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false)
+
+		assert.DeepEqual(t, jsonResp, util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: spec.RawJSON(`{"foo":"bar"}`),
+		})
+	})
+}
+
+func mustCreateStatePDU(t *testing.T, roomVer gomatrixserverlib.RoomVersion, roomID string, stateType string, stateKey string, stateContent map[string]interface{}) *types.HeaderedEvent {
+	t.Helper()
+	roomVerImpl := gomatrixserverlib.MustGetRoomVersion(roomVer)
+
+	evBytes, err := json.Marshal(map[string]interface{}{
+		"room_id":   roomID,
+		"type":      stateType,
+		"state_key": stateKey,
+		"content":   stateContent,
+	})
+	if err != nil {
+		t.Fatalf("failed to create event: %v", err)
+	}
+
+	ev, err := roomVerImpl.NewEventFromTrustedJSON(evBytes, false)
+	if err != nil {
+		t.Fatalf("failed to create event: %v", err)
+	}
+
+	return &types.HeaderedEvent{PDU: ev}
+}

clientapi/routing/thirdparty.go

@@ -21,8 +21,8 @@ import (
 	"github.com/matrix-org/util"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 // Protocols implements
@@ -33,13 +33,16 @@ func Protocols(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, dev
 	resp := &appserviceAPI.ProtocolResponse{}
 
 	if err := asAPI.Protocols(req.Context(), &appserviceAPI.ProtocolRequest{Protocol: protocol}, resp); err != nil {
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if !resp.Exists {
 		if protocol != "" {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
-				JSON: jsonerror.NotFound("The protocol is unknown."),
+				JSON: spec.NotFound("The protocol is unknown."),
 			}
 		}
 		return util.JSONResponse{
@@ -71,12 +74,15 @@ func User(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *
 		Protocol: protocol,
 		Params:   params.Encode(),
 	}, resp); err != nil {
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if !resp.Exists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("The Matrix User ID was not found"),
+			JSON: spec.NotFound("The Matrix User ID was not found"),
 		}
 	}
 	return util.JSONResponse{
@@ -97,12 +103,15 @@ func Location(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, devi
 		Protocol: protocol,
 		Params:   params.Encode(),
 	}, resp); err != nil {
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 	if !resp.Exists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("No portal rooms were found."),
+			JSON: spec.NotFound("No portal rooms were found."),
 		}
 	}
 	return util.JSONResponse{

clientapi/routing/threepid.go

@@ -19,12 +19,12 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -60,28 +60,37 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *co
 
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("threePIDAPI.QueryLocalpartForThreePID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	if len(res.Localpart) > 0 {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MatrixError{
-				ErrCode: "M_THREEPID_IN_USE",
+			JSON: spec.MatrixError{
+				ErrCode: spec.ErrorThreePIDInUse,
 				Err:     userdb.Err3PIDInUse.Error(),
 			},
 		}
 	}
 
 	resp.SID, err = threepid.CreateSession(req.Context(), body, cfg, client)
-	if err == threepid.ErrNotTrusted {
+	switch err.(type) {
+	case nil:
+	case threepid.ErrNotTrusted:
+		util.GetLogger(req.Context()).WithError(err).Error("threepid.CreateSession failed")
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.NotTrusted(body.IDServer),
+			JSON: spec.NotTrusted(body.IDServer),
 		}
-	} else if err != nil {
+	default:
 		util.GetLogger(req.Context()).WithError(err).Error("threepid.CreateSession failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -102,21 +111,27 @@ func CheckAndSave3PIDAssociation(
 
 	// Check if the association has been validated
 	verified, address, medium, err := threepid.CheckAssociation(req.Context(), body.Creds, cfg, client)
-	if err == threepid.ErrNotTrusted {
+	switch err.(type) {
+	case nil:
+	case threepid.ErrNotTrusted:
+		util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.NotTrusted(body.Creds.IDServer),
+			JSON: spec.NotTrusted(body.Creds.IDServer),
 		}
-	} else if err != nil {
+	default:
 		util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	if !verified {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MatrixError{
-				ErrCode: "M_THREEPID_AUTH_FAILED",
+			JSON: spec.MatrixError{
+				ErrCode: spec.ErrorThreePIDAuthFailed,
 				Err:     "Failed to auth 3pid",
 			},
 		}
@@ -127,7 +142,10 @@ func CheckAndSave3PIDAssociation(
 		err = threepid.PublishAssociation(req.Context(), body.Creds, device.UserID, cfg, client)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("threepid.PublishAssociation failed")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 	}
 
@@ -135,7 +153,10 @@ func CheckAndSave3PIDAssociation(
 	localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	if err = threePIDAPI.PerformSaveThreePIDAssociation(req.Context(), &api.PerformSaveThreePIDAssociationRequest{
@@ -145,7 +166,10 @@ func CheckAndSave3PIDAssociation(
 		Medium:     medium,
 	}, &struct{}{}); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("threePIDAPI.PerformSaveThreePIDAssociation failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -161,7 +185,10 @@ func GetAssociated3PIDs(
 	localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	res := &api.QueryThreePIDsForLocalpartResponse{}
@@ -171,7 +198,10 @@ func GetAssociated3PIDs(
 	}, res)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.QueryThreePIDsForLocalpart failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{
@@ -192,7 +222,10 @@ func Forget3PID(req *http.Request, threepidAPI api.ClientUserAPI) util.JSONRespo
 		Medium:   body.Medium,
 	}, &struct{}{}); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.PerformForgetThreePID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/upgrade_room.go

@@ -20,13 +20,13 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -55,26 +55,37 @@ func UpgradeRoom(
 	if _, err := version.SupportedRoomVersion(gomatrixserverlib.RoomVersion(r.NewVersion)); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion("This server does not support that room version"),
+			JSON: spec.UnsupportedRoomVersion("This server does not support that room version"),
 		}
 	}
 
-	newRoomID, err := rsAPI.PerformRoomUpgrade(req.Context(), roomID, device.UserID, gomatrixserverlib.RoomVersion(r.NewVersion))
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("device UserID is invalid")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+	newRoomID, err := rsAPI.PerformRoomUpgrade(req.Context(), roomID, *userID, gomatrixserverlib.RoomVersion(r.NewVersion))
 	switch e := err.(type) {
 	case nil:
 	case roomserverAPI.ErrNotAllowed:
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden(e.Error()),
+			JSON: spec.Forbidden(e.Error()),
 		}
 	default:
-		if errors.Is(err, eventutil.ErrRoomNoExists) {
+		if errors.Is(err, eventutil.ErrRoomNoExists{}) {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
-				JSON: jsonerror.NotFound("Room does not exist"),
+				JSON: spec.NotFound("Room does not exist"),
 			}
 		}
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	return util.JSONResponse{

clientapi/routing/voip.go

@@ -25,9 +25,9 @@ import (
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/util"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 // RequestTurnServer implements:
@@ -60,7 +60,10 @@ func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Client
 
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("mac.Write failed")
-			return jsonerror.InternalServerError()
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
 		}
 
 		resp.Password = base64.StdEncoding.EncodeToString(mac.Sum(nil))

clientapi/threepid/invites.go

@@ -64,14 +64,34 @@ type idServerStoreInviteResponse struct {
 }
 
 var (
-	// ErrMissingParameter is the error raised if a request for 3PID invite has
-	// an incomplete body
-	ErrMissingParameter = errors.New("'address', 'id_server' and 'medium' must all be supplied")
-	// ErrNotTrusted is the error raised if an identity server isn't in the list
-	// of trusted servers in the configuration file.
-	ErrNotTrusted = errors.New("untrusted server")
+	errMissingParameter = fmt.Errorf("'address', 'id_server' and 'medium' must all be supplied")
+	errNotTrusted       = fmt.Errorf("untrusted server")
 )
 
+// ErrMissingParameter is the error raised if a request for 3PID invite has
+// an incomplete body
+type ErrMissingParameter struct{}
+
+func (e ErrMissingParameter) Error() string {
+	return errMissingParameter.Error()
+}
+
+func (e ErrMissingParameter) Unwrap() error {
+	return errMissingParameter
+}
+
+// ErrNotTrusted is the error raised if an identity server isn't in the list
+// of trusted servers in the configuration file.
+type ErrNotTrusted struct{}
+
+func (e ErrNotTrusted) Error() string {
+	return errNotTrusted.Error()
+}
+
+func (e ErrNotTrusted) Unwrap() error {
+	return errNotTrusted
+}
+
 // CheckAndProcessInvite analyses the body of an incoming membership request.
 // If the fields relative to a third-party-invite are all supplied, lookups the
 // matching Matrix ID from the given identity server. If no Matrix ID is
@@ -99,7 +119,7 @@ func CheckAndProcessInvite(
 	} else if body.Address == "" || body.IDServer == "" || body.Medium == "" {
 		// If at least one of the 3PID-specific fields is supplied but not all
 		// of them, return an error
-		err = ErrMissingParameter
+		err = ErrMissingParameter{}
 		return
 	}
 
@@ -335,8 +355,22 @@ func emit3PIDInviteEvent(
 	rsAPI api.ClientRoomserverAPI,
 	evTime time.Time,
 ) error {
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return err
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return err
+	}
+	sender, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID)
+	if err != nil {
+		return err
+	} else if sender == nil {
+		return fmt.Errorf("sender ID not found for %s in %s", *userID, *validRoomID)
+	}
 	proto := &gomatrixserverlib.ProtoEvent{
-		Sender:   device.UserID,
+		SenderID: string(*sender),
 		RoomID:   roomID,
 		Type:     "m.room.third_party_invite",
 		StateKey: &res.Token,
@@ -350,7 +384,7 @@ func emit3PIDInviteEvent(
 		PublicKeys:     res.PublicKeys,
 	}
 
-	if err := proto.SetContent(content); err != nil {
+	if err = proto.SetContent(content); err != nil {
 		return err
 	}
 
@@ -360,7 +394,7 @@ func emit3PIDInviteEvent(
 	}
 
 	queryRes := api.QueryLatestEventsAndStateResponse{}
-	event, err := eventutil.QueryAndBuildEvent(ctx, proto, cfg.Matrix, identity, evTime, rsAPI, &queryRes)
+	event, err := eventutil.QueryAndBuildEvent(ctx, proto, identity, evTime, rsAPI, &queryRes)
 	if err != nil {
 		return err
 	}

clientapi/threepid/threepid.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 // EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken
@@ -133,7 +134,7 @@ func CheckAssociation(
 		return false, "", "", err
 	}
 
-	if respBody.ErrCode == "M_SESSION_NOT_VALIDATED" {
+	if respBody.ErrCode == string(spec.ErrorSessionNotValidated) {
 		return false, "", "", nil
 	} else if len(respBody.ErrCode) > 0 {
 		return false, "", "", errors.New(respBody.Error)
@@ -186,5 +187,5 @@ func isTrusted(idServer string, cfg *config.ClientAPI) error {
 			return nil
 		}
 	}
-	return ErrNotTrusted
+	return ErrNotTrusted{}
 }

cmd/dendrite-demo-pinecone/monolith/monolith.go

@@ -98,7 +98,7 @@ func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir st
 	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(storageDir, dbPrefix)))
 	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(storageDir, dbPrefix)))
 	cfg.RelayAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-relayapi.db", filepath.Join(storageDir, dbPrefix)))
-	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+	cfg.MSCs.MSCs = []string{"msc2836"}
 	cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(storageDir, dbPrefix)))
 	cfg.ClientAPI.RegistrationDisabled = false
 	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
@@ -126,7 +126,7 @@ func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) {
 }
 
 func (p *P2PMonolith) SetupDendrite(
-	processCtx *process.ProcessContext, cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers,
+	processCtx *process.ProcessContext, cfg *config.Dendrite, cm *sqlutil.Connections, routers httputil.Routers,
 	port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) {
 
 	p.port = port
@@ -143,13 +143,12 @@ func (p *P2PMonolith) SetupDendrite(
 	fsAPI := federationapi.NewInternalAPI(
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
-	rsAPI.SetFederationAPI(fsAPI, keyRing)
-
 	userProvider := users.NewPineconeUserProvider(p.Router, p.Sessions, userAPI, federation)
 	roomProvider := rooms.NewPineconeRoomProvider(p.Router, p.Sessions, fsAPI, federation)
 
@@ -222,8 +221,8 @@ func (p *P2PMonolith) closeAllResources() {
 	p.httpServerMu.Lock()
 	if p.httpServer != nil {
 		_ = p.httpServer.Shutdown(context.Background())
-		p.httpServerMu.Unlock()
 	}
+	p.httpServerMu.Unlock()
 
 	select {
 	case p.stopHandlingEvents <- true:

cmd/dendrite-demo-pinecone/relay/retriever.go

@@ -17,13 +17,13 @@ package relay
 import (
 	"context"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 )
 
 const (
@@ -54,7 +54,7 @@ func NewRelayServerRetriever(
 		federationAPI:       federationAPI,
 		relayAPI:            relayAPI,
 		relayServersQueried: make(map[spec.ServerName]bool),
-		running:             *atomic.NewBool(false),
+		running:             atomic.Bool{},
 		quit:                quit,
 	}
 }

cmd/dendrite-demo-yggdrasil/README.md

@@ -1,6 +1,6 @@
 # Yggdrasil Demo
 
-This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later.
+This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.20 or later.
 
 To run the homeserver, start at the root of the Dendrite repository and run: