Forgejo does not support build cache

https://github.com/matrix-org/sytest/pull/1383 updates some sytests in
line with MSC3967. Dendrite does not support MSC3967, so these tests
fail.

.forgejo/workflows/docker.yml

@@ -0,0 +1,59 @@
+on:
+  push:
+    tags:
+      - 'v*'
+
+env:
+  GHCR_NAMESPACE: sigb.us
+  PLATFORMS: linux/amd64
+  FORGEJO_USER: signaryk
+
+jobs:
+  monolith:
+    name: Monolith image
+    runs-on: docker
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Get release tag & build flags
+        if: github.event_name == 'release' # Only for GitHub releases
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to sigb.us container registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.sigb.us
+          username: ${{ env.FORGEJO_USER }}
+          password: ${{ secrets.FORGEJO_TOKEN }}
+
+      - name: Build main monolith image
+        id: docker_build_monolith
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ github.ref_name }}
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:devel
+
+      - name: Build release monolith image
+        if: github.event_name == 'release' # Only for GitHub releases
+        id: docker_build_monolith_release
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:stable
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ env.RELEASE_VERSION }}

.github/codecov.yaml

@@ -7,7 +7,7 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 0%
+        threshold: 0.1%
         base: auto
         flags:
           - unittests

.github/workflows/dendrite.yml

@@ -28,10 +28,10 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ false }} # disable for now
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
@@ -41,7 +41,7 @@ jobs:
         with:
           node-version: 14
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -66,11 +66,11 @@ jobs:
     name: Linting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
       - name: golangci-lint
@@ -102,14 +102,14 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
         with:
           path: |
@@ -141,12 +141,12 @@ jobs:
         goos: ["linux"]
         goarch: ["amd64", "386"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -174,12 +174,12 @@ jobs:
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -235,11 +235,11 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
       - name: Set up gotestfmt
@@ -247,7 +247,7 @@ jobs:
         with:
           # Optional: pass GITHUB_TOKEN to avoid rate limiting.
           token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -262,10 +262,11 @@ jobs:
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: dendrite
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           flags: unittests
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # run database upgrade tests
   upgrade_test:
@@ -274,12 +275,20 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade-test-
       - name: Docker version
         run: docker version
       - name: Build upgrade-tests
@@ -296,12 +305,20 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade-direct-test-
       - name: Docker version
         run: docker version
       - name: Build upgrade-tests
@@ -340,8 +357,8 @@ jobs:
         SYTEST_BRANCH: ${{ github.head_ref }}
         CGO_ENABLED: ${{ matrix.cgo && 1 }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -364,7 +381,7 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
@@ -404,8 +421,8 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v3 for dendrite
+      - name: Run actions/checkout@v4 for dendrite
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: dendrite
 

.github/workflows/docker.yml

@@ -27,22 +27,22 @@ jobs:
       security-events: write # To upload Trivy sarif files
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -98,22 +98,22 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -159,22 +159,22 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

.github/workflows/gh-pages.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Build with Jekyll

.github/workflows/helm.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/k8s.yml

@@ -17,7 +17,7 @@ jobs:
     outputs:
       changed: ${{ steps.list-changed.outputs.changed }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: azure/setup-helm@v3
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ inputs.checkoutCommit }}
@@ -66,7 +66,7 @@ jobs:
       - name: Create k3d cluster
         uses: nolar/setup-k3d-k3s@v1
         with:
-          version: v1.21
+          version: v1.28
       - name: Remove node taints
         run: |
           kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true

.github/workflows/schedules.yaml

@@ -10,8 +10,26 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  check_date: # https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where
+    runs-on: ubuntu-latest
+    name: Check latest commit
+    outputs:
+      should_run: ${{ steps.should_run.outputs.should_run }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: print latest_commit
+        run: echo ${{ github.sha }}
+
+      - id: should_run
+        continue-on-error: true
+        name: check latest commit is less than a day
+        if: ${{ github.event_name == 'schedule' }}
+        run: test -z $(git rev-list  --after="24 hours"  ${{ github.sha }}) && echo "::set-output name=should_run::false"
+
   # run Sytest in different variations
   sytest:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
     timeout-minutes: 60
     name: "Sytest (${{ matrix.label }})"
     runs-on: ubuntu-latest
@@ -38,8 +56,8 @@ jobs:
         RACE_DETECTION: 1
         COVER: 1
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -62,7 +80,7 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }})
@@ -75,31 +93,34 @@ jobs:
     timeout-minutes: 5
     name: "Sytest Coverage"
     runs-on: ubuntu-latest
-    needs: sytest # only run once Sytest is done
+    needs: [ sytest, check_date ] # only run once Sytest is done and there was a commit
-    if: ${{ always() }}
+    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 'stable'
           cache: true
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: Collect coverage
         run: |
           go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov
           grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov
           go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./final.cov
           flags: sytest
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # run Complement
   complement:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
     name: "Complement (${{ matrix.label }})"
     timeout-minutes: 60
     runs-on: ubuntu-latest
@@ -129,8 +150,8 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v3 for dendrite
+      - name: Run actions/checkout@v4 for dendrite
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: dendrite
 
@@ -174,7 +195,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:
@@ -185,7 +206,7 @@ jobs:
         working-directory: complement
 
       - name: Upload Complement logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }})
@@ -196,30 +217,32 @@ jobs:
     timeout-minutes: 5
     name: "Complement Coverage"
     runs-on: ubuntu-latest
-    needs: complement # only run once Complement is done
+    needs: [ complement, check_date ] # only run once Complements is done and there was a commit
-    if: ${{ always() }}
+    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 'stable'
           cache: true
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: Collect coverage
         run: |
           go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov
           grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
           go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./final.cov
           flags: complement
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }} # required
 
   element-web:
+    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -228,7 +251,7 @@ jobs:
           # Our test suite includes some screenshot tests with unusual diacritics, which are
           # supposed to be covered by STIXGeneral.
           tools: fonts-stix
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           repository: matrix-org/matrix-react-sdk
       - uses: actions/setup-node@v3
@@ -259,6 +282,7 @@ jobs:
           TMPDIR: ${{ runner.temp }}
 
   element-web-pinecone:
+    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -267,7 +291,7 @@ jobs:
           # Our test suite includes some screenshot tests with unusual diacritics, which are
           # supposed to be covered by STIXGeneral.
           tools: fonts-stix
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           repository: matrix-org/matrix-react-sdk
       - uses: actions/setup-node@v3

.gitignore

@@ -5,6 +5,7 @@
 
 # Allow GitHub config
 !.github
+!.forgejo
 
 # Downloads
 /.downloads
@@ -78,3 +79,6 @@ build
 
 # golang workspaces
 go.work*
+
+# helm chart
+helm/dendrite/charts/

.golangci.yml

@@ -6,7 +6,7 @@ run:
   concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  deadline: 30m
+  timeout: 5m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
@@ -18,24 +18,6 @@ run:
   #build-tags:
   #  - mytag
 
-  # which dirs to skip: they won't be analyzed;
-  # can use regexp here: generated.*, regexp is applied on full path;
-  # default value is empty list, but next dirs are always skipped independently
-  # from this option's value:
-  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs:
-    - bin
-    - docs
-
-  # which files to skip: they will be analyzed, but issues from them
-  # won't be reported. Default value is empty list, but there is
-  # no need to include all autogenerated files, we confidently recognize
-  # autogenerated files. If it's not please let us know.
-  skip-files:
-    - ".*\\.md$"
-    - ".*\\.sh$"
-    - "^cmd/syncserver-integration-tests/testdata.go$"
-
   # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
   # If invoked with -mod=readonly, the go command is disallowed from the implicit
   # automatic updating of go.mod described above. Instead, it fails when any changes
@@ -50,7 +32,8 @@ run:
 # output configuration options
 output:
   # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
-  format: colored-line-number
+  formats:
+    - format: colored-line-number
 
   # print lines of code with issue, default is true
   print-issued-lines: true
@@ -79,9 +62,8 @@ linters-settings:
     # see https://github.com/kisielk/errcheck#excluding-functions for details
     #exclude: /path/to/file.txt
   govet:
-    # report about shadowed variables
+    enable:
-    check-shadowing: true
+      - shadow
-
     # settings per analyzer
     settings:
       printf: # analyzer name, run `go tool vet help` to see all analyzers
@@ -217,6 +199,24 @@ linters:
 
 
 issues:
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  exclude-files:
+    - ".*\\.md$"
+    - ".*\\.sh$"
+    - "^cmd/syncserver-integration-tests/testdata.go$"
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  exclude-dirs:
+    - bin
+    - docs
+
   # List of regexps of issue texts to exclude, empty list by default.
   # But independently from this option we use default exclude patterns,
   # it can be disabled by `exclude-use-default: false`. To list all

CHANGES.md

@@ -1,5 +1,67 @@
 # Changelog
 
+## Dendrite 0.13.7 (2024-04-09)
+
+### Fixes
+
+- Fixed an issue where the displayname/avatar of an invited user was replaced with the inviter's details
+- Improved server startup performance by avoiding unnecessary room ACL queries
+  - This change reduces memory footprint as it caches ACL regex patterns once instead of for each room
+  - Unnecessary Relay related queries have been removed. **Note**: To use relays, you now need to explicitly enable them using the `federation_api.enable_relays` config
+- Fixed space summaries over federation
+- Improved usage of external NATS JetStream by reusing existing connections instead of opening new ones unnecessarily
+
+### Features
+
+- Modernized Appservices (contributed by [tulir](https://github.com/tulir))
+- Added event reporting with Synapse Admin endpoints for querying them
+- Updated dependencies
+
+## Dendrite 0.13.6 (2024-01-26)
+
+Upgrading to this version is **highly** recommended, as it contains several QoL improvements.
+
+### Fixes
+
+- Use `AckExplicitPolicy` for JetStream consumers, so messages don't pile up in NATS
+- A rare panic when assigning a state key NID has been fixed
+- A rare panic when checking powerlevels has been fixed
+- Notary keys requests for all keys now work correctly
+- Spec compliance:
+  - Return `M_INVALID_PARAM` when querying room aliases
+  - Handle empty `from` parameter when requesting `/messages`
+  - Add CORP headers on media endpoints
+  - Remove `aliases` from `/publicRooms` responses
+  - Allow `+` in MXIDs (Contributed by [RosstheRoss](https://github.com/RosstheRoss))
+- Fixes membership transitions from `knock` to `join` in `knock_restricted` rooms
+- Incremental syncs now batch querying events (Contributed by [recht](https://github.com/recht))
+- Move `/joined_members` back to the clientAPI/roomserver, which should make bridges happier again
+- Backfilling from other servers now only uses at max 100 events instead of potentially thousands
+
+## Dendrite 0.13.5 (2023-12-12)
+
+Upgrading to this version is **highly** recommended, as it fixes several long-standing bugs in
+our CanonicalJSON implementation.
+
+### Fixes
+
+- Convert unicode escapes to lowercase (gomatrixserverlib)
+- Fix canonical json utf-16 surrogate pair detection logic (gomatrixserverlib)
+- Handle negative zero and exponential numbers in Canonical JSON verification (gomatrixserverlib)
+- Avoid logging unnecessary messages when unable to fetch server keys if multiple fetchers are used (gomatrixserverlib)
+- Issues around the device list updater have been fixed, which should ensure that there are always
+  workers available to process incoming device list updates.
+- A panic in the `/hierarchy` endpoints used for spaces has been fixed (client-server and server-server API)
+- Fixes around the way we handle database transactions (including a potential connection leak)
+- ACLs are now updated when received as outliers
+- A race condition, which could lead to bridges instantly leaving a room after joining it, between the SyncAPI and
+  Appservices has been fixed
+
+### Features
+
+- **Appservice login is now supported!**
+- Users can now kick themselves (used by some bridges)
+
 ## Dendrite 0.13.4 (2023-10-25)
 
 Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution

Dockerfile

@@ -3,7 +3,8 @@
 #
 # base installs required dependencies and runs go mod download to cache dependencies
 #
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine3.18 AS base
 RUN apk --update --no-cache add bash build-base curl git
 
 #

appservice/api/query.go

@@ -82,9 +82,17 @@ type UserIDExistsResponse struct {
 }
 
 const (
-	ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
+	ASProtocolLegacyPath        = "/_matrix/app/unstable/thirdparty/protocol/"
-	ASUserPath     = "/_matrix/app/unstable/thirdparty/user"
+	ASUserLegacyPath            = "/_matrix/app/unstable/thirdparty/user"
-	ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
+	ASLocationLegacyPath        = "/_matrix/app/unstable/thirdparty/location"
+	ASRoomAliasExistsLegacyPath = "/rooms/"
+	ASUserExistsLegacyPath      = "/users/"
+
+	ASProtocolPath        = "/_matrix/app/v1/thirdparty/protocol/"
+	ASUserPath            = "/_matrix/app/v1/thirdparty/user"
+	ASLocationPath        = "/_matrix/app/v1/thirdparty/location"
+	ASRoomAliasExistsPath = "/_matrix/app/v1/rooms/"
+	ASUserExistsPath      = "/_matrix/app/v1/users/"
 )
 
 type ProtocolRequest struct {

appservice/appservice_test.go

@@ -14,8 +14,18 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/syncapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/appservice/api"
@@ -407,3 +417,190 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 		close(evChan)
 	})
 }
+
+// Note: If this test panics, it is because we timed out waiting for the
+// join event to come through to the appservice and we close the DB/shutdown Dendrite. This makes the
+// syncAPI unhappy, as it is unable to write to the database.
+func TestOutputAppserviceEvent(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		natsInstance := &jetstream.NATSInstance{}
+
+		evChan := make(chan struct{})
+
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		// Create required internal APIs
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+
+		// Create the router, so we can hit `/joined_members`
+		routers := httputil.NewRouters()
+
+		accessTokens := map[*test.User]userDevice{
+			bob: {},
+		}
+
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		clientapi.AddPublicRoutes(processCtx, routers, cfg, natsInstance, nil, rsAPI, nil, nil, nil, usrAPI, nil, nil, caching.DisableMetrics)
+		createAccessTokens(t, accessTokens, usrAPI, processCtx.Context(), routers)
+
+		room := test.NewRoom(t, alice)
+
+		// Invite Bob
+		room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
+			"membership": "invite",
+		}, test.WithStateKey(bob.ID))
+
+		// create a dummy AS url, handling the events
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var txn consumers.ApplicationServiceTransaction
+			err := json.NewDecoder(r.Body).Decode(&txn)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, ev := range txn.Events {
+				if ev.Type != spec.MRoomMember {
+					continue
+				}
+				if ev.StateKey != nil && *ev.StateKey == bob.ID {
+					membership := gjson.GetBytes(ev.Content, "membership").Str
+					t.Logf("Processing membership: %s", membership)
+					switch membership {
+					case spec.Invite:
+						// Accept the invite
+						joinEv := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+							"membership": "join",
+						}, test.WithStateKey(bob.ID))
+
+						if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, []*types.HeaderedEvent{joinEv}, "test", "test", "test", nil, false); err != nil {
+							t.Fatalf("failed to send events: %v", err)
+						}
+					case spec.Join: // the AS has received the join event, now hit `/joined_members` to validate that
+						rec := httptest.NewRecorder()
+						req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/rooms/"+room.ID+"/joined_members", nil)
+						req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+						routers.Client.ServeHTTP(rec, req)
+						if rec.Code != http.StatusOK {
+							t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
+						}
+
+						// Both Alice and Bob should be joined. If not, we have a race condition
+						if !gjson.GetBytes(rec.Body.Bytes(), "joined."+alice.ID).Exists() {
+							t.Errorf("Alice is not joined to the room") // in theory should not happen
+						}
+						if !gjson.GetBytes(rec.Body.Bytes(), "joined."+bob.ID).Exists() {
+							t.Errorf("Bob is not joined to the room")
+						}
+						evChan <- struct{}{}
+					default:
+						t.Fatalf("Unexpected membership: %s", membership)
+					}
+				}
+			}
+		}))
+		defer srv.Close()
+
+		as := &config.ApplicationService{
+			ID:              "someID",
+			URL:             srv.URL,
+			ASToken:         "",
+			HSToken:         "",
+			SenderLocalpart: "senderLocalPart",
+			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+				"users":   {{RegexpObject: regexp.MustCompile(bob.ID)}},
+				"aliases": {{RegexpObject: regexp.MustCompile(room.ID)}},
+			},
+		}
+		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
+
+		// Create a dummy application service
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+		// Prepare AS Streams on the old topic to validate that they get deleted
+		jsCtx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+
+		token := jetstream.Tokenise(as.ID)
+		if err := jetstream.JetStreamConsumer(
+			processCtx.Context(), jsCtx, cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
+			cfg.Global.JetStream.Durable("Appservice_"+token),
+			50, // maximum number of events to send in a single transaction
+			func(ctx context.Context, msgs []*nats.Msg) bool {
+				return true
+			},
+		); err != nil {
+			t.Fatal(err)
+		}
+
+		// Start the syncAPI to have `/joined_members` available
+		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, natsInstance, usrAPI, rsAPI, caches, caching.DisableMetrics)
+
+		// start the consumer
+		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
+
+		// At this point, the old JetStream consumers should be deleted
+		for consumer := range jsCtx.Consumers(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) {
+			if consumer.Name == cfg.Global.JetStream.Durable("Appservice_"+token)+"Pull" {
+				t.Fatalf("Consumer still exists")
+			}
+		}
+
+		// Create the room, this triggers the AS to receive an invite for Bob.
+		if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		select {
+		// Pretty generous timeout duration...
+		case <-time.After(time.Millisecond * 1000): // wait for the AS to process the events
+			t.Errorf("Timed out waiting for join event")
+		case <-evChan:
+		}
+		close(evChan)
+	})
+}
+
+type userDevice struct {
+	accessToken string
+	deviceID    string
+	password    string
+}
+
+func createAccessTokens(t *testing.T, accessTokens map[*test.User]userDevice, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) {
+	t.Helper()
+	for u := range accessTokens {
+		localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+		userRes := &uapi.PerformAccountCreationResponse{}
+		password := util.RandomString(8)
+		if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+			AccountType: u.AccountType,
+			Localpart:   localpart,
+			ServerName:  serverName,
+			Password:    password,
+		}, userRes); err != nil {
+			t.Errorf("failed to create account: %s", err)
+		}
+		req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+			"type": authtypes.LoginTypePassword,
+			"identifier": map[string]interface{}{
+				"type": "m.id.user",
+				"user": u.ID,
+			},
+			"password": password,
+		}))
+		rec := httptest.NewRecorder()
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("failed to login: %s", rec.Body.String())
+		}
+		accessTokens[u] = userDevice{
+			accessToken: gjson.GetBytes(rec.Body.Bytes(), "access_token").String(),
+			deviceID:    gjson.GetBytes(rec.Body.Bytes(), "device_id").String(),
+			password:    password,
+		}
+	}
+}

appservice/consumers/roomserver.go

@@ -71,13 +71,14 @@ func NewOutputRoomEventConsumer(
 		ctx:       process.Context(),
 		cfg:       cfg,
 		jetstream: js,
-		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
+		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputAppserviceEvent),
 		rsAPI:     rsAPI,
 	}
 }
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
+	durableNames := make([]string, 0, len(s.cfg.Derived.ApplicationServices))
 	for _, as := range s.cfg.Derived.ApplicationServices {
 		appsvc := as
 		state := &appserviceState{
@@ -95,6 +96,15 @@ func (s *OutputRoomEventConsumer) Start() error {
 		); err != nil {
 			return fmt.Errorf("failed to create %q consumer: %w", token, err)
 		}
+		durableNames = append(durableNames, s.cfg.Matrix.JetStream.Durable("Appservice_"+token))
+	}
+	// Cleanup any consumers still existing on the OutputRoomEvent stream
+	// to avoid messages not being deleted
+	for _, consumerName := range durableNames {
+		err := s.jetstream.DeleteConsumer(s.cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), consumerName+"Pull")
+		if err != nil && err != nats.ErrConsumerNotFound {
+			return err
+		}
 	}
 	return nil
 }
@@ -196,13 +206,21 @@ func (s *OutputRoomEventConsumer) sendEvents(
 	}
 
 	// Send the transaction to the appservice.
-	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
+	// https://spec.matrix.org/v1.9/application-service-api/#pushing-events
-	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.RequestUrl(), txnID, url.QueryEscape(state.HSToken))
+	path := "_matrix/app/v1/transactions"
+	if s.cfg.LegacyPaths {
+		path = "transactions"
+	}
+	address := fmt.Sprintf("%s/%s/%s", state.RequestUrl(), path, txnID)
+	if s.cfg.LegacyAuth {
+		address += "?access_token=" + url.QueryEscape(state.HSToken)
+	}
 	req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
 	if err != nil {
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", state.HSToken))
 	resp, err := state.HTTPClient.Do(req)
 	if err != nil {
 		return state.backoffAndPause(err)

appservice/query/query.go

@@ -19,10 +19,10 @@ package query
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"strings"
 	"sync"
 
 	log "github.com/sirupsen/logrus"
@@ -32,9 +32,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-const roomAliasExistsPath = "/rooms/"
-const userIDExistsPath = "/users/"
-
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
 	Cfg           *config.AppServiceAPI
@@ -55,14 +52,23 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
+			path := api.ASRoomAliasExistsPath
+			if a.Cfg.LegacyPaths {
+				path = api.ASRoomAliasExistsLegacyPath
+			}
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.RequestUrl() + roomAliasExistsPath)
+			URL, err := url.Parse(appservice.RequestUrl() + path)
 			if err != nil {
 				return err
 			}
 
 			URL.Path += request.Alias
-			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+			if a.Cfg.LegacyAuth {
+				q := URL.Query()
+				q.Set("access_token", appservice.HSToken)
+				URL.RawQuery = q.Encode()
+			}
+			apiURL := URL.String()
 
 			// Send a request to each application service. If one responds that it has
 			// created the room, immediately return.
@@ -70,6 +76,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			if err != nil {
 				return err
 			}
+			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
 			req = req.WithContext(ctx)
 
 			resp, err := appservice.HTTPClient.Do(req)
@@ -123,12 +130,21 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.RequestUrl() + userIDExistsPath)
+			path := api.ASUserExistsPath
+			if a.Cfg.LegacyPaths {
+				path = api.ASUserExistsLegacyPath
+			}
+			URL, err := url.Parse(appservice.RequestUrl() + path)
 			if err != nil {
 				return err
 			}
 			URL.Path += request.UserID
-			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+			if a.Cfg.LegacyAuth {
+				q := URL.Query()
+				q.Set("access_token", appservice.HSToken)
+				URL.RawQuery = q.Encode()
+			}
+			apiURL := URL.String()
 
 			// Send a request to each application service. If one responds that it has
 			// created the user, immediately return.
@@ -136,6 +152,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
+			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
 			resp, err := appservice.HTTPClient.Do(req.WithContext(ctx))
 			if resp != nil {
 				defer func() {
@@ -176,26 +193,23 @@ type thirdpartyResponses interface {
 	api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
 }
 
-func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
+func requestDo[T thirdpartyResponses](as *config.ApplicationService, url string, response *T) error {
-	origURL := url
+	req, err := http.NewRequest(http.MethodGet, url, nil)
-	// try v1 and unstable appservice endpoints
-	for _, version := range []string{"v1", "unstable"} {
-		var resp *http.Response
-		var body []byte
-		asURL := strings.Replace(origURL, "unstable", version, 1)
-		resp, err = client.Get(asURL)
 	if err != nil {
-			continue
+		return err
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", as.HSToken))
+	resp, err := as.HTTPClient.Do(req)
+	if err != nil {
+		return err
 	}
 	defer resp.Body.Close() // nolint: errcheck
-		body, err = io.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-			continue
+		return err
 	}
 	return json.Unmarshal(body, &response)
 }
-	return err
-}
 
 func (a *AppServiceQueryAPI) Locations(
 	ctx context.Context,
@@ -207,16 +221,22 @@ func (a *AppServiceQueryAPI) Locations(
 		return err
 	}
 
+	path := api.ASLocationPath
+	if a.Cfg.LegacyPaths {
+		path = api.ASLocationLegacyPath
+	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asLocations []api.ASLocationResponse
+		if a.Cfg.LegacyAuth {
 			params.Set("access_token", as.HSToken)
+		}
 
-		url := as.RequestUrl() + api.ASLocationPath
+		url := as.RequestUrl() + path
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
+		if err := requestDo[[]api.ASLocationResponse](&as, url+"?"+params.Encode(), &asLocations); err != nil {
 			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service")
 			continue
 		}
@@ -242,16 +262,22 @@ func (a *AppServiceQueryAPI) User(
 		return err
 	}
 
+	path := api.ASUserPath
+	if a.Cfg.LegacyPaths {
+		path = api.ASUserLegacyPath
+	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asUsers []api.ASUserResponse
+		if a.Cfg.LegacyAuth {
 			params.Set("access_token", as.HSToken)
+		}
 
-		url := as.RequestUrl() + api.ASUserPath
+		url := as.RequestUrl() + path
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
+		if err := requestDo[[]api.ASUserResponse](&as, url+"?"+params.Encode(), &asUsers); err != nil {
 			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service")
 			continue
 		}
@@ -272,6 +298,10 @@ func (a *AppServiceQueryAPI) Protocols(
 	req *api.ProtocolRequest,
 	resp *api.ProtocolResponse,
 ) error {
+	protocolPath := api.ASProtocolPath
+	if a.Cfg.LegacyPaths {
+		protocolPath = api.ASProtocolLegacyPath
+	}
 
 	// get a single protocol response
 	if req.Protocol != "" {
@@ -289,7 +319,7 @@ func (a *AppServiceQueryAPI) Protocols(
 		response := api.ASProtocolResponse{}
 		for _, as := range a.Cfg.Derived.ApplicationServices {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+req.Protocol, &proto); err != nil {
 				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
@@ -319,7 +349,7 @@ func (a *AppServiceQueryAPI) Protocols(
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		for _, p := range as.Protocols {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+p, &proto); err != nil {
 				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}

build/docker/Dockerfile.demo-pinecone

@@ -1,4 +1,5 @@
-FROM docker.io/golang:1.21-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM docker.io/golang:1.21-alpine3.18 AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/docker/Dockerfile.demo-yggdrasil

@@ -1,4 +1,5 @@
-FROM docker.io/golang:1.21-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM docker.io/golang:1.21-alpine3.18 AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

clientapi/admin_test.go

@@ -2,10 +2,12 @@ package clientapi
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -1092,3 +1094,382 @@ func TestAdminMarkAsStale(t *testing.T) {
 		}
 	})
 }
+
+func TestAdminQueryEventReports(t *testing.T) {
+	alice := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+	room2 := test.NewRoom(t, alice)
+
+	// room2 has a name and canonical alias
+	room2.CreateAndInsert(t, alice, spec.MRoomName, map[string]string{"name": "Testing"}, test.WithStateKey(""))
+	room2.CreateAndInsert(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": "#testing"}, test.WithStateKey(""))
+
+	// Join the rooms with Bob
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+	room2.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	// Create a few events to report
+	eventsToReportPerRoom := make(map[string][]string)
+	for i := 0; i < 10; i++ {
+		ev1 := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+		ev2 := room2.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+		eventsToReportPerRoom[room.ID] = append(eventsToReportPerRoom[room.ID], ev1.EventID())
+		eventsToReportPerRoom[room2.ID] = append(eventsToReportPerRoom[room2.ID], ev2.EventID())
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		/*if dbType == test.DBTypeSQLite {
+			t.Skip()
+		}*/
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room2.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		// Report all events
+		for roomID, eventIDs := range eventsToReportPerRoom {
+			for _, eventID := range eventIDs {
+				req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", roomID, eventID), strings.NewReader(string(body)))
+				req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+				routers.Client.ServeHTTP(w, req)
+
+				if w.Code != http.StatusOK {
+					t.Fatalf("expected report to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+				}
+			}
+		}
+
+		type response struct {
+			EventReports []api.QueryAdminEventReportsResponse `json:"event_reports"`
+			Total        int64                                `json:"total"`
+			NextToken    *int64                               `json:"next_token,omitempty"`
+		}
+
+		t.Run("Can query all reports", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			wantCount := 20
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+		})
+
+		t.Run("Can filter on room", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			wantCount := 10
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+		})
+
+		t.Run("Can filter on user_id", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?user_id=%s", "@doesnotexist:test"), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+
+			// The user does not exist, so we expect no results
+			wantCount := 0
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+		})
+
+		t.Run("Can set direction=f", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s&dir=f", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			wantCount := 10
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+			// we now should have the first reported event
+			wantEventID := eventsToReportPerRoom[room.ID][0]
+			gotEventID := resp.EventReports[0].EventID
+			if gotEventID != wantEventID {
+				t.Fatalf("expected eventID to be %v, got %v", wantEventID, gotEventID)
+			}
+		})
+
+		t.Run("Can limit and paginate", func(t *testing.T) {
+			var from int64 = 0
+			var limit int64 = 5
+			var wantTotal int64 = 10 // We expect there to be 10 events in total
+			var resp response
+			for from+limit <= wantTotal {
+				resp = response{}
+				t.Logf("Getting reports starting from %d", from)
+				w = httptest.NewRecorder()
+				req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s&limit=%d&from=%d", room2.ID, limit, from), strings.NewReader(string(body)))
+				req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+				routers.SynapseAdmin.ServeHTTP(w, req)
+
+				if w.Code != http.StatusOK {
+					t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+				}
+
+				if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+					t.Fatal(err)
+				}
+
+				wantCount := 5 // we are limited to 5
+				if len(resp.EventReports) != wantCount {
+					t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+				}
+				if resp.Total != int64(wantTotal) {
+					t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+				}
+
+				// We've reached the end
+				if (from + int64(len(resp.EventReports))) == wantTotal {
+					return
+				}
+
+				// The next_token should be set
+				if resp.NextToken == nil {
+					t.Fatal("expected nextToken to be set")
+				}
+				from = *resp.NextToken
+			}
+		})
+	})
+}
+
+func TestEventReportsGetDelete(t *testing.T) {
+	alice := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	// Add a name and alias
+	roomName := "Testing"
+	alias := "#testing"
+	room.CreateAndInsert(t, alice, spec.MRoomName, map[string]string{"name": roomName}, test.WithStateKey(""))
+	room.CreateAndInsert(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": alias}, test.WithStateKey(""))
+
+	// Join the rooms with Bob
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	// Create a few events to report
+
+	eventIDToReport := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		// Report the event
+		req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventIDToReport.EventID()), strings.NewReader(string(body)))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+		routers.Client.ServeHTTP(w, req)
+
+		if w.Code != http.StatusOK {
+			t.Fatalf("expected report to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+		}
+
+		t.Run("Can not query with invalid ID", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/abc", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusBadRequest {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Can query with valid ID", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			resp := api.QueryAdminEventReportResponse{}
+			if err = json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			// test a few things
+			if resp.EventID != eventIDToReport.EventID() {
+				t.Fatalf("expected eventID to be %s, got %s instead", eventIDToReport.EventID(), resp.EventID)
+			}
+			if resp.RoomName != roomName {
+				t.Fatalf("expected roomName to be %s, got %s instead", roomName, resp.RoomName)
+			}
+			if resp.CanonicalAlias != alias {
+				t.Fatalf("expected alias to be %s, got %s instead", alias, resp.CanonicalAlias)
+			}
+			if reflect.DeepEqual(resp.EventJSON, eventIDToReport.JSON()) {
+				t.Fatal("mismatching eventJSON")
+			}
+		})
+
+		t.Run("Can delete with a valid ID", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodDelete, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Can not query deleted report", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code == http.StatusOK {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+	})
+}

clientapi/auth/login.go

@@ -15,7 +15,6 @@
 package auth
 
 import (
-	"context"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -32,8 +31,13 @@ import (
 // called after authorization has completed, with the result of the authorization.
 // If the final return value is non-nil, an error occurred and the cleanup function
 // is nil.
-func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+func LoginFromJSONReader(
-	reqBytes, err := io.ReadAll(r)
+	req *http.Request,
+	useraccountAPI uapi.UserLoginAPI,
+	userAPI UserInternalAPIForLogin,
+	cfg *config.ClientAPI,
+) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	reqBytes, err := io.ReadAll(req.Body)
 	if err != nil {
 		err := &util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -57,7 +61,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 	switch header.Type {
 	case authtypes.LoginTypePassword:
 		typ = &LoginTypePassword{
-			GetAccountByPassword: useraccountAPI.QueryAccountByPassword,
+			UserAPI: useraccountAPI,
 			Config:  cfg,
 		}
 	case authtypes.LoginTypeToken:
@@ -65,6 +69,20 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 			UserAPI: userAPI,
 			Config:  cfg,
 		}
+	case authtypes.LoginTypeApplicationService:
+		token, err := ExtractAccessToken(req)
+		if err != nil {
+			err := &util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.MissingToken(err.Error()),
+			}
+			return nil, nil, err
+		}
+
+		typ = &LoginTypeApplicationService{
+			Config: cfg,
+			Token:  token,
+		}
 	default:
 		err := util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -73,7 +91,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 		return nil, nil, &err
 	}
 
-	return typ.LoginFromJSON(ctx, reqBytes)
+	return typ.LoginFromJSON(req.Context(), reqBytes)
 }
 
 // UserInternalAPIForLogin contains the aspects of UserAPI required for logging in.

clientapi/auth/login_application_service.go

@@ -0,0 +1,55 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/util"
+)
+
+// LoginTypeApplicationService describes how to authenticate as an
+// application service
+type LoginTypeApplicationService struct {
+	Config *config.ClientAPI
+	Token  string
+}
+
+// Name implements Type
+func (t *LoginTypeApplicationService) Name() string {
+	return authtypes.LoginTypeApplicationService
+}
+
+// LoginFromJSON implements Type
+func (t *LoginTypeApplicationService) LoginFromJSON(
+	ctx context.Context, reqBytes []byte,
+) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	var r Login
+	if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil {
+		return nil, nil, err
+	}
+
+	_, err := internal.ValidateApplicationServiceRequest(t.Config, r.Identifier.User, t.Token)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cleanup := func(ctx context.Context, j *util.JSONResponse) {}
+	return &r, cleanup, nil
+}

clientapi/auth/login_test.go

@@ -17,7 +17,9 @@ package auth
 import (
 	"context"
 	"net/http"
+	"net/http/httptest"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 
@@ -35,6 +37,7 @@ func TestLoginFromJSONReader(t *testing.T) {
 	tsts := []struct {
 		Name  string
 		Body  string
+		Token string
 
 		WantUsername      string
 		WantDeviceID      string
@@ -62,6 +65,30 @@ func TestLoginFromJSONReader(t *testing.T) {
 			WantDeviceID:      "adevice",
 			WantDeletedTokens: []string{"atoken"},
 		},
+		{
+			Name: "appServiceWorksUserID",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
+				"device_id": "adevice"
+			}`,
+			Token: "astoken",
+
+			WantUsername: "@alice:example.com",
+			WantDeviceID: "adevice",
+		},
+		{
+			Name: "appServiceWorksLocalpart",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "alice" },
+				"device_id": "adevice"
+			}`,
+			Token: "astoken",
+
+			WantUsername: "alice",
+			WantDeviceID: "adevice",
+		},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
@@ -72,11 +99,35 @@ func TestLoginFromJSONReader(t *testing.T) {
 						ServerName: serverName,
 					},
 				},
+				Derived: &config.Derived{
+					ApplicationServices: []config.ApplicationService{
+						{
+							ID:      "anapplicationservice",
+							ASToken: "astoken",
+							NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+								"users": {
+									{
+										Exclusive:    true,
+										Regex:        "@alice:example.com",
+										RegexpObject: regexp.MustCompile("@alice:example.com"),
+									},
+								},
+							},
+						},
+					},
+				},
 			}
-			login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
+
-			if err != nil {
+			req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body))
-				t.Fatalf("LoginFromJSONReader failed: %+v", err)
+			if tst.Token != "" {
+				req.Header.Add("Authorization", "Bearer "+tst.Token)
 			}
+
+			login, cleanup, jsonErr := LoginFromJSONReader(req, &userAPI, &userAPI, cfg)
+			if jsonErr != nil {
+				t.Fatalf("LoginFromJSONReader failed: %+v", jsonErr)
+			}
+
 			cleanup(ctx, &util.JSONResponse{Code: http.StatusOK})
 
 			if login.Username() != tst.WantUsername {
@@ -106,6 +157,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 	tsts := []struct {
 		Name  string
 		Body  string
+		Token string
 
 		WantErrCode spec.MatrixErrorCode
 	}{
@@ -142,6 +194,45 @@ func TestBadLoginFromJSONReader(t *testing.T) {
             }`,
 			WantErrCode: spec.ErrorInvalidParam,
 		},
+		{
+			Name: "noASToken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_MISSING_TOKEN",
+		},
+		{
+			Name:  "badASToken",
+			Token: "badastoken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_UNKNOWN_TOKEN",
+		},
+		{
+			Name:  "badASNamespace",
+			Token: "astoken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@bob:example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_EXCLUSIVE",
+		},
+		{
+			Name:  "badASUserID",
+			Token: "astoken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:wrong.example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_INVALID_USERNAME",
+		},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
@@ -152,8 +243,30 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 						ServerName: serverName,
 					},
 				},
+				Derived: &config.Derived{
+					ApplicationServices: []config.ApplicationService{
+						{
+							ID:      "anapplicationservice",
+							ASToken: "astoken",
+							NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+								"users": {
+									{
+										Exclusive:    true,
+										Regex:        "@alice:example.com",
+										RegexpObject: regexp.MustCompile("@alice:example.com"),
+									},
+								},
+							},
+						},
+					},
+				},
 			}
-			_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
+			req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body))
+			if tst.Token != "" {
+				req.Header.Add("Authorization", "Bearer "+tst.Token)
+			}
+
+			_, cleanup, errRes := LoginFromJSONReader(req, &userAPI, &userAPI, cfg)
 			if errRes == nil {
 				cleanup(ctx, nil)
 				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
@@ -179,6 +292,14 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *
 	return nil
 }
 
+func (ua *fakeUserInternalAPI) QueryAccountByLocalpart(ctx context.Context, req *uapi.QueryAccountByLocalpartRequest, res *uapi.QueryAccountByLocalpartResponse) error {
+	return nil
+}
+
+func (ua *fakeUserInternalAPI) PerformAccountCreation(ctx context.Context, req *uapi.PerformAccountCreationRequest, res *uapi.PerformAccountCreationResponse) error {
+	return nil
+}
+
 func (ua *fakeUserInternalAPI) PerformLoginTokenDeletion(ctx context.Context, req *uapi.PerformLoginTokenDeletionRequest, res *uapi.PerformLoginTokenDeletionResponse) error {
 	ua.DeletedTokens = append(ua.DeletedTokens, req.Token)
 	return nil

clientapi/auth/password.go

@@ -16,6 +16,9 @@ package auth
 
 import (
 	"context"
+	"database/sql"
+	"github.com/go-ldap/ldap/v3"
+	"github.com/google/uuid"
 	"net/http"
 	"strings"
 
@@ -28,8 +31,6 @@ import (
 	"github.com/matrix-org/util"
 )
 
-type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error
-
 type PasswordRequest struct {
 	Login
 	Password string `json:"password"`
@@ -37,8 +38,8 @@ type PasswordRequest struct {
 
 // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based
 type LoginTypePassword struct {
-	GetAccountByPassword GetAccountByPassword
 	Config  *config.ClientAPI
+	UserAPI api.UserLoginAPI
 }
 
 func (t *LoginTypePassword) Name() string {
@@ -59,22 +60,21 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
 	return login, func(context.Context, *util.JSONResponse) {}, nil
 }
 
-func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
+func (t *LoginTypePassword) Login(ctx context.Context, request *PasswordRequest) (*Login, *util.JSONResponse) {
-	r := req.(*PasswordRequest)
+	fullUsername := request.Username()
-	username := r.Username()
+	if fullUsername == "" {
-	if username == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
 			JSON: spec.BadJSON("A username must be supplied."),
 		}
 	}
-	if len(r.Password) == 0 {
+	if len(request.Password) == 0 {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
 			JSON: spec.BadJSON("A password must be supplied."),
 		}
 	}
-	localpart, domain, err := userutil.ParseUsernameParam(username, t.Config.Matrix)
+	username, domain, err := userutil.ParseUsernameParam(fullUsername, t.Config.Matrix)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
@@ -87,12 +87,38 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			JSON: spec.InvalidUsername("The server name is not known."),
 		}
 	}
-	// Squash username to all lowercase letters
+
+	var account *api.Account
+	if t.Config.Ldap.Enabled {
+		isAdmin, err := t.authenticateLdap(username, request.Password)
+		if err != nil {
+			return nil, err
+		}
+		acc, err := t.getOrCreateAccount(ctx, username, domain, isAdmin)
+		if err != nil {
+			return nil, err
+		}
+		account = acc
+	} else {
+		acc, err := t.authenticateDb(ctx, username, domain, request.Password)
+		if err != nil {
+			return nil, err
+		}
+		account = acc
+	}
+
+	// Set the user, so login.Username() can do the right thing
+	request.Identifier.User = account.UserID
+	request.User = account.UserID
+	return &request.Login, nil
+}
+
+func (t *LoginTypePassword) authenticateDb(ctx context.Context, username string, domain spec.ServerName, password string) (*api.Account, *util.JSONResponse) {
 	res := &api.QueryAccountByPasswordResponse{}
-	err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+	err := t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-		Localpart:         strings.ToLower(localpart),
+		Localpart:         strings.ToLower(username),
 		ServerName:        domain,
-		PlaintextPassword: r.Password,
+		PlaintextPassword: password,
 	}, res)
 	if err != nil {
 		return nil, &util.JSONResponse{
@@ -101,13 +127,11 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		}
 	}
 
-	// If we couldn't find the user by the lower cased localpart, try the provided
-	// localpart as is.
 	if !res.Exists {
-		err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+		err = t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-			Localpart:         localpart,
+			Localpart:         username,
 			ServerName:        domain,
-			PlaintextPassword: r.Password,
+			PlaintextPassword: password,
 		}, res)
 		if err != nil {
 			return nil, &util.JSONResponse{
@@ -115,8 +139,6 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 				JSON: spec.Unknown("Unable to fetch account by password."),
 			}
 		}
-		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
-		// but that would leak the existence of the user.
 		if !res.Exists {
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
@@ -124,8 +146,141 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			}
 		}
 	}
-	// Set the user, so login.Username() can do the right thing
+	return res.Account, nil
-	r.Identifier.User = res.Account.UserID
+}
-	r.User = res.Account.UserID
+func (t *LoginTypePassword) authenticateLdap(username, password string) (bool, *util.JSONResponse) {
-	return &r.Login, nil
+	var conn *ldap.Conn
+	conn, err := ldap.DialURL(t.Config.Ldap.Uri)
+	if err != nil {
+		return false, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("unable to connect to ldap: " + err.Error()),
+		}
+	}
+	defer conn.Close()
+
+	if t.Config.Ldap.AdminBindEnabled {
+		err = conn.Bind(t.Config.Ldap.AdminBindDn, t.Config.Ldap.AdminBindPassword)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("unable to bind to ldap: " + err.Error()),
+			}
+		}
+		filter := strings.ReplaceAll(t.Config.Ldap.SearchFilter, "{username}", username)
+		searchRequest := ldap.NewSearchRequest(
+			t.Config.Ldap.BaseDn, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases,
+			0, 0, false, filter, []string{t.Config.Ldap.SearchAttribute}, nil,
+		)
+		result, err := conn.Search(searchRequest)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("unable to bind to search ldap: " + err.Error()),
+			}
+		}
+		if len(result.Entries) > 1 {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.BadJSON("'user' must be duplicated."),
+			}
+		}
+		if len(result.Entries) < 1 {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.BadJSON("'user' not found."),
+			}
+		}
+
+		userDN := result.Entries[0].DN
+		err = conn.Bind(userDN, password)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.InvalidUsername(err.Error()),
+			}
+		}
+	} else {
+		bindDn := strings.ReplaceAll(t.Config.Ldap.UserBindDn, "{username}", username)
+		err = conn.Bind(bindDn, password)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.InvalidUsername(err.Error()),
+			}
+		}
+	}
+
+	isAdmin, err := t.isLdapAdmin(conn, username)
+	if err != nil {
+		return false, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: spec.InvalidUsername(err.Error()),
+		}
+	}
+	return isAdmin, nil
+}
+
+func (t *LoginTypePassword) isLdapAdmin(conn *ldap.Conn, username string) (bool, error) {
+	searchRequest := ldap.NewSearchRequest(
+		t.Config.Ldap.AdminGroupDn,
+		ldap.ScopeWholeSubtree, ldap.DerefAlways, 0, 0, false,
+		strings.ReplaceAll(t.Config.Ldap.AdminGroupFilter, "{username}", username),
+		[]string{t.Config.Ldap.AdminGroupAttribute},
+		nil)
+
+	sr, err := conn.Search(searchRequest)
+	if err != nil {
+		return false, err
+	}
+
+	if len(sr.Entries) < 1 {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, username string, domain spec.ServerName, admin bool) (*api.Account, *util.JSONResponse) {
+	var existing api.QueryAccountByLocalpartResponse
+	err := t.UserAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{
+		Localpart:  username,
+		ServerName: domain,
+	}, &existing)
+
+	if err == nil {
+		return existing.Account, nil
+	}
+	if err != sql.ErrNoRows {
+		return nil, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: spec.InvalidUsername(err.Error()),
+		}
+	}
+
+	accountType := api.AccountTypeUser
+	if admin {
+		accountType = api.AccountTypeAdmin
+	}
+	var created api.PerformAccountCreationResponse
+	err = t.UserAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{
+		AppServiceID: "ldap",
+		Localpart:    username,
+		Password:     uuid.New().String(),
+		AccountType:  accountType,
+		OnConflict:   api.ConflictAbort,
+	}, &created)
+
+	if err != nil {
+		if _, ok := err.(*api.ErrorConflict); ok {
+			return nil, &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.UserInUse("Desired user ID is already taken."),
+			}
+		}
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("failed to create account: " + err.Error()),
+		}
+	}
+	return created.Account, nil
 }

clientapi/auth/user_interactive.go

@@ -55,7 +55,7 @@ type LoginCleanupFunc func(context.Context, *util.JSONResponse)
 // https://matrix.org/docs/spec/client_server/r0.6.1#identifier-types
 type LoginIdentifier struct {
 	Type string `json:"type"`
-	// when type = m.id.user
+	// when type = m.id.user or m.id.application_service
 	User string `json:"user"`
 	// when type = m.id.thirdparty
 	Medium  string `json:"medium"`
@@ -113,7 +113,7 @@ type UserInteractive struct {
 
 func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
-		GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
+		UserAPI: userAccountAPI,
 		Config:  cfg,
 	}
 	return &UserInteractive{

clientapi/auth/user_interactive_test.go

@@ -45,6 +45,14 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a
 	return nil
 }
 
+func (d *fakeAccountDatabase) QueryAccountByLocalpart(ctx context.Context, req *api.QueryAccountByLocalpartRequest, res *api.QueryAccountByLocalpartResponse) error {
+	return nil
+}
+
+func (d *fakeAccountDatabase) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error {
+	return nil
+}
+
 func setup() *UserInteractive {
 	cfg := &config.ClientAPI{
 		Matrix: &config.Global{

clientapi/clientapi_test.go

@@ -958,7 +958,8 @@ func TestCapabilities(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 		// Needed to create accounts
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
@@ -1005,7 +1006,8 @@ func TestTurnserver(t *testing.T) {
 	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 	// Needed to create accounts
-	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 	rsAPI.SetFederationAPI(nil, nil)
 	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	//rsAPI.SetUserAPI(userAPI)
@@ -1103,7 +1105,8 @@ func Test3PID(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 		// Needed to create accounts
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
@@ -2151,3 +2154,284 @@ func TestKeyBackup(t *testing.T) {
 		}
 	})
 }
+
+func TestGetMembership(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	testCases := []struct {
+		name             string
+		roomID           string
+		user             *test.User
+		additionalEvents func(t *testing.T, room *test.Room)
+		request          func(t *testing.T, room *test.Room, accessToken string) *http.Request
+		wantOK           bool
+		wantMemberCount  int
+	}{
+
+		{
+			name: "/joined_members - Bob never joined",
+			user: bob,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Alice joined",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK:          true,
+			wantMemberCount: 1,
+		},
+		{
+			name: "/joined_members - Alice leaves, shouldn't be able to see members ",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
+					"membership": "leave",
+				}, test.WithStateKey(alice.ID))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Bob joins, Alice sees two members",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+					"membership": "join",
+				}, test.WithStateKey(bob.ID))
+			},
+			wantOK:          true,
+			wantMemberCount: 2,
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				room := test.NewRoom(t, alice)
+				t.Cleanup(func() {
+					t.Logf("running cleanup for %s", tc.name)
+				})
+				// inject additional events
+				if tc.additionalEvents != nil {
+					tc.additionalEvents(t, room)
+				}
+				if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+					t.Fatalf("failed to send events: %v", err)
+				}
+
+				w := httptest.NewRecorder()
+				routers.Client.ServeHTTP(w, tc.request(t, room, accessTokens[tc.user].accessToken))
+				if w.Code != 200 && tc.wantOK {
+					t.Logf("%s", w.Body.String())
+					t.Fatalf("got HTTP %d want %d", w.Code, 200)
+				}
+				t.Logf("[%s] Resp: %s", tc.name, w.Body.String())
+
+				// check we got the expected events
+				if tc.wantOK {
+					memberCount := len(gjson.GetBytes(w.Body.Bytes(), "joined").Map())
+					if memberCount != tc.wantMemberCount {
+						t.Fatalf("expected %d members, got %d", tc.wantMemberCount, memberCount)
+					}
+				}
+			})
+		}
+	})
+}
+
+func TestCreateRoomInvite(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"invite": []string{bob.ID},
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/createRoom", strings.NewReader(string(body)))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+		routers.Client.ServeHTTP(w, req)
+
+		if w.Code != http.StatusOK {
+			t.Fatalf("expected room creation to be successful, got HTTP %d instead: %s", w.Code, w.Body.String())
+		}
+
+		roomID := gjson.GetBytes(w.Body.Bytes(), "room_id").Str
+		validRoomID, _ := spec.NewRoomID(roomID)
+		// Now ask the roomserver about the membership event of Bob
+		ev, err := rsAPI.CurrentStateEvent(context.Background(), *validRoomID, spec.MRoomMember, bob.ID)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if ev == nil {
+			t.Fatal("Membership event for Bob does not exist")
+		}
+
+		// Validate that there is NO displayname in content
+		if gjson.GetBytes(ev.Content(), "displayname").Exists() {
+			t.Fatal("Found displayname in invite")
+		}
+	})
+}
+
+func TestReportEvent(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	charlie := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(charlie.ID))
+	eventToReport := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice:   {},
+			bob:     {},
+			charlie: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		t.Run("Bob is not joined and should not be able to report the event", func(t *testing.T) {
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusNotFound {
+				t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Charlie is joined but the event does not exist", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/$doesNotExist", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusNotFound {
+				t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Charlie is joined and allowed to report the event", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected report to be successful, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+	})
+}

clientapi/routing/admin.go

@@ -495,3 +495,93 @@ func AdminDownloadState(req *http.Request, device *api.Device, rsAPI roomserverA
 		JSON: struct{}{},
 	}
 }
+
+// GetEventReports returns reported events for a given user/room.
+func GetEventReports(
+	req *http.Request,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
+	from, limit uint64,
+	backwards bool,
+	userID, roomID string,
+) util.JSONResponse {
+
+	eventReports, count, err := rsAPI.QueryAdminEventReports(req.Context(), from, limit, backwards, userID, roomID)
+	if err != nil {
+		logrus.WithError(err).Error("failed to query event reports")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	resp := map[string]any{
+		"event_reports": eventReports,
+		"total":         count,
+	}
+
+	// Add a next_token if there are still reports
+	if int64(from+limit) < count {
+		resp["next_token"] = int(from) + len(eventReports)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: resp,
+	}
+}
+
+func GetEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
+	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			// Given this is an admin endpoint, let them know what didn't work.
+			JSON: spec.InvalidParam(err.Error()),
+		}
+	}
+
+	report, err := rsAPI.QueryAdminEventReport(req.Context(), parsedReportID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown(err.Error()),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: report,
+	}
+}
+
+func DeleteEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
+	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			// Given this is an admin endpoint, let them know what didn't work.
+			JSON: spec.InvalidParam(err.Error()),
+		}
+	}
+
+	err = rsAPI.PerformAdminDeleteEventReport(req.Context(), parsedReportID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown(err.Error()),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func parseUint64OrDefault(input string, defaultValue uint64) uint64 {
+	v, err := strconv.ParseUint(input, 10, 64)
+	if err != nil {
+		return defaultValue
+	}
+	return v
+}

clientapi/routing/directory.go

@@ -55,7 +55,7 @@ func DirectoryRoom(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"),
+			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 
@@ -134,7 +134,7 @@ func SetLocalAlias(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"),
+			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 

clientapi/routing/key_crosssigning.go

@@ -32,7 +32,7 @@ type crossSigningRequest struct {
 }
 
 func UploadCrossSigningDeviceKeys(
-	req *http.Request, userInteractiveAuth *auth.UserInteractive,
+	req *http.Request,
 	keyserverAPI api.ClientKeyAPI, device *api.Device,
 	accountAPI api.ClientUserAPI, cfg *config.ClientAPI,
 ) util.JSONResponse {
@@ -62,7 +62,7 @@ func UploadCrossSigningDeviceKeys(
 		}
 	}
 	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: accountAPI.QueryAccountByPassword,
+		UserAPI: accountAPI,
 		Config:  cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {

clientapi/routing/keys.go

@@ -93,7 +93,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device)
 
 type queryKeysRequest struct {
 	Timeout    int                 `json:"timeout"`
-	Token      string              `json:"token"`
 	DeviceKeys map[string][]string `json:"device_keys"`
 }
 
@@ -119,7 +118,6 @@ func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) u
 		UserID:        device.UserID,
 		UserToDevices: r.DeviceKeys,
 		Timeout:       r.GetTimeout(),
-		// TODO: Token?
 	}, &queryRes)
 	return util.JSONResponse{
 		Code: 200,

clientapi/routing/login.go

@@ -19,6 +19,7 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -40,28 +41,25 @@ type flow struct {
 	Type string `json:"type"`
 }
 
-func passwordLogin() flows {
-	f := flows{}
-	s := flow{
-		Type: "m.login.password",
-	}
-	f.Flows = append(f.Flows, s)
-	return f
-}
-
 // Login implements GET and POST /login
 func Login(
 	req *http.Request, userAPI userapi.ClientUserAPI,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	if req.Method == http.MethodGet {
-		// TODO: support other forms of login other than password, depending on config options
+		loginFlows := []flow{{Type: authtypes.LoginTypePassword}}
+		if len(cfg.Derived.ApplicationServices) > 0 {
+			loginFlows = append(loginFlows, flow{Type: authtypes.LoginTypeApplicationService})
+		}
+		// TODO: support other forms of login, depending on config options
 		return util.JSONResponse{
 			Code: http.StatusOK,
-			JSON: passwordLogin(),
+			JSON: flows{
+				Flows: loginFlows,
+			},
 		}
 	} else if req.Method == http.MethodPost {
-		login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg)
+		login, cleanup, authErr := auth.LoginFromJSONReader(req, userAPI, userAPI, cfg)
 		if authErr != nil {
 			return *authErr
 		}

clientapi/routing/login_test.go

@@ -114,6 +114,44 @@ func TestLogin(t *testing.T) {
 
 		ctx := context.Background()
 
+		// Inject a dummy application service, so we have a "m.login.application_service"
+		// in the login flows
+		as := &config.ApplicationService{}
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+		t.Run("Supported log-in flows are returned", func(t *testing.T) {
+			req := test.NewRequest(t, http.MethodGet, "/_matrix/client/v3/login")
+			rec := httptest.NewRecorder()
+			routers.Client.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("failed to get log-in flows: %s", rec.Body.String())
+			}
+
+			t.Logf("response: %s", rec.Body.String())
+			resp := flows{}
+			if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+
+			appServiceFound := false
+			passwordFound := false
+			for _, flow := range resp.Flows {
+				if flow.Type == "m.login.password" {
+					passwordFound = true
+				} else if flow.Type == "m.login.application_service" {
+					appServiceFound = true
+				} else {
+					t.Fatalf("got unknown login flow: %s", flow.Type)
+				}
+			}
+			if !appServiceFound {
+				t.Fatal("m.login.application_service missing from login flows")
+			}
+			if !passwordFound {
+				t.Fatal("m.login.password missing from login flows")
+			}
+		})
+
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
 				req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{

clientapi/routing/membership.go

@@ -181,18 +181,6 @@ func SendKick(
 		return *errRes
 	}
 
-	pl, errRes := getPowerlevels(req, rsAPI, roomID)
-	if errRes != nil {
-		return *errRes
-	}
-	allowedToKick := pl.UserLevel(*senderID) >= pl.Kick
-	if !allowedToKick {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("You don't have permission to kick this user, power level too low."),
-		}
-	}
-
 	bodyUserID, err := spec.NewUserID(body.UserID, true)
 	if err != nil {
 		return util.JSONResponse{
@@ -200,6 +188,19 @@ func SendKick(
 			JSON: spec.BadJSON("body userID is invalid"),
 		}
 	}
+
+	pl, errRes := getPowerlevels(req, rsAPI, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+	allowedToKick := pl.UserLevel(*senderID) >= pl.Kick || bodyUserID.String() == deviceUserID.String()
+	if !allowedToKick {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, power level too low."),
+		}
+	}
+
 	var queryRes roomserverAPI.QueryMembershipForUserResponse
 	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
@@ -323,19 +324,18 @@ func SendInvite(
 	}
 
 	// We already received the return value, so no need to check for an error here.
-	response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime)
+	response, _ := sendInvite(req.Context(), device, roomID, body.UserID, body.Reason, cfg, rsAPI, evTime)
 	return response
 }
 
 // sendInvite sends an invitation to a user. Returns a JSONResponse and an error
 func sendInvite(
 	ctx context.Context,
-	profileAPI userapi.ClientUserAPI,
 	device *userapi.Device,
 	roomID, userID, reason string,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
+	evTime time.Time,
 ) (util.JSONResponse, error) {
 	validRoomID, err := spec.NewRoomID(roomID)
 	if err != nil {
@@ -358,13 +358,7 @@ func sendInvite(
 			JSON: spec.InvalidParam("UserID is invalid"),
 		}, err
 	}
-	profile, err := loadProfile(ctx, userID, cfg, profileAPI, asAPI)
+
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}, err
-	}
 	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
 	if err != nil {
 		return util.JSONResponse{
@@ -377,8 +371,6 @@ func sendInvite(
 			RoomID:     *validRoomID,
 			Inviter:    *inviter,
 			Invitee:    *invitee,
-			DisplayName: profile.DisplayName,
-			AvatarURL:   profile.AvatarURL,
 			Reason:     reason,
 			IsDirect:   false,
 			KeyID:      identity.KeyID,

clientapi/routing/memberships.go

@@ -0,0 +1,139 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+)
+
+// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
+type getJoinedMembersResponse struct {
+	Joined map[string]joinedMember `json:"joined"`
+}
+
+type joinedMember struct {
+	DisplayName string `json:"display_name"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// The database stores 'displayname' without an underscore.
+// Deserialize into this and then change to the actual API response
+type databaseJoinedMember struct {
+	DisplayName string `json:"displayname"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// GetJoinedMembers implements
+//
+//	GET /rooms/{roomId}/joined_members
+func GetJoinedMembers(
+	req *http.Request, device *userapi.Device, roomID string,
+	rsAPI api.ClientRoomserverAPI,
+) util.JSONResponse {
+	// Validate the userID
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("Device UserID is invalid"),
+		}
+	}
+
+	// Validate the roomID
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("RoomID is invalid"),
+		}
+	}
+
+	// Get the current memberships for the requesting user to determine
+	// if they are allowed to query this endpoint.
+	queryReq := api.QueryMembershipForUserRequest{
+		RoomID: validRoomID.String(),
+		UserID: *userID,
+	}
+
+	var queryRes api.QueryMembershipForUserResponse
+	if queryErr := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); queryErr != nil {
+		util.GetLogger(req.Context()).WithError(queryErr).Error("rsAPI.QueryMembershipsForRoom failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	if !queryRes.HasBeenInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	if !queryRes.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	// Get the current membership events
+	var membershipsForRoomResp api.QueryMembershipsForRoomResponse
+	if err = rsAPI.QueryMembershipsForRoom(req.Context(), &api.QueryMembershipsForRoomRequest{
+		JoinedOnly: true,
+		RoomID:     validRoomID.String(),
+	}, &membershipsForRoomResp); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryEventsByID failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	var res getJoinedMembersResponse
+	res.Joined = make(map[string]joinedMember)
+	for _, ev := range membershipsForRoomResp.JoinEvents {
+		var content databaseJoinedMember
+		if err := json.Unmarshal(ev.Content, &content); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(ev.Sender))
+		if err != nil || userID == nil {
+			util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		res.Joined[userID.String()] = joinedMember(content)
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}

clientapi/routing/password.go

@@ -73,7 +73,7 @@ func Password(
 
 	// Check if the existing password is correct.
 	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: userAPI.QueryAccountByPassword,
+		UserAPI: userAPI,
 		Config:  cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {

clientapi/routing/receipt.go

@@ -23,13 +23,12 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 
-	"github.com/matrix-org/dendrite/userapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )
 
-func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
+func SetReceipt(req *http.Request, userAPI userapi.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
 	timestamp := spec.AsTimestamp(time.Now())
 	logrus.WithFields(logrus.Fields{
 		"roomID":      roomID,
@@ -54,13 +53,13 @@ func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *prod
 			}
 		}
 
-		dataReq := api.InputAccountDataRequest{
+		dataReq := userapi.InputAccountDataRequest{
 			UserID:      device.UserID,
 			DataType:    "m.fully_read",
 			RoomID:      roomID,
 			AccountData: data,
 		}
-		dataRes := api.InputAccountDataResponse{}
+		dataRes := userapi.InputAccountDataResponse{}
 		if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
 			return util.ErrorResponse(err)

clientapi/routing/register.go

@@ -630,6 +630,7 @@ func handleGuestRegistration(
 		AccessToken:       token,
 		IPAddr:            req.RemoteAddr,
 		UserAgent:         req.UserAgent(),
+		FromRegistration:  true,
 	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
@@ -647,6 +648,16 @@ func handleGuestRegistration(
 	}
 }
 
+// localpartMatchesExclusiveNamespaces will check if a given username matches any
+// application service's exclusive users namespace
+func localpartMatchesExclusiveNamespaces(
+	cfg *config.ClientAPI,
+	localpart string,
+) bool {
+	userID := userutil.MakeUserID(localpart, cfg.Matrix.ServerName)
+	return cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(userID)
+}
+
 // handleRegistrationFlow will direct and complete registration flow stages
 // that the client has requested.
 // nolint: gocyclo
@@ -695,7 +706,7 @@ func handleRegistrationFlow(
 	// If an access token is provided, ignore this check this is an appservice
 	// request and we will validate in validateApplicationService
 	if len(cfg.Derived.ApplicationServices) != 0 &&
-		UsernameMatchesExclusiveNamespaces(cfg, r.Username) {
+		localpartMatchesExclusiveNamespaces(cfg, r.Username) {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: spec.ASExclusive("This username is reserved by an application service."),
@@ -772,7 +783,7 @@ func handleApplicationServiceRegistration(
 
 	// Check application service register user request is valid.
 	// The application service's ID is returned if so.
-	appserviceID, err := validateApplicationService(
+	appserviceID, err := internal.ValidateApplicationServiceRequest(
 		cfg, r.Username, accessToken,
 	)
 	if err != nil {
@@ -909,6 +920,7 @@ func completeRegistration(
 		DeviceID:          deviceID,
 		IPAddr:            ipAddr,
 		UserAgent:         userAgent,
+		FromRegistration:  true,
 	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{

clientapi/routing/register_test.go

@@ -298,13 +298,16 @@ func Test_register(t *testing.T) {
 		guestsDisabled       bool
 		enableRecaptcha      bool
 		captchaBody          string
-		wantResponse         util.JSONResponse
+		// in case of an error, the expected response
+		wantErrorResponse util.JSONResponse
+		// in case of success, the expected username assigned
+		wantUsername string
 	}{
 		{
 			name:           "disallow guests",
 			kind:           "guest",
 			guestsDisabled: true,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: spec.Forbidden(`Guest registration is disabled on "test"`),
 			},
@@ -312,11 +315,12 @@ func Test_register(t *testing.T) {
 		{
 			name:         "allow guests",
 			kind:         "guest",
+			wantUsername: "1",
 		},
 		{
 			name:      "unknown login type",
 			loginType: "im.not.known",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusNotImplemented,
 				JSON: spec.Unknown("unknown/unimplemented auth type"),
 			},
@@ -324,7 +328,7 @@ func Test_register(t *testing.T) {
 		{
 			name:                 "disabled registration",
 			registrationDisabled: true,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: spec.Forbidden(`Registration is disabled on "test"`),
 			},
@@ -334,15 +338,23 @@ func Test_register(t *testing.T) {
 			username:     "",
 			password:     "someRandomPassword",
 			forceEmpty:   true,
+			wantUsername: "2",
 		},
 		{
 			name:     "successful registration",
 			username: "success",
 		},
+		{
+			name:         "successful registration, sequential numeric ID",
+			username:     "",
+			password:     "someRandomPassword",
+			forceEmpty:   true,
+			wantUsername: "3",
+		},
 		{
 			name:     "failing registration - user already exists",
 			username: "success",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.UserInUse("Desired user ID is already taken."),
 			},
@@ -354,12 +366,12 @@ func Test_register(t *testing.T) {
 		{
 			name:              "invalid username",
 			username:          "#totalyNotValid",
-			wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
+			wantErrorResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
 		},
 		{
 			name:     "numeric username is forbidden",
 			username: "1337",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.InvalidUsername("Numeric user IDs are reserved"),
 			},
@@ -367,7 +379,7 @@ func Test_register(t *testing.T) {
 		{
 			name:      "disabled recaptcha login",
 			loginType: authtypes.LoginTypeRecaptcha,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: spec.Unknown(ErrCaptchaDisabled.Error()),
 			},
@@ -376,7 +388,7 @@ func Test_register(t *testing.T) {
 			name:            "enabled recaptcha, no response defined",
 			enableRecaptcha: true,
 			loginType:       authtypes.LoginTypeRecaptcha,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.BadJSON(ErrMissingResponse.Error()),
 			},
@@ -386,7 +398,7 @@ func Test_register(t *testing.T) {
 			enableRecaptcha: true,
 			loginType:       authtypes.LoginTypeRecaptcha,
 			captchaBody:     `notvalid`,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusUnauthorized,
 				JSON: spec.BadJSON(ErrInvalidCaptcha.Error()),
 			},
@@ -402,7 +414,7 @@ func Test_register(t *testing.T) {
 			enableRecaptcha:   true,
 			loginType:         authtypes.LoginTypeRecaptcha,
 			captchaBody:       `i should fail for other reasons`,
-			wantResponse:    util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}},
+			wantErrorResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}},
 		},
 	}
 
@@ -486,8 +498,8 @@ func Test_register(t *testing.T) {
 						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows)
 					}
 				case spec.MatrixError:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
+					if !reflect.DeepEqual(tc.wantErrorResponse, resp) {
-						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse)
+						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantErrorResponse)
 					}
 					return
 				case registerResponse:
@@ -505,6 +517,13 @@ func Test_register(t *testing.T) {
 					if r.DeviceID == "" {
 						t.Fatalf("missing deviceID in response")
 					}
+					// if an expected username is provided, assert that it is a match
+					if tc.wantUsername != "" {
+						wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test"))
+						if wantUserID != r.UserID {
+							t.Fatalf("unexpected userID: %s, want %s", r.UserID, wantUserID)
+						}
+					}
 					return
 				default:
 					t.Logf("Got response: %T", resp.JSON)
@@ -541,45 +560,30 @@ func Test_register(t *testing.T) {
 
 				resp = Register(req, userAPI, &cfg.ClientAPI)
 
-				switch resp.JSON.(type) {
+				switch rr := resp.JSON.(type) {
-				case spec.InternalServerError:
+				case spec.InternalServerError, spec.MatrixError, util.JSONResponse:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
+					if !reflect.DeepEqual(tc.wantErrorResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
+						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantErrorResponse)
 					}
 					return
-				case spec.MatrixError:
+				case registerResponse:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
-					}
-					return
-				case util.JSONResponse:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
-					}
-					return
-				}
-
-				rr, ok := resp.JSON.(registerResponse)
-				if !ok {
-					t.Fatalf("expected a registerresponse, got %T", resp.JSON)
-				}
-
 					// validate the response
-				if tc.forceEmpty {
+					if tc.wantUsername != "" {
-					// when not supplying a username, one will be generated. Given this _SHOULD_ be
+						// if an expected username is provided, assert that it is a match
-					// the second user, set the username accordingly
+						wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test"))
-					reg.Username = "2"
-				}
-				wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test"))
 						if wantUserID != rr.UserID {
 							t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID)
 						}
+					}
 					if rr.DeviceID != *reg.DeviceID {
 						t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID)
 					}
 					if rr.AccessToken == "" {
 						t.Fatalf("missing accessToken in response")
 					}
+				default:
+					t.Fatalf("expected one of internalservererror, matrixerror, jsonresponse, registerresponse, got %T", resp.JSON)
+				}
 			})
 		}
 	})

clientapi/routing/report_event.go

@@ -0,0 +1,93 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userAPI "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+)
+
+type reportEventRequest struct {
+	Reason string `json:"reason"`
+	Score  int64  `json:"score"`
+}
+
+func ReportEvent(
+	req *http.Request,
+	device *userAPI.Device,
+	roomID, eventID string,
+	rsAPI api.ClientRoomserverAPI,
+) util.JSONResponse {
+	defer req.Body.Close() // nolint: errcheck
+
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.NotFound("You don't have permission to report this event, bad userID"),
+		}
+	}
+	// The requesting user must be a member of the room
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
+	if errRes != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound, // Spec demands this...
+			JSON: spec.NotFound("The event was not found or you are not joined to the room."),
+		}
+	}
+
+	// Parse the request
+	report := reportEventRequest{}
+	if resErr := httputil.UnmarshalJSONRequest(req, &report); resErr != nil {
+		return *resErr
+	}
+
+	queryRes := &api.QueryEventsByIDResponse{}
+	if err = rsAPI.QueryEventsByID(req.Context(), &api.QueryEventsByIDRequest{
+		RoomID:   roomID,
+		EventIDs: []string{eventID},
+	}, queryRes); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{Err: err.Error()},
+		}
+	}
+
+	// No event was found or it was already redacted
+	if len(queryRes.Events) == 0 || queryRes.Events[0].Redacted() {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The event was not found or you are not joined to the room."),
+		}
+	}
+
+	_, err = rsAPI.InsertReportedEvent(req.Context(), roomID, eventID, device.UserID, report.Reason, report.Score)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{Err: err.Error()},
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/room_hierarchy.go

@@ -138,7 +138,7 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str
 		walker = *cachedWalker
 	}
 
-	discoveredRooms, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit)
+	discoveredRooms, _, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit)
 
 	if err != nil {
 		switch err.(type) {

clientapi/routing/routing.go

@@ -255,7 +255,7 @@ func Setup(
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
 		serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
 		if err != nil {
-			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+			logrus.WithError(err).Fatal("unable to get account for sending server notices")
 		}
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
@@ -1448,7 +1448,7 @@ func Setup(
 	// Cross-signing device keys
 
 	postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg)
+		return UploadCrossSigningDeviceKeys(req, userAPI, device, userAPI, cfg)
 	})
 
 	postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -1513,4 +1513,58 @@ func Setup(
 			return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomID}/joined_members",
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetJoinedMembers(req, device, vars["roomID"], rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomID}/report/{eventID}",
+		httputil.MakeAuthAPI("report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return ReportEvent(req, device, vars["roomID"], vars["eventID"], rsAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports",
+		httputil.MakeAdminAPI("admin_report_events", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			from := parseUint64OrDefault(req.URL.Query().Get("from"), 0)
+			limit := parseUint64OrDefault(req.URL.Query().Get("limit"), 100)
+			dir := req.URL.Query().Get("dir")
+			userID := req.URL.Query().Get("user_id")
+			roomID := req.URL.Query().Get("room_id")
+
+			// Go backwards if direction is empty or "b"
+			backwards := dir == "" || dir == "b"
+			return GetEventReports(req, rsAPI, from, limit, backwards, userID, roomID)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}",
+		httputil.MakeAdminAPI("admin_report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetEventReport(req, rsAPI, vars["reportID"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}",
+		httputil.MakeAdminAPI("admin_report_event_delete", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return DeleteEventReport(req, rsAPI, vars["reportID"])
+		}),
+	).Methods(http.MethodDelete, http.MethodOptions)
 }

clientapi/routing/sendevent.go

@@ -224,7 +224,7 @@ func SendEvent(
 		req.Context(), rsAPI,
 		api.KindNew,
 		[]*types.HeaderedEvent{
-			&types.HeaderedEvent{PDU: e},
+			{PDU: e},
 		},
 		device.UserDomain(),
 		domain,

clientapi/routing/server_notices.go

@@ -215,7 +215,7 @@ func SendServerNotice(
 		}
 		if !membershipRes.IsInRoom {
 			// re-invite the user
-			res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
+			res, err := sendInvite(ctx, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, time.Now())
 			if err != nil {
 				return res
 			}

cmd/dendrite-demo-pinecone/relay/retriever.go

@@ -17,13 +17,13 @@ package relay
 import (
 	"context"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 )
 
 const (
@@ -54,7 +54,7 @@ func NewRelayServerRetriever(
 		federationAPI:       federationAPI,
 		relayAPI:            relayAPI,
 		relayServersQueried: make(map[spec.ServerName]bool),
-		running:             *atomic.NewBool(false),
+		running:             atomic.Bool{},
 		quit:                quit,
 	}
 }

dendrite-sample.yaml

@@ -154,6 +154,13 @@ app_service_api:
   # to be sent to an insecure endpoint.
   disable_tls_validation: false
 
+  # Send the access_token query parameter with appservice requests in addition
+  # to the Authorization header. This can cause hs_tokens to be saved to logs,
+  # so it should not be enabled unless absolutely necessary.
+  legacy_auth: false
+  # Use the legacy unprefixed paths for appservice requests.
+  legacy_paths: false
+
   # Appservice configuration files to load into this homeserver.
   config_files:
   #  - /path/to/appservice_registration.yaml

docs/FAQ.md

@@ -24,7 +24,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo
 
 Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be 
 specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting 
-the development efforts through [contributing](../development/contributing). 
+the development efforts through [contributing](./development/CONTRIBUTING.md). 
 
 ## Is there a migration path from Synapse to Dendrite?
 
@@ -105,7 +105,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom <ve
 
 ## How do I reset somebody's password on my server?
 
-Use the admin endpoint [resetpassword](./administration/adminapi#post-_dendriteadminresetpassworduserid)
+Use the admin endpoint [resetpassword](./administration/4_adminapi.md#post-_dendriteadminresetpassworduserid)
 
 ## Should I use PostgreSQL or SQLite for my databases?
 
@@ -117,6 +117,7 @@ The list of files that need to be stored is:
 - matrix-key.pem
 - dendrite.yaml
 - the postgres or sqlite DB
+- the jetstream directory
 - the media store
 - the search index (although this can be regenerated)
 

docs/Gemfile.lock

@@ -231,9 +231,9 @@ GEM
       jekyll-seo-tag (~> 2.1)
     minitest (5.17.0)
     multipart-post (2.1.1)
-    nokogiri (1.14.3-arm64-darwin)
+    nokogiri (1.16.2-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.3-x86_64-linux)
+    nokogiri (1.16.2-x86_64-linux)
       racc (~> 1.4)
     octokit (4.22.0)
       faraday (>= 0.9)
@@ -241,7 +241,7 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.7)
-    racc (1.6.2)
+    racc (1.7.3)
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)

docs/installation/docker/1_docker.md

@@ -26,6 +26,8 @@ docker run --rm --entrypoint="/usr/bin/generate-keys" \
   -v $(pwd)/config:/mnt \
   matrixdotorg/dendrite-monolith:latest \
   -private-key /mnt/matrix_key.pem
+
+# Windows equivalent: docker run --rm --entrypoint="/usr/bin/generate-keys" -v %cd%/config:/mnt matrixdotorg/dendrite-monolith:latest -private-key /mnt/matrix_key.pem
 ```
 (**NOTE**: This only needs to be executed **once**, as you otherwise overwrite the key)
 
@@ -44,6 +46,8 @@ docker run --rm --entrypoint="/bin/sh" \
     -dir /var/dendrite/ \
     -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable \
     -server YourDomainHere > /mnt/dendrite.yaml"
+
+# Windows equivalent: docker run --rm --entrypoint="/bin/sh" -v %cd%/config:/mnt matrixdotorg/dendrite-monolith:latest -c "/usr/bin/generate-config -dir /var/dendrite/ -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable -server YourDomainHere > /mnt/dendrite.yaml"
 ```
 
 You can then change `config/dendrite.yaml` to your liking.

federationapi/federationapi.go

@@ -113,10 +113,7 @@ func NewInternalAPI(
 		_ = federationDB.RemoveAllServersFromBlacklist()
 	}
 
-	stats := statistics.NewStatistics(
+	stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1, cfg.P2PFederationRetriesUntilAssumedOffline+1, cfg.EnableRelays)
-		federationDB,
-		cfg.FederationMaxRetries+1,
-		cfg.P2PFederationRetriesUntilAssumedOffline+1)
 
 	js, nats := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream)
 
@@ -125,7 +122,7 @@ func NewInternalAPI(
 	queues := queue.NewOutgoingQueues(
 		federationDB, processContext,
 		cfg.Matrix.DisableFederation,
-		cfg.Matrix.ServerName, federation, rsAPI, &stats,
+		cfg.Matrix.ServerName, federation, &stats,
 		signingInfo,
 	)
 

federationapi/federationapi_test.go

@@ -5,11 +5,14 @@ import (
 	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -17,7 +20,10 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
@@ -362,3 +368,126 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 		}
 	}
 }
+
+func TestNotaryServer(t *testing.T) {
+	testCases := []struct {
+		name          string
+		httpBody      string
+		pubKeyRequest *gomatrixserverlib.PublicKeyNotaryLookupRequest
+		validateFunc  func(t *testing.T, response util.JSONResponse)
+	}{
+		{
+			name: "empty httpBody",
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusBadRequest, resp.Code)
+				nk, ok := resp.JSON.(spec.MatrixError)
+				assert.True(t, ok)
+				assert.Equal(t, spec.ErrorBadJSON, nk.ErrCode)
+			},
+		},
+		{
+			name:     "valid but empty httpBody",
+			httpBody: "{}",
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				want := util.JSONResponse{
+					Code: http.StatusOK,
+					JSON: routing.NotaryKeysResponse{ServerKeys: []json.RawMessage{}},
+				}
+				assert.Equal(t, want, resp)
+			},
+		},
+		{
+			name:     "request all keys using an empty criteria",
+			httpBody: `{"server_keys":{"servera":{}}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
+				assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
+			},
+		},
+		{
+			name:     "request all keys using null as the criteria",
+			httpBody: `{"server_keys":{"servera":null}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
+				assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
+			},
+		},
+		{
+			name:     "request specific key",
+			httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}}}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
+				assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
+			},
+		},
+		{
+			name:     "request multiple servers",
+			httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}},"serverb":{"ed25519:someID":{}}}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				wantServers := map[string]struct{}{
+					"servera": {},
+					"serverb": {},
+				}
+				for _, js := range nk.ServerKeys {
+					serverName := gjson.GetBytes(js, "server_name").Str
+					_, ok = wantServers[serverName]
+					assert.True(t, ok, "unexpected servername: %s", serverName)
+					delete(wantServers, serverName)
+					assert.True(t, gjson.GetBytes(js, "verify_keys.ed25519:someID").Exists())
+				}
+				if len(wantServers) > 0 {
+					t.Fatalf("expected response to also contain: %#v", wantServers)
+				}
+			},
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		fc := &fedClient{
+			keys: map[spec.ServerName]struct {
+				key   ed25519.PrivateKey
+				keyID gomatrixserverlib.KeyID
+			}{
+				"servera": {
+					key:   test.PrivateKeyA,
+					keyID: "ed25519:someID",
+				},
+				"serverb": {
+					key:   test.PrivateKeyB,
+					keyID: "ed25519:someID",
+				},
+			},
+		}
+
+		fedAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, fc, nil, caches, nil, true)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tc.httpBody))
+				req.Host = string(cfg.Global.ServerName)
+
+				resp := routing.NotaryKeys(req, &cfg.FederationAPI, fedAPI, tc.pubKeyRequest)
+				// assert that we received the expected response
+				tc.validateFunc(t, resp)
+			})
+		}
+
+	})
+}

federationapi/internal/federationclient_test.go

@@ -61,11 +61,11 @@ func TestFederationClientQueryKeys(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -92,11 +92,11 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -122,11 +122,11 @@ func TestFederationClientQueryKeysFailure(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{shouldFail: true}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -152,11 +152,11 @@ func TestFederationClientClaimKeys(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -183,11 +183,11 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{

federationapi/internal/perform_test.go

@@ -66,11 +66,11 @@ func TestPerformWakeupServers(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -112,11 +112,11 @@ func TestQueryRelayServers(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -153,11 +153,11 @@ func TestRemoveRelayServers(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -193,11 +193,11 @@ func TestPerformDirectoryLookup(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -232,11 +232,11 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(

federationapi/internal/query.go

@@ -43,6 +43,15 @@ func (a *FederationInternalAPI) fetchServerKeysFromCache(
 	ctx context.Context, req *api.QueryServerKeysRequest,
 ) ([]gomatrixserverlib.ServerKeys, error) {
 	var results []gomatrixserverlib.ServerKeys
+
+	// We got a request for _all_ server keys, return them.
+	if len(req.KeyIDToCriteria) == 0 {
+		serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{})
+		if len(serverKeysResponses) == 0 {
+			return nil, fmt.Errorf("failed to find server key response for server %s", req.ServerName)
+		}
+		return serverKeysResponses, nil
+	}
 	for keyID, criteria := range req.KeyIDToCriteria {
 		serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{keyID})
 		if len(serverKeysResponses) == 0 {

federationapi/queue/destinationqueue.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/matrix-org/gomatrix"
@@ -26,12 +27,10 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
-	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -53,7 +52,6 @@ type destinationQueue struct {
 	db                 storage.Database
 	process            *process.ProcessContext
 	signing            map[spec.ServerName]*fclient.SigningIdentity
-	rsAPI              api.FederationRoomserverAPI
 	client             fclient.FederationClient        // federation client
 	origin             spec.ServerName                 // origin of requests
 	destination        spec.ServerName                 // destination of requests
@@ -296,6 +294,10 @@ func (oq *destinationQueue) checkNotificationsOnClose() {
 
 // backgroundSend is the worker goroutine for sending events.
 func (oq *destinationQueue) backgroundSend() {
+	// Don't try to send transactions if we are shutting down.
+	if oq.process.Context().Err() != nil {
+		return
+	}
 	// Check if a worker is already running, and if it isn't, then
 	// mark it as started.
 	if !oq.running.CompareAndSwap(false, true) {

federationapi/queue/queue.go

@@ -27,12 +27,10 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
-	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
-	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -43,7 +41,6 @@ type OutgoingQueues struct {
 	db          storage.Database
 	process     *process.ProcessContext
 	disabled    bool
-	rsAPI       api.FederationRoomserverAPI
 	origin      spec.ServerName
 	client      fclient.FederationClient
 	statistics  *statistics.Statistics
@@ -90,7 +87,6 @@ func NewOutgoingQueues(
 	disabled bool,
 	origin spec.ServerName,
 	client fclient.FederationClient,
-	rsAPI api.FederationRoomserverAPI,
 	statistics *statistics.Statistics,
 	signing []*fclient.SigningIdentity,
 ) *OutgoingQueues {
@@ -98,7 +94,6 @@ func NewOutgoingQueues(
 		disabled:   disabled,
 		process:    process,
 		db:         db,
-		rsAPI:      rsAPI,
 		origin:     origin,
 		client:     client,
 		statistics: statistics,
@@ -162,7 +157,6 @@ func (oqs *OutgoingQueues) getQueue(destination spec.ServerName) *destinationQue
 			queues:      oqs,
 			db:          oqs.db,
 			process:     oqs.process,
-			rsAPI:       oqs.rsAPI,
 			origin:      oqs.origin,
 			destination: destination,
 			client:      oqs.client,
@@ -213,18 +207,6 @@ func (oqs *OutgoingQueues) SendEvent(
 		delete(destmap, local)
 	}
 
-	// Check if any of the destinations are prohibited by server ACLs.
-	for destination := range destmap {
-		if api.IsServerBannedFromRoom(
-			oqs.process.Context(),
-			oqs.rsAPI,
-			ev.RoomID().String(),
-			destination,
-		) {
-			delete(destmap, destination)
-		}
-	}
-
 	// If there are no remaining destinations then give up.
 	if len(destmap) == 0 {
 		return nil
@@ -303,24 +285,6 @@ func (oqs *OutgoingQueues) SendEDU(
 		delete(destmap, local)
 	}
 
-	// There is absolutely no guarantee that the EDU will have a room_id
-	// field, as it is not required by the spec. However, if it *does*
-	// (e.g. typing notifications) then we should try to make sure we don't
-	// bother sending them to servers that are prohibited by the server
-	// ACLs.
-	if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() {
-		for destination := range destmap {
-			if api.IsServerBannedFromRoom(
-				oqs.process.Context(),
-				oqs.rsAPI,
-				result.Str,
-				destination,
-			) {
-				delete(destmap, destination)
-			}
-		}
-	}
-
 	// If there are no remaining destinations then give up.
 	if len(destmap) == 0 {
 		return nil

federationapi/queue/queue_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -26,7 +27,6 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
-	"go.uber.org/atomic"
 	"gotest.tools/v3/poll"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -34,7 +34,6 @@ import (
 
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -65,15 +64,6 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase
 	}
 }
 
-type stubFederationRoomServerAPI struct {
-	rsapi.FederationRoomserverAPI
-}
-
-func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error {
-	res.Banned = false
-	return nil
-}
-
 type stubFederationClient struct {
 	fclient.FederationClient
 	shouldTxSucceed      bool
@@ -123,12 +113,11 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
 	fc := &stubFederationClient{
 		shouldTxSucceed:      shouldTxSucceed,
 		shouldTxRelaySucceed: shouldTxRelaySucceed,
-		txCount:              *atomic.NewUint32(0),
+		txCount:              atomic.Uint32{},
-		txRelayCount:         *atomic.NewUint32(0),
+		txRelayCount:         atomic.Uint32{},
 	}
-	rs := &stubFederationRoomServerAPI{}
 
-	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline, false)
 	signingInfo := []*fclient.SigningIdentity{
 		{
 			KeyID:      "ed21019:auto",
@@ -136,7 +125,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
 			ServerName: "localhost",
 		},
 	}
-	queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, rs, &stats, signingInfo)
+	queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, &stats, signingInfo)
 
 	return db, fc, queues, processContext, close
 }

federationapi/routing/backfill.go

@@ -95,6 +95,12 @@ func Backfill(
 		}
 	}
 
+	// Enforce a limit of 100 events, as not to hit the DB to hard.
+	// Synapse has a hard limit of 100 events as well.
+	if req.Limit > 100 {
+		req.Limit = 100
+	}
+
 	// Query the Roomserver.
 	if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed")

federationapi/routing/keys.go

@@ -197,6 +197,10 @@ func localKeys(cfg *config.FederationAPI, serverName spec.ServerName) (*gomatrix
 	return &keys, err
 }
 
+type NotaryKeysResponse struct {
+	ServerKeys []json.RawMessage `json:"server_keys"`
+}
+
 func NotaryKeys(
 	httpReq *http.Request, cfg *config.FederationAPI,
 	fsAPI federationAPI.FederationInternalAPI,
@@ -217,10 +221,9 @@ func NotaryKeys(
 		}
 	}
 
-	var response struct {
+	response := NotaryKeysResponse{
-		ServerKeys []json.RawMessage `json:"server_keys"`
+		ServerKeys: []json.RawMessage{},
 	}
-	response.ServerKeys = []json.RawMessage{}
 
 	for serverName, kidToCriteria := range req.ServerKeys {
 		var keyList []gomatrixserverlib.ServerKeys

federationapi/routing/query.go

@@ -146,7 +146,7 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques
 	}
 
 	walker := roomserverAPI.NewRoomHierarchyWalker(types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1)
-	discoveredRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1)
+	discoveredRooms, inaccessibleRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1)
 
 	if err != nil {
 		switch err.(type) {
@@ -177,6 +177,7 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques
 		JSON: fclient.RoomHierarchyResponse{
 			Room:                 discoveredRooms[0],
 			Children:             discoveredRooms[1:],
+			InaccessibleChildren: inaccessibleRooms,
 		},
 	}
 }

federationapi/routing/routing.go

@@ -647,6 +647,8 @@ func MakeFedAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
+			hub = hub.Clone()
 			hub.Scope().SetTag("origin", string(fedReq.Origin()))
 			hub.Scope().SetTag("uri", fedReq.RequestURI())
 		}

federationapi/statistics/statistics.go

@@ -5,10 +5,10 @@ import (
 	"math"
 	"math/rand"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -34,12 +34,15 @@ type Statistics struct {
 	// mark the destination as offline. At this point we should attempt
 	// to send messages to the user's async relay servers if we know them.
 	FailuresUntilAssumedOffline uint32
+
+	enableRelays bool
 }
 
 func NewStatistics(
 	db storage.Database,
 	failuresUntilBlacklist uint32,
 	failuresUntilAssumedOffline uint32,
+	enableRelays bool,
 ) Statistics {
 	return Statistics{
 		DB:                          db,
@@ -47,6 +50,7 @@ func NewStatistics(
 		FailuresUntilAssumedOffline: failuresUntilAssumedOffline,
 		backoffTimers:               make(map[spec.ServerName]*time.Timer),
 		servers:                     make(map[spec.ServerName]*ServerStatistics),
+		enableRelays:                enableRelays,
 	}
 }
 
@@ -73,6 +77,13 @@ func (s *Statistics) ForServer(serverName spec.ServerName) *ServerStatistics {
 		} else {
 			server.blacklisted.Store(blacklisted)
 		}
+
+		// Don't bother hitting the database 2 additional times
+		// if we don't want to use relays.
+		if !s.enableRelays {
+			return server
+		}
+
 		assumedOffline, err := s.DB.IsServerAssumedOffline(context.Background(), serverName)
 		if err != nil {
 			logrus.WithError(err).Errorf("Failed to get assumed offline entry %q", serverName)
@@ -158,7 +169,7 @@ func (s *ServerStatistics) Success(method SendMethod) {
 	// NOTE : Sending to the final destination vs. a relay server has
 	// slightly different semantics.
 	if method == SendDirect {
-		s.successCounter.Inc()
+		s.successCounter.Add(1)
 		if s.blacklisted.Load() && s.statistics.DB != nil {
 			if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil {
 				logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName)
@@ -184,7 +195,7 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 	// start a goroutine which will wait out the backoff and
 	// unset the backoffStarted flag when done.
 	if s.backoffStarted.CompareAndSwap(false, true) {
-		backoffCount := s.backoffCount.Inc()
+		backoffCount := s.backoffCount.Add(1)
 
 		if backoffCount >= s.statistics.FailuresUntilAssumedOffline {
 			s.assumedOffline.CompareAndSwap(false, true)

federationapi/statistics/statistics_test.go

@@ -16,7 +16,7 @@ const (
 )
 
 func TestBackoff(t *testing.T) {
-	stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	server := ServerStatistics{
 		statistics: &stats,
 		serverName: "test.com",
@@ -106,7 +106,7 @@ func TestBackoff(t *testing.T) {
 }
 
 func TestRelayServersListing(t *testing.T) {
-	stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	server := ServerStatistics{statistics: &stats}
 	server.AddRelayServers([]spec.ServerName{"relay1", "relay1", "relay2"})
 	relayServers := server.KnownRelayServers()

federationapi/storage/postgres/server_key_table.go

@@ -94,12 +94,14 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed")
 	results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
-	for rows.Next() {
+
 	var serverName string
 	var keyID string
 	var key string
 	var validUntilTS int64
 	var expiredTS int64
+	var vk gomatrixserverlib.VerifyKey
+	for rows.Next() {
 		if err = rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil {
 			return nil, err
 		}
@@ -107,7 +109,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 			ServerName: spec.ServerName(serverName),
 			KeyID:      gomatrixserverlib.KeyID(keyID),
 		}
-		vk := gomatrixserverlib.VerifyKey{}
 		err = vk.Key.Decode(key)
 		if err != nil {
 			return nil, err

federationapi/storage/sqlite3/server_key_table.go

@@ -98,12 +98,13 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 	err := sqlutil.RunLimitedVariablesQuery(
 		ctx, bulkSelectServerSigningKeysSQL, s.db, iKeyIDs, sqlutil.SQLite3MaxVariables,
 		func(rows *sql.Rows) error {
-			for rows.Next() {
 			var serverName string
 			var keyID string
 			var key string
 			var validUntilTS int64
 			var expiredTS int64
+			var vk gomatrixserverlib.VerifyKey
+			for rows.Next() {
 				if err := rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil {
 					return fmt.Errorf("bulkSelectServerKeys: %v", err)
 				}
@@ -111,7 +112,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 					ServerName: spec.ServerName(serverName),
 					KeyID:      gomatrixserverlib.KeyID(keyID),
 				}
-				vk := gomatrixserverlib.VerifyKey{}
 				err := vk.Key.Decode(key)
 				if err != nil {
 					return fmt.Errorf("bulkSelectServerKeys: %v", err)

federationapi/storage/tables/server_key_table_test.go

@@ -0,0 +1,116 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/federationapi/storage/postgres"
+	"github.com/matrix-org/dendrite/federationapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/federationapi/storage/tables"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateServerKeyDB(t *testing.T, dbType test.DBType) (tables.FederationServerSigningKeys, func()) {
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	if err != nil {
+		t.Fatalf("failed to open database: %s", err)
+	}
+	var tab tables.FederationServerSigningKeys
+	switch dbType {
+	case test.DBTypePostgres:
+		tab, err = postgres.NewPostgresServerSigningKeysTable(db)
+	case test.DBTypeSQLite:
+		tab, err = sqlite3.NewSQLiteServerSigningKeysTable(db)
+	}
+	if err != nil {
+		t.Fatalf("failed to create table: %s", err)
+	}
+	return tab, close
+}
+
+func TestServerKeysTable(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		ctx, cancel := context.WithCancel(context.Background())
+		tab, close := mustCreateServerKeyDB(t, dbType)
+		t.Cleanup(func() {
+			close()
+			cancel()
+		})
+
+		req := gomatrixserverlib.PublicKeyLookupRequest{
+			ServerName: "localhost",
+			KeyID:      "ed25519:test",
+		}
+		expectedTimestamp := spec.AsTimestamp(time.Now().Add(time.Hour))
+		res := gomatrixserverlib.PublicKeyLookupResult{
+			VerifyKey:    gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)},
+			ExpiredTS:    0,
+			ValidUntilTS: expectedTimestamp,
+		}
+
+		// Insert the key
+		err := tab.UpsertServerKeys(ctx, nil, req, res)
+		assert.NoError(t, err)
+
+		selectKeys := map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp{
+			req: spec.AsTimestamp(time.Now()),
+		}
+		gotKeys, err := tab.BulkSelectServerKeys(ctx, nil, selectKeys)
+		assert.NoError(t, err)
+
+		// Now we should have a key for the req above
+		assert.NotNil(t, gotKeys[req])
+		assert.Equal(t, res, gotKeys[req])
+
+		// "Expire" the key by setting ExpireTS to a non-zero value and ValidUntilTS to 0
+		expectedTimestamp = spec.AsTimestamp(time.Now())
+		res.ExpiredTS = expectedTimestamp
+		res.ValidUntilTS = 0
+
+		// Update the key
+		err = tab.UpsertServerKeys(ctx, nil, req, res)
+		assert.NoError(t, err)
+
+		gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys)
+		assert.NoError(t, err)
+
+		// The key should be expired
+		assert.NotNil(t, gotKeys[req])
+		assert.Equal(t, res, gotKeys[req])
+
+		// Upsert a different key to validate querying multiple keys
+		req2 := gomatrixserverlib.PublicKeyLookupRequest{
+			ServerName: "notlocalhost",
+			KeyID:      "ed25519:test2",
+		}
+		expectedTimestamp2 := spec.AsTimestamp(time.Now().Add(time.Hour))
+		res2 := gomatrixserverlib.PublicKeyLookupResult{
+			VerifyKey:    gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)},
+			ExpiredTS:    0,
+			ValidUntilTS: expectedTimestamp2,
+		}
+
+		err = tab.UpsertServerKeys(ctx, nil, req2, res2)
+		assert.NoError(t, err)
+
+		// Select multiple keys
+		selectKeys[req2] = spec.AsTimestamp(time.Now())
+
+		gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys)
+		assert.NoError(t, err)
+
+		// We now should receive two keys, one of which is expired
+		assert.Equal(t, 2, len(gotKeys))
+		assert.Equal(t, res2, gotKeys[req2])
+		assert.Equal(t, res, gotKeys[req])
+	})
+}

go.mod

@@ -6,14 +6,15 @@ require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
-	github.com/blevesearch/bleve/v2 v2.3.8
+	github.com/blevesearch/bleve/v2 v2.4.0
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/dgraph-io/ristretto v0.1.1
-	github.com/docker/docker v24.0.7+incompatible
+	github.com/docker/docker v24.0.9+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.14.0
+	github.com/go-ldap/ldap/v3 v3.4.4
 	github.com/gologme/log v1.3.0
-	github.com/google/go-cmp v0.5.9
+	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
@@ -22,66 +23,70 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20240328203753-c2391f7113a5
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
-	github.com/mattn/go-sqlite3 v1.14.17
+	github.com/mattn/go-sqlite3 v1.14.22
-	github.com/nats-io/nats-server/v2 v2.9.23
+	github.com/nats-io/nats-server/v2 v2.10.7
-	github.com/nats-io/nats.go v1.28.0
+	github.com/nats-io/nats.go v1.31.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_golang v1.19.0
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.9.0
-	github.com/tidwall/gjson v1.17.0
+	github.com/tidwall/gjson v1.17.1
 	github.com/tidwall/sjson v1.2.5
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
-	go.uber.org/atomic v1.10.0
+	go.uber.org/atomic v1.11.0
-	golang.org/x/crypto v0.14.0
+	golang.org/x/crypto v0.22.0
-	golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
+	golang.org/x/exp v0.0.0-20231108232855-2478ac86f678
 	golang.org/x/image v0.10.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
-	golang.org/x/sync v0.3.0
+	golang.org/x/sync v0.7.0
-	golang.org/x/term v0.13.0
+	golang.org/x/term v0.19.0
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.4.0
 	maunium.net/go/mautrix v0.15.1
-	modernc.org/sqlite v1.23.1
+	modernc.org/sqlite v1.29.5
 	nhooyr.io/websocket v1.8.7
 )
 
 require (
+	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/RoaringBitmap/roaring v1.2.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.5.0 // indirect
-	github.com/blevesearch/bleve_index_api v1.0.5 // indirect
+	github.com/blevesearch/bleve_index_api v1.1.6 // indirect
-	github.com/blevesearch/geo v0.1.17 // indirect
+	github.com/blevesearch/geo v0.1.20 // indirect
+	github.com/blevesearch/go-faiss v1.0.13 // indirect
 	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
 	github.com/blevesearch/gtreap v0.1.1 // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
-	github.com/blevesearch/scorch_segment_api/v2 v2.1.4 // indirect
+	github.com/blevesearch/scorch_segment_api/v2 v2.2.9 // indirect
 	github.com/blevesearch/segment v0.9.1 // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
 	github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
-	github.com/blevesearch/vellum v1.0.9 // indirect
+	github.com/blevesearch/vellum v1.0.10 // indirect
-	github.com/blevesearch/zapx/v11 v11.3.7 // indirect
+	github.com/blevesearch/zapx/v11 v11.3.10 // indirect
-	github.com/blevesearch/zapx/v12 v12.3.7 // indirect
+	github.com/blevesearch/zapx/v12 v12.3.10 // indirect
-	github.com/blevesearch/zapx/v13 v13.3.7 // indirect
+	github.com/blevesearch/zapx/v13 v13.3.10 // indirect
-	github.com/blevesearch/zapx/v14 v14.3.7 // indirect
+	github.com/blevesearch/zapx/v14 v14.3.10 // indirect
-	github.com/blevesearch/zapx/v15 v15.3.10 // indirect
+	github.com/blevesearch/zapx/v15 v15.3.13 // indirect
+	github.com/blevesearch/zapx/v16 v16.0.12 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
@@ -91,56 +96,52 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v1.0.0 // indirect
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
-	github.com/nats-io/jwt/v2 v2.5.0 // indirect
+	github.com/nats-io/jwt/v2 v2.5.5 // indirect
-	github.com/nats-io/nkeys v0.4.6 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.2 // indirect
-	github.com/quic-go/quic-go v0.37.4 // indirect
+	github.com/quic-go/quic-go v0.37.7 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/zerolog v1.29.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
-	go.etcd.io/bbolt v1.3.6 // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.12.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/uint128 v1.2.0 // indirect
 	maunium.net/go/maulogger/v2 v2.4.1 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
+	modernc.org/libc v1.41.0 // indirect
-	modernc.org/libc v1.22.5 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
-	modernc.org/memory v1.5.0 // indirect
+	modernc.org/strutil v1.2.0 // indirect
-	modernc.org/opt v0.1.3 // indirect
+	modernc.org/token v1.1.0 // indirect
-	modernc.org/strutil v1.1.3 // indirect
-	modernc.org/token v1.0.1 // indirect
 )
 
 go 1.20

go.sum

@@ -5,6 +5,8 @@ github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3
 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
+github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
@@ -36,38 +38,42 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8=
 github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/blevesearch/bleve/v2 v2.3.8 h1:IqFyMJ73n4gY8AmVqM8Sa6EtAZ5beE8yramVqCvs2kQ=
+github.com/blevesearch/bleve/v2 v2.4.0 h1:2xyg+Wv60CFHYccXc+moGxbL+8QKT/dZK09AewHgKsg=
-github.com/blevesearch/bleve/v2 v2.3.8/go.mod h1:Lh9aZEHrLKxwPnW4z4lsBEGnflZQ1V/aWP/t+htsiDw=
+github.com/blevesearch/bleve/v2 v2.4.0/go.mod h1:IhQHoFAbHgWKYavb9rQgQEJJVMuY99cKdQ0wPpst2aY=
-github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw=
+github.com/blevesearch/bleve_index_api v1.1.6 h1:orkqDFCBuNU2oHW9hN2YEJmet+TE9orml3FCGbl1cKk=
-github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
+github.com/blevesearch/bleve_index_api v1.1.6/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
-github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA=
+github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
-github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
+github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
+github.com/blevesearch/go-faiss v1.0.13 h1:zfFs7ZYD0NqXVSY37j0JZjZT1BhE9AE4peJfcx/NB4A=
+github.com/blevesearch/go-faiss v1.0.13/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
 github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
 github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
 github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
 github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
 github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.4 h1:LmGmo5twU3gV+natJbKmOktS9eMhokPGKWuR+jX84vk=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.9 h1:3nBaSBRFokjE4FtPW3eUDgcAu3KphBg1GP07zy/6Uyk=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.4/go.mod h1:PgVnbbg/t1UkgezPDu8EHLi1BHQ17xUwsFdU6NnOYS0=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.9/go.mod h1:ckbeb7knyOOvAdZinn/ASbB7EA3HoagnJkmEV3J7+sg=
 github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
 github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
 github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
 github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
 github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
 github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
-github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ=
+github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
-github.com/blevesearch/vellum v1.0.9/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
+github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
-github.com/blevesearch/zapx/v11 v11.3.7 h1:Y6yIAF/DVPiqZUA/jNgSLXmqewfzwHzuwfKyfdG+Xaw=
+github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
-github.com/blevesearch/zapx/v11 v11.3.7/go.mod h1:Xk9Z69AoAWIOvWudNDMlxJDqSYGf90LS0EfnaAIvXCA=
+github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
-github.com/blevesearch/zapx/v12 v12.3.7 h1:DfQ6rsmZfEK4PzzJJRXjiM6AObG02+HWvprlXQ1Y7eI=
+github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
-github.com/blevesearch/zapx/v12 v12.3.7/go.mod h1:SgEtYIBGvM0mgIBn2/tQE/5SdrPXaJUaT/kVqpAPxm0=
+github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
-github.com/blevesearch/zapx/v13 v13.3.7 h1:igIQg5eKmjw168I7av0Vtwedf7kHnQro/M+ubM4d2l8=
+github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
-github.com/blevesearch/zapx/v13 v13.3.7/go.mod h1:yyrB4kJ0OT75UPZwT/zS+Ru0/jYKorCOOSY5dBzAy+s=
+github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
-github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20=
+github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
-github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w=
+github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
-github.com/blevesearch/zapx/v15 v15.3.10 h1:bQ9ZxJCj6rKp873EuVJu2JPxQ+EWQZI1cjJGeroovaQ=
+github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
-github.com/blevesearch/zapx/v15 v15.3.10/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY=
+github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
+github.com/blevesearch/zapx/v16 v16.0.12 h1:Uccxvjmn+hQ6ywQP+wIiTpdq9LnAviGoryJOmGwAo/I=
+github.com/blevesearch/zapx/v16 v16.0.12/go.mod h1:MYnOshRfSm4C4drxx1LGRI+MVFByykJ2anDY1fxdk9Q=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
@@ -89,8 +95,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -111,8 +117,12 @@ github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwv
 github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8=
 github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
+github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
+github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -141,7 +151,6 @@ github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -158,8 +167,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c=
 github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
@@ -174,6 +183,8 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -185,13 +196,11 @@ github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0=
 github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -208,8 +217,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca h1:JCP72vU4Vcmur2071RwYVOSoekR+ZjbC03wZD5lAAK0=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240328203753-c2391f7113a5 h1:GuxmpyjZQoqb6UFQgKq8Td3wIITlXln/sItqp1jbTTA=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240328203753-c2391f7113a5/go.mod h1:HZGsVJ3bUE+DkZtufkH9H0mlsvbhEGK5CpX0Zlavylg=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
@@ -222,10 +231,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
@@ -242,16 +249,18 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
-github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak=
+github.com/nats-io/jwt/v2 v2.5.5 h1:ROfXb50elFq5c9+1ztaUbdlrArNFl2+fQWP6B8HGEq4=
-github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
+github.com/nats-io/jwt/v2 v2.5.5/go.mod h1:ZdWS1nZa6WMZfFwwgpEaqBV8EPGVgOTDHN/wTbz0Y5A=
-github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU=
+github.com/nats-io/nats-server/v2 v2.10.7 h1:f5VDy+GMu7JyuFA0Fef+6TfulfCs5nBTgq7MMkFJx5Y=
-github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0=
+github.com/nats-io/nats-server/v2 v2.10.7/go.mod h1:V2JHOvPiPdtfDXTuEUsthUnCvSDeFrK4Xn9hRo6du7c=
-github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c=
+github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
-github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
+github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
-github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
-github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
+github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
@@ -276,23 +285,21 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
-github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
-github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
-github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
-github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
-github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
-github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
-github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
 github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
-github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4=
+github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU=
-github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
+github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
 github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
@@ -304,22 +311,18 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
-github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
@@ -343,10 +346,10 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
-go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -354,15 +357,15 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w=
-golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -377,8 +380,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -386,31 +389,31 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -422,24 +425,25 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -452,8 +456,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -464,8 +468,8 @@ gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6d
 gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -485,34 +489,25 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
 gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
-lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
-lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8=
 maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho=
 maunium.net/go/mautrix v0.15.1 h1:pmCtMjYRpd83+2UL+KTRFYQo5to0373yulimvLK+1k0=
 maunium.net/go/mautrix v0.15.1/go.mod h1:icQIrvz2NldkRLTuzSGzmaeuMUmw+fzO7UVycPeauN8=
-modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
-modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
-modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
-modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
+modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk=
-modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
+modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY=
-modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
-modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
-modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
-modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
-modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/sqlite v1.29.5 h1:8l/SQKAjDtZFo9lkJLdk8g9JEOeYRG4/ghStDCCTiTE=
-modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
+modernc.org/sqlite v1.29.5/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=
-modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
-modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
-modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
-modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
-modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
-modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
-modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
-modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
-modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

helm/dendrite/Chart.yaml

@@ -1,9 +1,10 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.5"
+version: "0.14.1"
-appVersion: "0.13.4"
+appVersion: "0.13.7"
 description: Dendrite Matrix Homeserver
 type: application
+icon: https://avatars.githubusercontent.com/u/8418310?s=48&v=4
 keywords:
   - matrix
   - chat
@@ -14,6 +15,6 @@ sources:
   - https://github.com/matrix-org/dendrite
 dependencies:
   - name: postgresql
-  version: 12.1.7
+    version: 14.2.3
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled

helm/dendrite/README.md

@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.5](https://img.shields.io/badge/Version-0.13.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.4](https://img.shields.io/badge/AppVersion-0.13.4-informational?style=flat-square)
+![Version: 0.14.0](https://img.shields.io/badge/Version-0.14.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.7](https://img.shields.io/badge/AppVersion-0.13.7-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**
@@ -37,7 +37,7 @@ Create a folder `appservices` and place your configurations in there.  The confi
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 |
+| https://charts.bitnami.com/bitnami | postgresql | 14.2.3 |
 ## Values
 
 | Key | Type | Default | Description |
@@ -48,16 +48,19 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | signing_key.create | bool | `true` | Create a new signing key, if not exists |
 | signing_key.existingSecret | string | `""` | Use an existing secret |
 | resources | object | sets some sane default values | Default resource requests/limits. |
-| persistence.jetstream | object | `{"capacity":"1Gi","existingClaim":""}` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. # If defined, storageClassName: <storageClass> # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is #   set, choosing the default provisioner.  (gp2 on AWS, standard on #   GKE, AWS & OpenStack) # storageClass: "" |
+| persistence.storageClass | string | `nil` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack)  |
 | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream |
 | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume |
+| persistence.jetstream.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files |
 | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume |
+| persistence.media.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index |
 | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume |
+| persistence.search.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod |
 | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod |
-| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
+| strategy.type | string | `"Recreate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
 | strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process |
 | strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods |
 | dendrite_config.version | int | `2` |  |
@@ -139,7 +142,7 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration |
 | postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. |
 | postgresql.image.repository | string | `"bitnami/postgresql"` |  |
-| postgresql.image.tag | string | `"15.1.0"` |  |
+| postgresql.image.tag | string | `"16.2.0"` |  |
 | postgresql.auth.username | string | `"dendrite"` |  |
 | postgresql.auth.password | string | `"changeme"` |  |
 | postgresql.auth.database | string | `"dendrite"` |  |
@@ -186,3 +189,5 @@ grafana:
 ```
 PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`)
 
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)

helm/dendrite/grafana_dashboards/dendrite-rev2.json

@@ -119,7 +119,7 @@
           "refId": "A"
         }
       ],
-      "title": "Registerd Users",
+      "title": "Registered Users",
       "type": "stat"
     },
     {

helm/dendrite/templates/ingress.yaml

@@ -4,6 +4,7 @@
 {{- $wellKnownServerHost := default $serverNameHost (regexFind "^(\\[.+\\])?[^:]*" .Values.dendrite_config.global.well_known_server_name) -}}
 {{- $wellKnownClientHost := default $serverNameHost (regexFind "//(\\[.+\\])?[^:/]*" .Values.dendrite_config.global.well_known_client_name | trimAll "/") -}}
 {{- $allHosts := list $serverNameHost $wellKnownServerHost $wellKnownClientHost | uniq -}}
+
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
@@ -56,7 +57,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
@@ -72,7 +73,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
@@ -88,7 +89,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
@@ -105,7 +106,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http

helm/dendrite/templates/service.yaml

@@ -14,4 +14,4 @@ spec:
     - name: http
       protocol: TCP
       port: {{ .Values.service.port }}
-      targetPort: 8008
+      targetPort: http

helm/dendrite/values.yaml

@@ -26,13 +26,13 @@ persistence:
   # -- The storage class to use for volume claims.
   # Used unless specified at the specific component.
   # Defaults to the cluster default storage class.
-  ## If defined, storageClassName: <storageClass>
+  # If defined, storageClassName: <storageClass>
-  ## If set to "-", storageClassName: "", which disables dynamic provisioning
+  # If set to "-", storageClassName: "", which disables dynamic provisioning
-  ## If undefined (the default) or set to null, no storageClassName spec is
+  # If undefined (the default) or set to null, no storageClassName spec is
-  ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+  #   set, choosing the default provisioner.  (gp2 on AWS, standard on
-  ##   GKE, AWS & OpenStack)
+  #   GKE, AWS & OpenStack)
-  ##
+  #
-  # storageClass: ""
+  storageClass:
   jetstream:
     # -- Use an existing volume claim for jetstream
     existingClaim: ""
@@ -40,13 +40,12 @@ persistence:
     capacity: "1Gi"
     # -- The storage class to use for volume claims.
     # Defaults to persistence.storageClass
-    ## If defined, storageClassName: <storageClass>
+    # If defined, storageClassName: <storageClass>
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    # If set to "-", storageClassName: "", which disables dynamic provisioning
-    ## If undefined (the default) or set to null, no storageClassName spec is
+    # If undefined (the default) or set to null, no storageClassName spec is
-    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    #   set, choosing the default provisioner.  (gp2 on AWS, standard on
-    ##   GKE, AWS & OpenStack)
+    #   GKE, AWS & OpenStack)
-    ##
+    storageClass:
-    # storageClass: ""
   media:
     # -- Use an existing volume claim for media files
     existingClaim: ""
@@ -54,13 +53,12 @@ persistence:
     capacity: "1Gi"
     # -- The storage class to use for volume claims.
     # Defaults to persistence.storageClass
-    ## If defined, storageClassName: <storageClass>
+    # If defined, storageClassName: <storageClass>
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    # If set to "-", storageClassName: "", which disables dynamic provisioning
-    ## If undefined (the default) or set to null, no storageClassName spec is
+    # If undefined (the default) or set to null, no storageClassName spec is
-    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    #   set, choosing the default provisioner.  (gp2 on AWS, standard on
-    ##   GKE, AWS & OpenStack)
+    #   GKE, AWS & OpenStack)
-    ##
+    storageClass:
-    # storageClass: ""
   search:
     # -- Use an existing volume claim for the fulltext search index
     existingClaim: ""
@@ -68,13 +66,12 @@ persistence:
     capacity: "1Gi"
     # -- The storage class to use for volume claims.
     # Defaults to persistence.storageClass
-    ## If defined, storageClassName: <storageClass>
+    # If defined, storageClassName: <storageClass>
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    # If set to "-", storageClassName: "", which disables dynamic provisioning
-    ## If undefined (the default) or set to null, no storageClassName spec is
+    # If undefined (the default) or set to null, no storageClassName spec is
-    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    #   set, choosing the default provisioner.  (gp2 on AWS, standard on
-    ##   GKE, AWS & OpenStack)
+    #   GKE, AWS & OpenStack)
-    ##
+    storageClass:
-    # storageClass: ""
 
 # -- Add additional volumes to the Dendrite Pod
 extraVolumes: []
@@ -92,7 +89,7 @@ extraVolumeMounts: []
 strategy:
   # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate)
   # If you are using ReadWriteOnce volumes, you should probably use Recreate
-  type: RollingUpdate
+  type: Recreate
   rollingUpdate:
     # -- Maximum number of pods that can be unavailable during the update process
     maxUnavailable: 25%
@@ -378,7 +375,7 @@ postgresql:
   enabled: false
   image:
     repository: bitnami/postgresql
-    tag: "15.1.0"
+    tag: "16.2.0"
   auth:
     username: dendrite
     password: changeme

internal/httputil/httpapi.go

@@ -76,6 +76,8 @@ func MakeAuthAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
+			hub = hub.Clone()
 			hub.Scope().SetUser(sentry.User{
 				Username: device.UserID,
 			})

internal/httputil/routing.go

@@ -66,15 +66,15 @@ func NewRouters() Routers {
 }
 
 var NotAllowedHandler = WrapHandlerInCORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusMethodNotAllowed)
 	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusMethodNotAllowed)
 	unrecognizedErr, _ := json.Marshal(spec.Unrecognized("Unrecognized request")) // nolint:misspell
 	_, _ = w.Write(unrecognizedErr)                                               // nolint:misspell
 }))
 
 var NotFoundCORSHandler = WrapHandlerInCORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusNotFound)
 	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusNotFound)
 	unrecognizedErr, _ := json.Marshal(spec.Unrecognized("Unrecognized request")) // nolint:misspell
 	_, _ = w.Write(unrecognizedErr)                                               // nolint:misspell
 }))

internal/httputil/routing_test.go

@@ -17,7 +17,7 @@ func TestRoutersError(t *testing.T) {
 	if rec.Code != http.StatusNotFound {
 		t.Fatalf("unexpected status code: %d - %s", rec.Code, rec.Body.String())
 	}
-	if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
+	if ct := rec.Result().Header.Get("Content-Type"); ct != "application/json" {
 		t.Fatalf("unexpected content-type: %s", ct)
 	}
 
@@ -32,7 +32,7 @@ func TestRoutersError(t *testing.T) {
 	if rec.Code != http.StatusMethodNotAllowed {
 		t.Fatalf("unexpected status code: %d - %s", rec.Code, rec.Body.String())
 	}
-	if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
+	if ct := rec.Result().Header.Get("Content-Type"); ct != "application/json" {
 		t.Fatalf("unexpected content-type: %s", ct)
 	}
 }

internal/sqlutil/writer_exclusive.go

@@ -3,8 +3,7 @@ package sqlutil
 import (
 	"database/sql"
 	"errors"
-
+	"sync/atomic"
-	"go.uber.org/atomic"
 )
 
 // ExclusiveWriter implements sqlutil.Writer.

internal/transactionrequest_test.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strconv"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -26,7 +27,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
-	"go.uber.org/atomic"
 	"gotest.tools/v3/poll"
 
 	"github.com/matrix-org/dendrite/federationapi/producers"
@@ -228,7 +228,7 @@ func TestProcessTransactionRequestEDUTyping(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 		room := msg.Header.Get(jetstream.RoomID)
@@ -294,7 +294,7 @@ func TestProcessTransactionRequestEDUToDevice(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 
@@ -371,7 +371,7 @@ func TestProcessTransactionRequestEDUDeviceListUpdate(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 
@@ -468,7 +468,7 @@ func TestProcessTransactionRequestEDUReceipt(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 
@@ -512,7 +512,7 @@ func TestProcessTransactionRequestEDUSigningKeyUpdate(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 
@@ -569,7 +569,7 @@ func TestProcessTransactionRequestEDUPresence(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 

internal/validate.go

@@ -20,6 +20,9 @@ import (
 	"net/http"
 	"regexp"
 
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
@@ -35,9 +38,9 @@ var (
 	ErrPasswordTooLong    = fmt.Errorf("password too long: max %d characters", maxPasswordLength)
 	ErrPasswordWeak       = fmt.Errorf("password too weak: min %d characters", minPasswordLength)
 	ErrUsernameTooLong    = fmt.Errorf("username exceeds the maximum length of %d characters", maxUsernameLength)
-	ErrUsernameInvalid    = errors.New("username can only contain characters a-z, 0-9, or '_-./='")
+	ErrUsernameInvalid    = errors.New("username can only contain characters a-z, 0-9, or '_+-./='")
 	ErrUsernameUnderscore = errors.New("username cannot start with a '_'")
-	validUsernameRegex    = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
+	validUsernameRegex    = regexp.MustCompile(`^[0-9a-z_\-+=./]+$`)
 )
 
 // ValidatePassword returns an error if the password is invalid
@@ -100,10 +103,139 @@ func UsernameResponse(err error) *util.JSONResponse {
 
 // ValidateApplicationServiceUsername returns an error if the username is invalid for an application service
 func ValidateApplicationServiceUsername(localpart string, domain spec.ServerName) error {
-	if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
+	userID := userutil.MakeUserID(localpart, domain)
+	return ValidateApplicationServiceUserID(userID)
+}
+
+func ValidateApplicationServiceUserID(userID string) error {
+	if len(userID) > maxUsernameLength {
 		return ErrUsernameTooLong
-	} else if !validUsernameRegex.MatchString(localpart) {
+	}
+
+	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil || !validUsernameRegex.MatchString(localpart) {
 		return ErrUsernameInvalid
 	}
+
 	return nil
 }
+
+// userIDIsWithinApplicationServiceNamespace checks to see if a given userID
+// falls within any of the namespaces of a given Application Service. If no
+// Application Service is given, it will check to see if it matches any
+// Application Service's namespace.
+func userIDIsWithinApplicationServiceNamespace(
+	cfg *config.ClientAPI,
+	userID string,
+	appservice *config.ApplicationService,
+) bool {
+	var localpart, domain, err = gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		// Not a valid userID
+		return false
+	}
+
+	if !cfg.Matrix.IsLocalServerName(domain) {
+		// This is a federated userID
+		return false
+	}
+
+	if localpart == appservice.SenderLocalpart {
+		// This is the application service bot userID
+		return true
+	}
+
+	// Loop through given application service's namespaces and see if any match
+	for _, namespace := range appservice.NamespaceMap["users"] {
+		// Application service namespaces are checked for validity in config
+		if namespace.RegexpObject.MatchString(userID) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// usernameMatchesMultipleExclusiveNamespaces will check if a given username matches
+// more than one exclusive namespace. More than one is not allowed
+func userIDMatchesMultipleExclusiveNamespaces(
+	cfg *config.ClientAPI,
+	userID string,
+) bool {
+	// Check namespaces and see if more than one match
+	matchCount := 0
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		if appservice.OwnsNamespaceCoveringUserId(userID) {
+			if matchCount++; matchCount > 1 {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// ValidateApplicationServiceRequest checks if a provided application service
+// token corresponds to one that is registered, and, if so, checks if the
+// supplied userIDOrLocalpart is within that application service's namespace.
+//
+// As long as these two requirements are met, the matched application service
+// ID will be returned. Otherwise, it will return a JSON response with the
+// appropriate error message.
+func ValidateApplicationServiceRequest(
+	cfg *config.ClientAPI,
+	userIDOrLocalpart string,
+	accessToken string,
+) (string, *util.JSONResponse) {
+	localpart, domain, err := userutil.ParseUsernameParam(userIDOrLocalpart, cfg.Matrix)
+	if err != nil {
+		return "", &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: spec.InvalidUsername(err.Error()),
+		}
+	}
+
+	userID := userutil.MakeUserID(localpart, domain)
+
+	// Check if the token if the application service is valid with one we have
+	// registered in the config.
+	var matchedApplicationService *config.ApplicationService
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		if appservice.ASToken == accessToken {
+			matchedApplicationService = &appservice
+			break
+		}
+	}
+	if matchedApplicationService == nil {
+		return "", &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: spec.UnknownToken("Supplied access_token does not match any known application service"),
+		}
+	}
+
+	// Ensure the desired username is within at least one of the application service's namespaces.
+	if !userIDIsWithinApplicationServiceNamespace(cfg, userID, matchedApplicationService) {
+		// If we didn't find any matches, return M_EXCLUSIVE
+		return "", &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.ASExclusive(fmt.Sprintf(
+				"Supplied username %s did not match any namespaces for application service ID: %s", userIDOrLocalpart, matchedApplicationService.ID)),
+		}
+	}
+
+	// Check this user does not fit multiple application service namespaces
+	if userIDMatchesMultipleExclusiveNamespaces(cfg, userID) {
+		return "", &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.ASExclusive(fmt.Sprintf(
+				"Supplied username %s matches multiple exclusive application service namespaces. Only 1 match allowed", userIDOrLocalpart)),
+		}
+	}
+
+	// Check username application service is trying to register is valid
+	if err := ValidateApplicationServiceUserID(userID); err != nil {
+		return "", UsernameResponse(err)
+	}
+
+	// No errors, registration valid
+	return matchedApplicationService.ID, nil
+}

internal/validate_test.go

@@ -3,9 +3,11 @@ package internal
 import (
 	"net/http"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
@@ -38,7 +40,7 @@ func Test_validatePassword(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			gotErr := ValidatePassword(tt.password)
 			if !reflect.DeepEqual(gotErr, tt.wantError) {
-				t.Errorf("validatePassword() = %v, wantJSON %v", gotErr, tt.wantError)
+				t.Errorf("validatePassword() = %v, wantError %v", gotErr, tt.wantError)
 			}
 
 			if got := PasswordResponse(gotErr); !reflect.DeepEqual(got, tt.wantJSON) {
@@ -127,6 +129,11 @@ func Test_validateUsername(t *testing.T) {
 			localpart: "i_am_allowed=1",
 			domain:    "localhost",
 		},
+		{
+			name:      "special characters are allowed 3",
+			localpart: "+55555555555",
+			domain:    "localhost",
+		},
 		{
 			name:      "not all special characters are allowed",
 			localpart: "notallowed#", // contains #
@@ -137,6 +144,16 @@ func Test_validateUsername(t *testing.T) {
 				JSON: spec.InvalidUsername(ErrUsernameInvalid.Error()),
 			},
 		},
+		{
+			name:      "not all special characters are allowed 2",
+			localpart: "<notallowed", // contains <
+			domain:    "localhost",
+			wantErr:   ErrUsernameInvalid,
+			wantJSON: &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidUsername(ErrUsernameInvalid.Error()),
+			},
+		},
 		{
 			name:      "username containing numbers",
 			localpart: "hello1337",
@@ -167,3 +184,133 @@ func Test_validateUsername(t *testing.T) {
 		})
 	}
 }
+
+// This method tests validation of the provided Application Service token and
+// username that they're registering
+func TestValidateApplicationServiceRequest(t *testing.T) {
+	// Create a fake application service
+	regex := "@_appservice_.*"
+	fakeNamespace := config.ApplicationServiceNamespace{
+		Exclusive:    true,
+		Regex:        regex,
+		RegexpObject: regexp.MustCompile(regex),
+	}
+	fakeSenderLocalpart := "_appservice_bot"
+	fakeApplicationService := config.ApplicationService{
+		ID:              "FakeAS",
+		URL:             "null",
+		ASToken:         "1234",
+		HSToken:         "4321",
+		SenderLocalpart: fakeSenderLocalpart,
+		NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+			"users": {fakeNamespace},
+		},
+	}
+
+	// Create a second fake application service where userIDs ending in
+	// "_overlap" overlap with the first.
+	regex = "@_.*_overlap"
+	fakeNamespace = config.ApplicationServiceNamespace{
+		Exclusive:    true,
+		Regex:        regex,
+		RegexpObject: regexp.MustCompile(regex),
+	}
+	fakeApplicationServiceOverlap := config.ApplicationService{
+		ID:              "FakeASOverlap",
+		URL:             fakeApplicationService.URL,
+		ASToken:         fakeApplicationService.ASToken,
+		HSToken:         fakeApplicationService.HSToken,
+		SenderLocalpart: "_appservice_bot_overlap",
+		NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+			"users": {fakeNamespace},
+		},
+	}
+
+	// Set up a config
+	fakeConfig := &config.Dendrite{}
+	fakeConfig.Defaults(config.DefaultOpts{
+		Generate: true,
+	})
+	fakeConfig.Global.ServerName = "localhost"
+	fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService, fakeApplicationServiceOverlap}
+
+	tests := []struct {
+		name      string
+		localpart string
+		asToken   string
+		wantError bool
+		wantASID  string
+	}{
+		// Access token is correct, userID omitted so we are acting as SenderLocalpart
+		{
+			name:      "correct access token but omitted userID",
+			localpart: fakeSenderLocalpart,
+			asToken:   fakeApplicationService.ASToken,
+			wantError: false,
+			wantASID:  fakeApplicationService.ID,
+		},
+		// Access token is incorrect, userID omitted so we are acting as SenderLocalpart
+		{
+			name:      "incorrect access token but omitted userID",
+			localpart: fakeSenderLocalpart,
+			asToken:   "xxxx",
+			wantError: true,
+			wantASID:  "",
+		},
+		// Access token is correct, acting as valid userID
+		{
+			name:      "correct access token and valid userID",
+			localpart: "_appservice_bob",
+			asToken:   fakeApplicationService.ASToken,
+			wantError: false,
+			wantASID:  fakeApplicationService.ID,
+		},
+		// Access token is correct, acting as invalid userID
+		{
+			name:      "correct access token but invalid userID",
+			localpart: "_something_else",
+			asToken:   fakeApplicationService.ASToken,
+			wantError: true,
+			wantASID:  "",
+		},
+		// Access token is correct, acting as userID that matches two exclusive namespaces
+		{
+			name:      "correct access token but non-exclusive userID",
+			localpart: "_appservice_overlap",
+			asToken:   fakeApplicationService.ASToken,
+			wantError: true,
+			wantASID:  "",
+		},
+		// Access token is correct, acting as matching userID that is too long
+		{
+			name:      "correct access token but too long userID",
+			localpart: "_appservice_" + strings.Repeat("a", maxUsernameLength),
+			asToken:   fakeApplicationService.ASToken,
+			wantError: true,
+			wantASID:  "",
+		},
+		// Access token is correct, acting as userID that matches but is invalid
+		{
+			name:      "correct access token and matching but invalid userID",
+			localpart: "@_appservice_bob::",
+			asToken:   fakeApplicationService.ASToken,
+			wantError: true,
+			wantASID:  "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotASID, gotResp := ValidateApplicationServiceRequest(&fakeConfig.ClientAPI, tt.localpart, tt.asToken)
+			if tt.wantError && gotResp == nil {
+				t.Error("expected an error, but succeeded")
+			}
+			if !tt.wantError && gotResp != nil {
+				t.Errorf("expected success, but returned error: %v", *gotResp)
+			}
+			if gotASID != tt.wantASID {
+				t.Errorf("returned '%s', but expected '%s'", gotASID, tt.wantASID)
+			}
+		})
+	}
+}

internal/version.go

@@ -18,7 +18,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 13
-	VersionPatch = 4
+	VersionPatch = 7
 	VersionTag   = "" // example: "rc1"
 
 	gitRevLen = 7 // 7 matches the displayed characters on github.com

mediaapi/routing/download.go

@@ -63,6 +63,40 @@ type downloadRequest struct {
 	DownloadFilename   string
 }
 
+// Taken from: https://github.com/matrix-org/synapse/blob/c3627d0f99ed5a23479305dc2bd0e71ca25ce2b1/synapse/media/_base.py#L53C1-L84
+// A list of all content types that are "safe" to be rendered inline in a browser.
+var allowInlineTypes = map[types.ContentType]struct{}{
+	"text/css":            {},
+	"text/plain":          {},
+	"text/csv":            {},
+	"application/json":    {},
+	"application/ld+json": {},
+	// We allow some media files deemed as safe, which comes from the matrix-react-sdk.
+	// https://github.com/matrix-org/matrix-react-sdk/blob/a70fcfd0bcf7f8c85986da18001ea11597989a7c/src/utils/blobs.ts#L51
+	// SVGs are *intentionally* omitted.
+	"image/jpeg":      {},
+	"image/gif":       {},
+	"image/png":       {},
+	"image/apng":      {},
+	"image/webp":      {},
+	"image/avif":      {},
+	"video/mp4":       {},
+	"video/webm":      {},
+	"video/ogg":       {},
+	"video/quicktime": {},
+	"audio/mp4":       {},
+	"audio/webm":      {},
+	"audio/aac":       {},
+	"audio/mpeg":      {},
+	"audio/ogg":       {},
+	"audio/wave":      {},
+	"audio/wav":       {},
+	"audio/x-wav":     {},
+	"audio/x-pn-wav":  {},
+	"audio/flac":      {},
+	"audio/x-flac":    {},
+}
+
 // Download implements GET /download and GET /thumbnail
 // Files from this server (i.e. origin == cfg.ServerName) are served directly
 // Files from remote servers (i.e. origin != cfg.ServerName) are cached locally.
@@ -353,7 +387,7 @@ func (r *downloadRequest) addDownloadFilenameToHeaders(
 	}
 
 	if len(filename) == 0 {
-		w.Header().Set("Content-Disposition", "attachment")
+		w.Header().Set("Content-Disposition", contentDispositionFor(""))
 		return nil
 	}
 
@@ -383,20 +417,21 @@ func (r *downloadRequest) addDownloadFilenameToHeaders(
 	unescaped = strings.ReplaceAll(unescaped, `\`, `\\"`)
 	unescaped = strings.ReplaceAll(unescaped, `"`, `\"`)
 
+	disposition := contentDispositionFor(responseMetadata.ContentType)
 	if isASCII {
 		// For ASCII filenames, we should only quote the filename if
 		// it needs to be done, e.g. it contains a space or a character
 		// that would otherwise be parsed as a control character in the
 		// Content-Disposition header
 		w.Header().Set("Content-Disposition", fmt.Sprintf(
-			`attachment; filename=%s%s%s`,
+			`%s; filename=%s%s%s`,
-			quote, unescaped, quote,
+			disposition, quote, unescaped, quote,
 		))
 	} else {
 		// For UTF-8 filenames, we quote always, as that's the standard
 		w.Header().Set("Content-Disposition", fmt.Sprintf(
-			`attachment; filename*=utf-8''%s`,
+			`%s; filename*=utf-8''%s`,
-			url.QueryEscape(unescaped),
+			disposition, url.QueryEscape(unescaped),
 		))
 	}
 
@@ -808,3 +843,12 @@ func (r *downloadRequest) fetchRemoteFile(
 
 	return types.Path(finalPath), duplicate, nil
 }
+
+// contentDispositionFor returns the Content-Disposition for a given
+// content type.
+func contentDispositionFor(contentType types.ContentType) string {
+	if _, ok := allowInlineTypes[contentType]; ok {
+		return "inline"
+	}
+	return "attachment"
+}

mediaapi/routing/download_test.go

@@ -0,0 +1,13 @@
+package routing
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_dispositionFor(t *testing.T) {
+	assert.Equal(t, "attachment", contentDispositionFor(""), "empty content type")
+	assert.Equal(t, "attachment", contentDispositionFor("image/svg"), "image/svg")
+	assert.Equal(t, "inline", contentDispositionFor("image/jpeg"), "image/jpg")
+}

mediaapi/routing/routing.go

@@ -123,6 +123,7 @@ func makeDownloadAPI(
 
 		// Set internal headers returned regardless of the outcome of the request
 		util.SetCORSHeaders(w)
+		w.Header().Set("Cross-Origin-Resource-Policy", "cross-origin")
 		// Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors
 		w.Header().Set("Content-Type", "application/json")
 

relayapi/routing/routing.go

@@ -108,6 +108,8 @@ func MakeRelayAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
+			hub = hub.Clone()
 			hub.Scope().SetTag("origin", string(fedReq.Origin()))
 			hub.Scope().SetTag("uri", fedReq.RequestURI())
 		}

roomserver/acls/acls.go

@@ -23,49 +23,57 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 )
 
+const MRoomServerACL = "m.room.server_acl"
+
 type ServerACLDatabase interface {
-	// GetKnownRooms returns a list of all rooms we know about.
+	// RoomsWithACLs returns all room IDs for rooms with ACLs
-	GetKnownRooms(ctx context.Context) ([]string, error)
+	RoomsWithACLs(ctx context.Context) ([]string, error)
-	// GetStateEvent returns the state event of a given type for a given room with a given state key
+
-	// If no event could be found, returns nil
+	// GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match.
-	// If there was an issue during the retrieval, returns an error
+	// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
-	GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*types.HeaderedEvent, error)
+	GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
 }
 
 type ServerACLs struct {
 	acls               map[string]*serverACL      // room ID -> ACL
 	aclsMutex          sync.RWMutex               // protects the above
+	aclRegexCache      map[string]**regexp.Regexp // Cache from "serverName" -> pointer to a regex
+	aclRegexCacheMutex sync.RWMutex               // protects the above
 }
 
 func NewServerACLs(db ServerACLDatabase) *ServerACLs {
 	ctx := context.TODO()
 	acls := &ServerACLs{
 		acls: make(map[string]*serverACL),
+		// Be generous when creating the cache, as in reality
+		// there are hundreds of servers in an ACL.
+		aclRegexCache: make(map[string]**regexp.Regexp, 100),
 	}
+
 	// Look up all of the rooms that the current state server knows about.
-	rooms, err := db.GetKnownRooms(ctx)
+	rooms, err := db.RoomsWithACLs(ctx)
 	if err != nil {
 		logrus.WithError(err).Fatalf("Failed to get known rooms")
 	}
 	// For each room, let's see if we have a server ACL state event. If we
 	// do then we'll process it into memory so that we have the regexes to
 	// hand.
-	for _, room := range rooms {
+
-		state, err := db.GetStateEvent(ctx, room, "m.room.server_acl", "")
+	events, err := db.GetBulkStateContent(ctx, rooms, []gomatrixserverlib.StateKeyTuple{{EventType: MRoomServerACL, StateKey: ""}}, false)
 	if err != nil {
-			logrus.WithError(err).Errorf("Failed to get server ACLs for room %q", room)
+		logrus.WithError(err).Errorf("Failed to get server ACLs for all rooms: %q", err)
-			continue
-		}
-		if state != nil {
-			acls.OnServerACLUpdate(state.PDU)
 	}
+
+	for _, event := range events {
+		acls.OnServerACLUpdate(event)
 	}
+
 	return acls
 }
 
@@ -77,8 +85,8 @@ type ServerACL struct {
 
 type serverACL struct {
 	ServerACL
-	allowedRegexes []*regexp.Regexp
+	allowedRegexes []**regexp.Regexp
-	deniedRegexes  []*regexp.Regexp
+	deniedRegexes  []**regexp.Regexp
 }
 
 func compileACLRegex(orig string) (*regexp.Regexp, error) {
@@ -88,9 +96,28 @@ func compileACLRegex(orig string) (*regexp.Regexp, error) {
 	return regexp.Compile(escaped)
 }
 
-func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) {
+// cachedCompileACLRegex is a wrapper around compileACLRegex with added caching
+func (s *ServerACLs) cachedCompileACLRegex(orig string) (**regexp.Regexp, error) {
+	s.aclRegexCacheMutex.RLock()
+	re, ok := s.aclRegexCache[orig]
+	if ok {
+		s.aclRegexCacheMutex.RUnlock()
+		return re, nil
+	}
+	s.aclRegexCacheMutex.RUnlock()
+	compiled, err := compileACLRegex(orig)
+	if err != nil {
+		return nil, err
+	}
+	s.aclRegexCacheMutex.Lock()
+	defer s.aclRegexCacheMutex.Unlock()
+	s.aclRegexCache[orig] = &compiled
+	return &compiled, nil
+}
+
+func (s *ServerACLs) OnServerACLUpdate(strippedEvent tables.StrippedEvent) {
 	acls := &serverACL{}
-	if err := json.Unmarshal(state.Content(), &acls.ServerACL); err != nil {
+	if err := json.Unmarshal([]byte(strippedEvent.ContentValue), &acls.ServerACL); err != nil {
 		logrus.WithError(err).Errorf("Failed to unmarshal state content for server ACLs")
 		return
 	}
@@ -99,14 +126,14 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) {
 	// special characters and then replace * and ? with their regex counterparts.
 	// https://matrix.org/docs/spec/client_server/r0.6.1#m-room-server-acl
 	for _, orig := range acls.Allowed {
-		if expr, err := compileACLRegex(orig); err != nil {
+		if expr, err := s.cachedCompileACLRegex(orig); err != nil {
 			logrus.WithError(err).Errorf("Failed to compile allowed regex")
 		} else {
 			acls.allowedRegexes = append(acls.allowedRegexes, expr)
 		}
 	}
 	for _, orig := range acls.Denied {
-		if expr, err := compileACLRegex(orig); err != nil {
+		if expr, err := s.cachedCompileACLRegex(orig); err != nil {
 			logrus.WithError(err).Errorf("Failed to compile denied regex")
 		} else {
 			acls.deniedRegexes = append(acls.deniedRegexes, expr)
@@ -116,10 +143,15 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) {
 		"allow_ip_literals": acls.AllowIPLiterals,
 		"num_allowed":       len(acls.allowedRegexes),
 		"num_denied":        len(acls.deniedRegexes),
-	}).Debugf("Updating server ACLs for %q", state.RoomID())
+	}).Debugf("Updating server ACLs for %q", strippedEvent.RoomID)
+
+	// Clear out Denied and Allowed, now that we have the compiled regexes.
+	// They are not needed anymore from this point on.
+	acls.Denied = nil
+	acls.Allowed = nil
 	s.aclsMutex.Lock()
 	defer s.aclsMutex.Unlock()
-	s.acls[state.RoomID().String()] = acls
+	s.acls[strippedEvent.RoomID] = acls
 }
 
 func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID string) bool {
@@ -149,14 +181,14 @@ func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID s
 	// Check if the hostname matches one of the denied regexes. If it does then
 	// the server is banned from the room.
 	for _, expr := range acls.deniedRegexes {
-		if expr.MatchString(string(serverName)) {
+		if (*expr).MatchString(string(serverName)) {
 			return true
 		}
 	}
 	// Check if the hostname matches one of the allowed regexes. If it does then
 	// the server is NOT banned from the room.
 	for _, expr := range acls.allowedRegexes {
-		if expr.MatchString(string(serverName)) {
+		if (*expr).MatchString(string(serverName)) {
 			return false
 		}
 	}

roomserver/acls/acls_test.go

@@ -15,8 +15,14 @@
 package acls
 
 import (
+	"context"
 	"regexp"
 	"testing"
+
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestOpenACLsWithBlacklist(t *testing.T) {
@@ -38,8 +44,8 @@ func TestOpenACLsWithBlacklist(t *testing.T) {
 		ServerACL: ServerACL{
 			AllowIPLiterals: true,
 		},
-		allowedRegexes: []*regexp.Regexp{allowRegex},
+		allowedRegexes: []**regexp.Regexp{&allowRegex},
-		deniedRegexes:  []*regexp.Regexp{denyRegex},
+		deniedRegexes:  []**regexp.Regexp{&denyRegex},
 	}
 
 	if acls.IsServerBannedFromRoom("1.2.3.4", roomID) {
@@ -77,8 +83,8 @@ func TestDefaultACLsWithWhitelist(t *testing.T) {
 		ServerACL: ServerACL{
 			AllowIPLiterals: false,
 		},
-		allowedRegexes: []*regexp.Regexp{allowRegex},
+		allowedRegexes: []**regexp.Regexp{&allowRegex},
-		deniedRegexes:  []*regexp.Regexp{},
+		deniedRegexes:  []**regexp.Regexp{},
 	}
 
 	if !acls.IsServerBannedFromRoom("1.2.3.4", roomID) {
@@ -103,3 +109,45 @@ func TestDefaultACLsWithWhitelist(t *testing.T) {
 		t.Fatal("Expected qux.com:4567 to be allowed but wasn't")
 	}
 }
+
+var (
+	content1 = `{"allow":["*"],"allow_ip_literals":false,"deny":["hello.world", "*.hello.world"]}`
+)
+
+type dummyACLDB struct{}
+
+func (d dummyACLDB) RoomsWithACLs(ctx context.Context) ([]string, error) {
+	return []string{"1", "2"}, nil
+}
+
+func (d dummyACLDB) GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) {
+	return []tables.StrippedEvent{
+		{
+			RoomID:       "1",
+			ContentValue: content1,
+		},
+		{
+			RoomID:       "2",
+			ContentValue: content1,
+		},
+	}, nil
+}
+
+func TestCachedRegex(t *testing.T) {
+	db := dummyACLDB{}
+	wantBannedServer := spec.ServerName("hello.world")
+
+	acls := NewServerACLs(db)
+
+	// Check that hello.world is banned in room 1
+	banned := acls.IsServerBannedFromRoom(wantBannedServer, "1")
+	assert.True(t, banned)
+
+	// Check that hello.world is banned in room 2
+	banned = acls.IsServerBannedFromRoom(wantBannedServer, "2")
+	assert.True(t, banned)
+
+	// Check that matrix.hello.world is banned in room 2
+	banned = acls.IsServerBannedFromRoom("matrix."+wantBannedServer, "2")
+	assert.True(t, banned)
+}

roomserver/api/api.go

@@ -86,6 +86,9 @@ type RoomserverInternalAPI interface {
 		req *QueryAuthChainRequest,
 		res *QueryAuthChainResponse,
 	) error
+
+	// RoomsWithACLs returns all room IDs for rooms with ACLs
+	RoomsWithACLs(ctx context.Context) ([]string, error)
 }
 
 type UserRoomPrivateKeyCreator interface {
@@ -138,7 +141,12 @@ type QueryRoomHierarchyAPI interface {
 	//
 	// If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it
 	// can be cached.
-	QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *RoomHierarchyWalker, error)
+	QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) (
+		hierarchyRooms []fclient.RoomHierarchyRoom,
+		inaccessibleRooms []string,
+		hierarchyWalker *RoomHierarchyWalker,
+		err error,
+	)
 }
 
 type QueryMembershipAPI interface {
@@ -220,6 +228,7 @@ type ClientRoomserverAPI interface {
 	UserRoomPrivateKeyCreator
 	QueryRoomHierarchyAPI
 	DefaultRoomVersionAPI
+
 	QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error
 	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
 	QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error)
@@ -261,6 +270,15 @@ type ClientRoomserverAPI interface {
 	RemoveRoomAlias(ctx context.Context, senderID spec.SenderID, alias string) (aliasFound bool, aliasRemoved bool, err error)
 
 	SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error)
+
+	InsertReportedEvent(
+		ctx context.Context,
+		roomID, eventID, reportingUserID, reason string,
+		score int64,
+	) (int64, error)
+	QueryAdminEventReports(ctx context.Context, from, limit uint64, backwards bool, userID, roomID string) ([]QueryAdminEventReportsResponse, int64, error)
+	QueryAdminEventReport(ctx context.Context, reportID uint64) (QueryAdminEventReportResponse, error)
+	PerformAdminDeleteEventReport(ctx context.Context, reportID uint64) error
 }
 
 type UserRoomserverAPI interface {

roomserver/api/perform.go

@@ -8,7 +8,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
-	"github.com/matrix-org/util"
 )
 
 type PerformCreateRoomRequest struct {
@@ -54,8 +53,6 @@ type InviteInput struct {
 	RoomID     spec.RoomID
 	Inviter    spec.UserID
 	Invitee    spec.UserID
-	DisplayName string
-	AvatarURL   string
 	Reason     string
 	IsDirect   bool
 	KeyID      gomatrixserverlib.KeyID
@@ -91,14 +88,44 @@ type PerformBackfillRequest struct {
 	VirtualHost spec.ServerName `json:"virtual_host"`
 }
 
-// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order.
+// limitPrevEventIDs is the maximum of eventIDs we
+// return when calling PrevEventIDs.
+const limitPrevEventIDs = 100
+
+// PrevEventIDs returns the prev_event IDs of either 100 backwards extremities or
+// len(r.BackwardsExtremities). Limited to 100, due to Synapse/Dendrite stopping after reaching
+// this limit. (which sounds sane)
 func (r *PerformBackfillRequest) PrevEventIDs() []string {
-	var prevEventIDs []string
+	var uniqueIDs map[string]struct{}
-	for _, pes := range r.BackwardsExtremities {
+
-		prevEventIDs = append(prevEventIDs, pes...)
+	// Create a unique eventID map of either 100 or len(r.BackwardsExtremities).
+	// 100 since Synapse/Dendrite stops after reaching 100 events.
+	if len(r.BackwardsExtremities) > limitPrevEventIDs {
+		uniqueIDs = make(map[string]struct{}, limitPrevEventIDs)
+	} else {
+		uniqueIDs = make(map[string]struct{}, len(r.BackwardsExtremities))
 	}
-	prevEventIDs = util.UniqueStrings(prevEventIDs)
+
-	return prevEventIDs
+outerLoop:
+	for _, pes := range r.BackwardsExtremities {
+		for _, evID := range pes {
+			uniqueIDs[evID] = struct{}{}
+			// We found enough unique eventIDs.
+			if len(uniqueIDs) >= limitPrevEventIDs {
+				break outerLoop
+			}
+		}
+	}
+
+	// map -> []string
+	result := make([]string, len(uniqueIDs))
+	i := 0
+	for evID := range uniqueIDs {
+		result[i] = evID
+		i++
+	}
+
+	return result
 }
 
 // PerformBackfillResponse is a response to PerformBackfill.

roomserver/api/perform_test.go

@@ -0,0 +1,81 @@
+package api
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func BenchmarkPrevEventIDs(b *testing.B) {
+	for _, x := range []int64{1, 10, 100, 500, 1000, 2000} {
+		benchPrevEventIDs(b, int(x))
+	}
+}
+
+func benchPrevEventIDs(b *testing.B, count int) {
+	bwExtrems := generateBackwardsExtremities(b, count)
+	backfiller := PerformBackfillRequest{
+		BackwardsExtremities: bwExtrems,
+	}
+
+	b.Run(fmt.Sprintf("Original%d", count), func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			prevIDs := backfiller.PrevEventIDs()
+			_ = prevIDs
+		}
+	})
+}
+
+type testLike interface {
+	Helper()
+}
+
+const randomIDCharsCount = 10
+
+func generateBackwardsExtremities(t testLike, count int) map[string][]string {
+	t.Helper()
+	result := make(map[string][]string, count)
+	for i := 0; i < count; i++ {
+		eventID := randomEventId(int64(i))
+		result[eventID] = []string{
+			randomEventId(int64(i + 1)),
+			randomEventId(int64(i + 2)),
+			randomEventId(int64(i + 3)),
+		}
+	}
+	return result
+}
+
+const alphanumerics = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+// randomEventId generates a pseudo-random string of length n.
+func randomEventId(src int64) string {
+	randSrc := rand.NewSource(src)
+	b := make([]byte, randomIDCharsCount)
+	for i := range b {
+		b[i] = alphanumerics[randSrc.Int63()%int64(len(alphanumerics))]
+	}
+	return string(b)
+}
+
+func TestPrevEventIDs(t *testing.T) {
+	// generate 10 backwards extremities
+	bwExtrems := generateBackwardsExtremities(t, 10)
+	backfiller := PerformBackfillRequest{
+		BackwardsExtremities: bwExtrems,
+	}
+
+	prevIDs := backfiller.PrevEventIDs()
+	// Given how "generateBackwardsExtremities" works, this
+	// generates 12 unique event IDs
+	assert.Equal(t, 12, len(prevIDs))
+
+	// generate 200 backwards extremities
+	backfiller.BackwardsExtremities = generateBackwardsExtremities(t, 200)
+	prevIDs = backfiller.PrevEventIDs()
+	// PrevEventIDs returns at max 100 event IDs
+	assert.Equal(t, 100, len(prevIDs))
+}

roomserver/api/query.go

@@ -346,6 +346,28 @@ type QueryServerBannedFromRoomResponse struct {
 	Banned bool `json:"banned"`
 }
 
+type QueryAdminEventReportsResponse struct {
+	ID               int64                  `json:"id"`
+	Score            int64                  `json:"score"`
+	EventNID         types.EventNID         `json:"-"` // only used to query the state
+	RoomNID          types.RoomNID          `json:"-"` // only used to query the state
+	ReportingUserNID types.EventStateKeyNID `json:"-"` // only used in the DB
+	SenderNID        types.EventStateKeyNID `json:"-"` // only used in the DB
+	RoomID           string                 `json:"room_id"`
+	EventID          string                 `json:"event_id"`
+	UserID           string                 `json:"user_id"` // the user reporting the event
+	Reason           string                 `json:"reason"`
+	Sender           string                 `json:"sender"` // the user sending the reported event
+	CanonicalAlias   string                 `json:"canonical_alias"`
+	RoomName         string                 `json:"name"`
+	ReceivedTS       spec.Timestamp         `json:"received_ts"`
+}
+
+type QueryAdminEventReportResponse struct {
+	QueryAdminEventReportsResponse
+	EventJSON json.RawMessage `json:"event_json"`
+}
+
 // MarshalJSON stringifies the room ID and StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
 func (r *QueryBulkStateContentResponse) MarshalJSON() ([]byte, error) {
 	se := make(map[string]string)

roomserver/api/wrapper.go

@@ -189,7 +189,7 @@ func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkS
 			RoomID: roomID,
 		}
 		joinCount := 0
-		var joinRule, guestAccess string
+		var guestAccess string
 		for tuple, contentVal := range data {
 			if tuple.EventType == spec.MRoomMember && contentVal == "join" {
 				joinCount++
@@ -210,12 +210,12 @@ func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkS
 				pub.WorldReadable = contentVal == "world_readable"
 			// need both of these to determine whether guests can join
 			case joinRuleTuple:
-				joinRule = contentVal
+				pub.JoinRule = contentVal
 			case guestTuple:
 				guestAccess = contentVal
 			}
 		}
-		if joinRule == spec.Public && guestAccess == "can_join" {
+		if pub.JoinRule == spec.Public && guestAccess == "can_join" {
 			pub.GuestCanJoin = true
 		}
 		pub.JoinedMembersCount = joinCount

roomserver/internal/api.go

@@ -340,3 +340,11 @@ func (r *RoomserverInternalAPI) SigningIdentityFor(ctx context.Context, roomID s
 func (r *RoomserverInternalAPI) AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) {
 	return r.DB.AssignRoomNID(ctx, roomID, roomVersion)
 }
+
+func (r *RoomserverInternalAPI) InsertReportedEvent(
+	ctx context.Context,
+	roomID, eventID, reportingUserID, reason string,
+	score int64,
+) (int64, error) {
+	return r.DB.InsertReportedEvent(ctx, roomID, eventID, reportingUserID, reason, score)
+}

roomserver/internal/input/input.go

@@ -108,20 +108,27 @@ type worker struct {
 	r            *Inputer
 	roomID       string
 	subscription *nats.Subscription
+	sentryHub    *sentry.Hub
 }
 
 func (r *Inputer) startWorkerForRoom(roomID string) {
 	v, loaded := r.workers.LoadOrStore(roomID, &worker{
 		r:         r,
 		roomID:    roomID,
+		sentryHub: sentry.CurrentHub().Clone(),
 	})
 	w := v.(*worker)
 	w.Lock()
 	defer w.Unlock()
 	if !loaded || w.subscription == nil {
+		streamName := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent)
 		consumer := r.Cfg.Matrix.JetStream.Prefixed("RoomInput" + jetstream.Tokenise(w.roomID))
 		subject := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEventSubj(w.roomID))
 
+		logger := logrus.WithFields(logrus.Fields{
+			"stream_name": streamName,
+			"consumer":    consumer,
+		})
 		// Create the consumer. We do this as a specific step rather than
 		// letting PullSubscribe create it for us because we need the consumer
 		// to outlive the subscription. If we do it this way, we can Bind in the
@@ -135,20 +142,61 @@ func (r *Inputer) startWorkerForRoom(roomID string) {
 		// before it. This is necessary because otherwise our consumer will never
 		// acknowledge things we filtered out for other subjects and therefore they
 		// will linger around forever.
-		if _, err := w.r.JetStream.AddConsumer(
+
-			r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent),
+		info, err := w.r.JetStream.ConsumerInfo(streamName, consumer)
-			&nats.ConsumerConfig{
+		if err != nil && !errors.Is(err, nats.ErrConsumerNotFound) {
+			// log and return, we will retry anyway
+			logger.WithError(err).Errorf("failed to get consumer info")
+			return
+		}
+
+		consumerConfig := &nats.ConsumerConfig{
 			Durable:           consumer,
-				AckPolicy:         nats.AckAllPolicy,
+			AckPolicy:         nats.AckExplicitPolicy,
 			DeliverPolicy:     nats.DeliverAllPolicy,
 			FilterSubject:     subject,
 			AckWait:           MaximumMissingProcessingTime + (time.Second * 10),
 			InactiveThreshold: inactiveThreshold,
-			},
+		}
-		); err != nil {
+
-			logrus.WithError(err).Errorf("Failed to create consumer for room %q", w.roomID)
+		// The consumer already exists, try to update if necessary.
+		if info != nil {
+			// Not using reflect.DeepEqual here, since consumerConfig does not explicitly set
+			// e.g. the consumerName, which is added by NATS later. So this would result
+			// in constantly updating/recreating the consumer.
+			switch {
+			case info.Config.AckWait.Nanoseconds() != consumerConfig.AckWait.Nanoseconds():
+				// Initially we had a AckWait of 2m 10s, now we have 5m 10s, so we need to update
+				// existing consumers.
+				fallthrough
+			case info.Config.AckPolicy != consumerConfig.AckPolicy:
+				// We've changed the AckPolicy from AckAll to AckExplicit, this needs a
+				// recreation of the consumer. (Note: Only a few changes actually need a recreat)
+				logger.Warn("Consumer already exists, trying to update it.")
+				// Try updating the consumer first
+				if _, err = w.r.JetStream.UpdateConsumer(streamName, consumerConfig); err != nil {
+					// We failed to update the consumer, recreate it
+					logger.WithError(err).Warn("Unable to update consumer, recreating...")
+					if err = w.r.JetStream.DeleteConsumer(streamName, consumer); err != nil {
+						logger.WithError(err).Fatal("Unable to delete consumer")
 						return
 					}
+					// Set info to nil, so it can be recreated with the correct config.
+					info = nil
+				}
+			}
+		}
+
+		if info == nil {
+			// Create the consumer with the correct config
+			if _, err = w.r.JetStream.AddConsumer(
+				r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent),
+				consumerConfig,
+			); err != nil {
+				logger.WithError(err).Errorf("Failed to create consumer for room %q", w.roomID)
+				return
+			}
+		}
 
 		// Bind to our durable consumer. We want to receive all messages waiting
 		// for this subject and we want to manually acknowledge them, so that we
@@ -162,7 +210,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) {
 			nats.InactiveThreshold(inactiveThreshold),
 		)
 		if err != nil {
-			logrus.WithError(err).Errorf("Failed to subscribe to stream for room %q", w.roomID)
+			logger.WithError(err).Errorf("Failed to subscribe to stream for room %q", w.roomID)
 			return
 		}
 
@@ -219,9 +267,9 @@ func (w *worker) _next() {
 	// Look up what the next event is that's waiting to be processed.
 	ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute)
 	defer cancel()
-	if scope := sentry.CurrentHub().Scope(); scope != nil {
+	w.sentryHub.ConfigureScope(func(scope *sentry.Scope) {
 		scope.SetTag("room_id", w.roomID)
-	}
+	})
 	msgs, err := w.subscription.Fetch(1, nats.Context(ctx))
 	switch err {
 	case nil:
@@ -263,21 +311,23 @@ func (w *worker) _next() {
 		return
 	}
 
+	// Since we either Ack() or Term() the message at this point, we can defer decrementing the room backpressure
+	defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec()
+
 	// Try to unmarshal the input room event. If the JSON unmarshalling
 	// fails then we'll terminate the message — this notifies NATS that
 	// we are done with the message and never want to see it again.
 	msg := msgs[0]
 	var inputRoomEvent api.InputRoomEvent
 	if err = json.Unmarshal(msg.Data, &inputRoomEvent); err != nil {
-		_ = msg.Term()
+		// using AckWait here makes the call synchronous; 5 seconds is the default value used by NATS
+		_ = msg.Term(nats.AckWait(time.Second * 5))
 		return
 	}
 
-	if scope := sentry.CurrentHub().Scope(); scope != nil {
+	w.sentryHub.ConfigureScope(func(scope *sentry.Scope) {
 		scope.SetTag("event_id", inputRoomEvent.Event.EventID())
-	}
+	})
-	roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Inc()
-	defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec()
 
 	// Process the room event. If something goes wrong then we'll tell
 	// NATS to terminate the message. We'll store the error result as
@@ -299,7 +349,7 @@ func (w *worker) _next() {
 			}).Warn("Roomserver rejected event")
 		default:
 			if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
-				sentry.CaptureException(err)
+				w.sentryHub.CaptureException(err)
 			}
 			logrus.WithError(err).WithFields(logrus.Fields{
 				"room_id":  w.roomID,
@@ -307,10 +357,15 @@ func (w *worker) _next() {
 				"type":     inputRoomEvent.Event.Type(),
 			}).Warn("Roomserver failed to process event")
 		}
-		_ = msg.Term()
+		// Even though we failed to process this message (e.g. due to Dendrite restarting and receiving a context canceled),
+		// the message may already have been queued for redelivery or will be, so this makes sure that we still reprocess the msg
+		// after restarting. We only Ack if the context was not yet canceled.
+		if w.r.ProcessContext.Context().Err() == nil {
+			_ = msg.AckSync()
+		}
 		errString = err.Error()
 	} else {
-		_ = msg.Ack()
+		_ = msg.AckSync()
 	}
 
 	// If it was a synchronous input request then the "sync" field
@@ -381,6 +436,9 @@ func (r *Inputer) queueInputRoomEvents(
 			}).Error("Roomserver failed to queue async event")
 			return nil, fmt.Errorf("r.JetStream.PublishMsg: %w", err)
 		}
+
+		// Now that the event is queued, increment the room backpressure
+		roomserverInputBackpressure.With(prometheus.Labels{"room_id": roomID}).Inc()
 	}
 	return
 }

roomserver/internal/input/input_events.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -33,6 +34,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
+	"github.com/matrix-org/dendrite/roomserver/acls"
 	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 
 	userAPI "github.com/matrix-org/dendrite/userapi/api"
@@ -47,8 +49,10 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
 
-// TODO: Does this value make sense?
+// MaximumMissingProcessingTime is the maximum time we allow "processRoomEvent" to fetch
-const MaximumMissingProcessingTime = time.Minute * 2
+// e.g. missing auth/prev events. This duration is used for AckWait, and if it is exceeded
+// NATS queues the event for redelivery.
+const MaximumMissingProcessingTime = time.Minute * 5
 
 var processRoomEventDuration = prometheus.NewHistogramVec(
 	prometheus.HistogramOpts{
@@ -491,6 +495,33 @@ func (r *Inputer) processRoomEvent(
 		}
 	}
 
+	// If this is a membership event, it is possible we newly joined a federated room and eventually
+	// missed to update our m.room.server_acl - the following ensures we set the ACLs
+	// TODO: This probably performs badly in benchmarks
+	if event.Type() == spec.MRoomMember {
+		membership, _ := event.Membership()
+		if membership == spec.Join {
+			_, serverName, _ := gomatrixserverlib.SplitID('@', *event.StateKey())
+			// only handle local membership events
+			if r.Cfg.Matrix.IsLocalServerName(serverName) {
+				var aclEvent *types.HeaderedEvent
+				aclEvent, err = r.DB.GetStateEvent(ctx, event.RoomID().String(), acls.MRoomServerACL, "")
+				if err != nil {
+					logrus.WithError(err).Error("failed to get server ACLs")
+				}
+				if aclEvent != nil {
+					strippedEvent := tables.StrippedEvent{
+						RoomID:       aclEvent.RoomID().String(),
+						EventType:    aclEvent.Type(),
+						StateKey:     *aclEvent.StateKey(),
+						ContentValue: string(aclEvent.Content()),
+					}
+					r.ACLs.OnServerACLUpdate(strippedEvent)
+				}
+			}
+		}
+	}
+
 	// Handle remote room upgrades, e.g. remove published room
 	if event.Type() == "m.room.tombstone" && event.StateKeyEquals("") && !r.Cfg.Matrix.IsLocalServerName(senderDomain) {
 		if err = r.handleRemoteRoomUpgrade(ctx, event); err != nil {

roomserver/internal/input/input_latest_events.go

@@ -298,6 +298,7 @@ func (u *latestEventsUpdater) latestState() error {
 		}).Warnf("State reset detected (removing %d events)", removed)
 		sentry.WithScope(func(scope *sentry.Scope) {
 			scope.SetLevel("warning")
+			scope.SetTag("room_id", u.event.RoomID().String())
 			scope.SetContext("State reset", map[string]interface{}{
 				"Event ID":      u.event.EventID(),
 				"Old state NID": fmt.Sprintf("%d", u.oldStateNID),

roomserver/internal/perform/perform_admin.go

@@ -354,3 +354,7 @@ func (r *Admin) PerformAdminDownloadState(
 
 	return nil
 }
+
+func (r *Admin) PerformAdminDeleteEventReport(ctx context.Context, reportID uint64) error {
+	return r.DB.AdminDeleteEventReport(ctx, reportID)
+}

roomserver/internal/perform/perform_create_room.go

@@ -506,8 +506,6 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo
 					RoomID:     roomID,
 					Inviter:    userID,
 					Invitee:    *inviteeUserID,
-					DisplayName: createRequest.UserDisplayName,
-					AvatarURL:   createRequest.UserAvatarURL,
 					Reason:     "",
 					IsDirect:   createRequest.IsDirect,
 					KeyID:      createRequest.KeyID,

roomserver/internal/perform/perform_invite.go

@@ -145,8 +145,6 @@ func (r *Inviter) PerformInvite(
 
 	content := gomatrixserverlib.MemberContent{
 		Membership: spec.Invite,
-		DisplayName: req.InviteInput.DisplayName,
-		AvatarURL:   req.InviteInput.AvatarURL,
 		Reason:     req.InviteInput.Reason,
 		IsDirect:   req.InviteInput.IsDirect,
 	}

roomserver/internal/query/query.go

@@ -1099,3 +1099,18 @@ func (r *Queryer) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID,
 
 	return nil, nil
 }
+
+// RoomsWithACLs returns all room IDs for rooms with ACLs
+func (r *Queryer) RoomsWithACLs(ctx context.Context) ([]string, error) {
+	return r.DB.RoomsWithACLs(ctx)
+}
+
+// QueryAdminEventReports returns event reports given a filter.
+func (r *Queryer) QueryAdminEventReports(ctx context.Context, from uint64, limit uint64, backwards bool, userID, roomID string) ([]api.QueryAdminEventReportsResponse, int64, error) {
+	return r.DB.QueryAdminEventReports(ctx, from, limit, backwards, userID, roomID)
+}
+
+// QueryAdminEventReport returns a single event report.
+func (r *Queryer) QueryAdminEventReport(ctx context.Context, reportID uint64) (api.QueryAdminEventReportResponse, error) {
+	return r.DB.QueryAdminEventReport(ctx, reportID)
+}