Merge remote-tracking branch 'origin' into release/upstream-0.10.7

.cloudbuild/dev.yaml

@@ -0,0 +1,12 @@
+steps:
+  - name: gcr.io/cloud-builders/docker
+    args: ['build', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA', '-f', 'build/docker/Dockerfile.monolith', '.']
+  - name: gcr.io/cloud-builders/kubectl
+    args: ['-n', 'dendrite', 'set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA']
+    env:
+      - CLOUDSDK_CORE_PROJECT=globekeeper-development
+      - CLOUDSDK_COMPUTE_ZONE=europe-west2-a
+      - CLOUDSDK_CONTAINER_CLUSTER=synapse
+images:
+  - gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA
+timeout: 480s

.cloudbuild/prod.yaml

@@ -0,0 +1,12 @@
+steps:
+  - name: gcr.io/cloud-builders/docker
+    args: ['build', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME', '-f', 'build/docker/Dockerfile.monolith', '.']
+  - name: gcr.io/cloud-builders/kubectl
+    args: ['set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME']
+    env:
+      - CLOUDSDK_CORE_PROJECT=globekeeper-production
+      - CLOUDSDK_COMPUTE_ZONE=europe-west2-a
+      - CLOUDSDK_CONTAINER_CLUSTER=synapse-production
+images:
+  - gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME
+timeout: 480s

.github/workflows/dendrite.yml

@@ -14,51 +14,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  wasm:
-    name: WASM build test
-    timeout-minutes: 5
-    runs-on: ubuntu-latest
-    if: ${{ false }} # disable for now
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.18
-
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-wasm-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-wasm
-
-      - name: Install Node
-        uses: actions/setup-node@v2
-        with:
-          node-version: 14
-
-      - uses: actions/cache@v2
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
-
-      - name: Reconfigure Git to use HTTPS auth for repo packages
-        run: >
-          git config --global url."https://github.com/".insteadOf
-          ssh://git@github.com/
-
-      - name: Install test dependencies
-        working-directory: ./test/wasm
-        run: npm ci
-
-      - name: Test
-        run: ./test-dendritejs.sh
 
   # Run golangci-lint
   lint:
@@ -74,7 +29,7 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
 
-  # run go test with different go versions
+  # run go test with go 1.19
   test:
     timeout-minutes: 5
     name: Unit tests (Go ${{ matrix.go }})
@@ -102,7 +57,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.18", "1.19"]
+        go: ["1.19"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
@@ -129,7 +84,7 @@ jobs:
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: dendrite
 
-  # build Dendrite for linux with different architectures and go versions
+  # build Dendrite for linux amd64 with go 1.18
   build:
     name: Build for Linux
     timeout-minutes: 10
@@ -137,9 +92,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.18", "1.19"]
+        go: ["1.19"]
         goos: ["linux"]
-        goarch: ["amd64", "386"]
+        goarch: ["amd64"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
@@ -164,43 +119,10 @@ jobs:
           CGO_CFLAGS: -fno-stack-protector
         run: go build -trimpath -v -o "bin/" ./cmd/...
 
-  # build for Windows 64-bit
-  build_windows:
-    name: Build for Windows
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go: ["1.18", "1.19"]
-        goos: ["windows"]
-        goarch: ["amd64"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup Go ${{ matrix.go }}
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ matrix.go }}
-      - name: Install dependencies
-        run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}
-      - env:
-          GOOS: ${{ matrix.goos }}
-          GOARCH: ${{ matrix.goarch }}
-          CGO_ENABLED: 1
-          CC: "/usr/bin/x86_64-w64-mingw32-gcc"
-        run: go build -trimpath -v -o "bin/" ./cmd/...
-
   # Dummy step to gate other tests on without repeating the whole list
   initial-tests-done:
     name: Initial tests passed
-    needs: [lint, test, build, build_windows]
+    needs: [lint, test, build]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
     steps:
@@ -209,56 +131,6 @@ jobs:
         with:
           jobs: ${{ toJSON(needs) }}
 
-  # run database upgrade tests
-  upgrade_test:
-    name: Upgrade tests
-    timeout-minutes: 20
-    needs: initial-tests-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.18"
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-upgrade
-      - name: Build upgrade-tests
-        run: go build ./cmd/dendrite-upgrade-tests
-      - name: Test upgrade
-        run: ./dendrite-upgrade-tests --head .
-
-  # run database upgrade tests, skipping over one version
-  upgrade_test_direct:
-    name: Upgrade tests from HEAD-2
-    timeout-minutes: 20
-    needs: initial-tests-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.18"
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-upgrade
-      - name: Build upgrade-tests
-        run: go build ./cmd/dendrite-upgrade-tests
-      - name: Test upgrade
-        run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
-
   # run Sytest in different variations
   sytest:
     timeout-minutes: 20
@@ -269,18 +141,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - label: SQLite native
-
-          - label: SQLite Cgo
-            cgo: 1
-
-          - label: SQLite native, full HTTP APIs
-            api: full-http
-
-          - label: SQLite Cgo, full HTTP APIs
-            api: full-http
-            cgo: 1
-
           - label: PostgreSQL
             postgres: postgres
 
@@ -313,13 +173,12 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
           path: |
-            /logs/results.tap
+            /logs
-            /logs/**/*.log*
 
   # run Complement
   complement:
@@ -331,18 +190,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - label: SQLite native
-
-          - label: SQLite Cgo
-            cgo: 1
-
-          - label: SQLite native, full HTTP APIs
-            api: full-http
-
-          - label: SQLite Cgo, full HTTP APIs
-            api: full-http
-            cgo: 1
-
           - label: PostgreSQL
             postgres: Postgres
 
@@ -390,7 +237,7 @@ jobs:
               continue
             fi
 
-            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+            (wget -O - "https://github.com/globekeeper/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
           done
 
       # Build initial Dendrite image
@@ -416,8 +263,6 @@ jobs:
     needs:
       [
         initial-tests-done,
-        upgrade_test,
-        upgrade_test_direct,
         sytest,
         complement,
       ]
@@ -428,15 +273,3 @@ jobs:
         uses: re-actors/alls-green@release/v1
         with:
           jobs: ${{ toJSON(needs) }}
-
-  update-docker-images:
-    name: Update Docker images
-    permissions:
-      packages: write
-      contents: read
-      security-events: write   # To upload Trivy sarif files
-    if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
-    needs: [integration-tests-done]
-    uses: matrix-org/dendrite/.github/workflows/docker.yml@main
-    secrets:
-      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}

.github/workflows/docker.yml

@@ -1,313 +0,0 @@
-# Based on https://github.com/docker/build-push-action
-
-name: "Docker"
-
-on:
-  release: # A GitHub release was published
-    types: [published]
-  workflow_dispatch: # A build was manually requested
-  workflow_call: # Another pipeline called us
-    secrets:
-      DOCKER_TOKEN:
-        required: true
-
-env:
-  DOCKER_NAMESPACE: matrixdotorg
-  DOCKER_HUB_USER: dendritegithub
-  GHCR_NAMESPACE: matrix-org
-  PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
-
-jobs:
-  monolith:
-    name: Monolith image
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-      security-events: write # To upload Trivy sarif files
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
-        if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ env.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Login to GitHub Containers
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build main monolith image
-        if: github.ref_name == 'main'
-        id: docker_build_monolith
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: monolith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
-
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
-      - name: Build release monolith image
-        if: github.event_name == 'release' # Only for GitHub releases
-        id: docker_build_monolith_release
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: monolith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
-
-  polylith:
-    name: Polylith image
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-      security-events: write # To upload Trivy sarif files
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
-        if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ env.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Login to GitHub Containers
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build main polylith image
-        if: github.ref_name == 'main'
-        id: docker_build_polylith
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: polylith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
-      - name: Build release polylith image
-        if: github.event_name == 'release' # Only for GitHub releases
-        id: docker_build_polylith_release
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: polylith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
-
-  demo-pinecone:
-    name: Pinecone demo image
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
-        if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ env.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Login to GitHub Containers
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build main Pinecone demo image
-        if: github.ref_name == 'main'
-        id: docker_build_demo_pinecone
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          file: ./build/docker/Dockerfile.demo-pinecone
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
-
-      - name: Build release Pinecone demo image
-        if: github.event_name == 'release' # Only for GitHub releases
-        id: docker_build_demo_pinecone_release
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          file: ./build/docker/Dockerfile.demo-pinecone
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
-
-  demo-yggdrasil:
-    name: Yggdrasil demo image
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
-        if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ env.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Login to GitHub Containers
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build main Yggdrasil demo image
-        if: github.ref_name == 'main'
-        id: docker_build_demo_yggdrasil
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          file: ./build/docker/Dockerfile.demo-yggdrasil
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
-
-      - name: Build release Yggdrasil demo image
-        if: github.event_name == 'release' # Only for GitHub releases
-        id: docker_build_demo_yggdrasil_release
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          file: ./build/docker/Dockerfile.demo-yggdrasil
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}

.gitignore

@@ -2,6 +2,8 @@
 
 # Hidden files
 .*
+!.vscode
+!.cloudbuild
 
 # Allow GitHub config
 !.github
@@ -73,3 +75,7 @@ complement/
 docs/_site
 
 media_store/
+
+__debug_bin
+
+cmd/dendrite-monolith-server/dendrite-monolith-server

.golangci.yml

@@ -179,7 +179,6 @@ linters-settings:
 
 linters:
   enable:
-    - deadcode
     - errcheck
     - goconst
     - gocyclo
@@ -191,10 +190,8 @@ linters:
     - misspell # Check code comments, whereas misspell in CI checks *.md files
     - nakedret
     - staticcheck
-    - structcheck
     - unparam
     - unused
-    - varcheck
   enable-all: false
   disable:
     - bodyclose

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/dendrite-monolith-server",
+            "args": [
+                "-really-enable-open-registration",
+                "-config",
+                "../../../adminas/.ci/config/dendrite-local/dendrite.yaml"
+            ],
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,9 @@
+{
+    "go.lintTool": "golangci-lint",
+    "go.testEnvVars": {
+        "POSTGRES_HOST": "localhost",
+        "POSTGRES_USER": "postgres",
+        "POSTGRES_PASSWORD": "foobar",
+        "POSTGRES_DB": "postgres"
+    }
+}

build/dendritejs-pinecone/jsServer.go

@@ -34,13 +34,16 @@ type JSServer struct {
 
 // OnRequestFromJS is the function that JS will invoke when there is a new request.
 // The JS function signature is:
-//   function(reqString: string): Promise<{result: string, error: string}>
+//
+//	function(reqString: string): Promise<{result: string, error: string}>
+//
 // Usage is like:
-//   const res = await global._go_js_server.fetch(reqString);
+//
-//   if (res.error) {
+//	const res = await global._go_js_server.fetch(reqString);
-//     // handle error: this is a 'network' error, not a non-2xx error.
+//	if (res.error) {
-//   }
+//	  // handle error: this is a 'network' error, not a non-2xx error.
-//   const rawHttpResponse = res.result;
+//	}
+//	const rawHttpResponse = res.result;
 func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
 	// we HAVE to spawn a new goroutine and return immediately or else Go will deadlock
 	// if this request blocks at all e.g for /sync calls

clientapi/auth/authtypes/logintypes.go

@@ -11,4 +11,6 @@ const (
 	LoginTypeRecaptcha          = "m.login.recaptcha"
 	LoginTypeApplicationService = "m.login.application_service"
 	LoginTypeToken              = "m.login.token"
+	LoginTypeJwt                = "org.matrix.login.jwt"
+	LoginTypeEmail              = "m.login.email.identity"
 )

clientapi/auth/login.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/ratelimit"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
@@ -32,7 +33,7 @@ import (
 // called after authorization has completed, with the result of the authorization.
 // If the final return value is non-nil, an error occurred and the cleanup function
 // is nil.
-func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.ClientUserAPI, cfg *config.ClientAPI, rt *ratelimit.RtFailedLogin) (*Login, LoginCleanupFunc, *util.JSONResponse) {
 	reqBytes, err := io.ReadAll(r)
 	if err != nil {
 		err := &util.JSONResponse{
@@ -43,7 +44,8 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 	}
 
 	var header struct {
-		Type string `json:"type"`
+		Type          string `json:"type"`
+		InhibitDevice bool   `json:"inhibit_device"`
 	}
 	if err := json.Unmarshal(reqBytes, &header); err != nil {
 		err := &util.JSONResponse{
@@ -57,14 +59,20 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 	switch header.Type {
 	case authtypes.LoginTypePassword:
 		typ = &LoginTypePassword{
-			GetAccountByPassword: useraccountAPI.QueryAccountByPassword,
+			UserApi:       useraccountAPI,
-			Config:               cfg,
+			Config:        cfg,
+			Rt:            rt,
+			InhibitDevice: header.InhibitDevice,
 		}
 	case authtypes.LoginTypeToken:
 		typ = &LoginTypeToken{
-			UserAPI: userAPI,
+			UserAPI: useraccountAPI,
 			Config:  cfg,
 		}
+	case authtypes.LoginTypeJwt:
+		typ = &LoginTypeTokenJwt{
+			Config: cfg,
+		}
 	default:
 		err := util.JSONResponse{
 			Code: http.StatusBadRequest,

clientapi/auth/login_jwt.go

@@ -0,0 +1,74 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/golang-jwt/jwt/v4"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/util"
+)
+
+// LoginTypeToken describes how to authenticate with a login token.
+type LoginTypeTokenJwt struct {
+	// UserAPI uapi.LoginTokenInternalAPI
+	Config *config.ClientAPI
+}
+
+// Name implements Type.
+func (t *LoginTypeTokenJwt) Name() string {
+	return authtypes.LoginTypeJwt
+}
+
+type Claims struct {
+	jwt.StandardClaims
+}
+
+const mIdUser = "m.id.user"
+
+// LoginFromJSON implements Type. The cleanup function deletes the token from
+// the database on success.
+func (t *LoginTypeTokenJwt) LoginFromJSON(ctx context.Context, reqBytes []byte) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	var r loginTokenRequest
+	if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil {
+		return nil, nil, err
+	}
+
+	if r.Token == "" {
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Token field for JWT is missing"),
+		}
+	}
+	c := &Claims{}
+	token, err := jwt.ParseWithClaims(r.Token, c, func(token *jwt.Token) (interface{}, error) {
+		if _, ok := token.Method.(*jwt.SigningMethodEd25519); !ok {
+			return nil, fmt.Errorf("unexpected signing method: %v", token.Method.Alg())
+		}
+		return t.Config.JwtConfig.SecretKey, nil
+	})
+
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("jwt.ParseWithClaims failed")
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Couldn't parse JWT"),
+		}
+	}
+
+	if !token.Valid {
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Invalid JWT"),
+		}
+	}
+
+	r.Login.Identifier.User = c.Subject
+	r.Login.Identifier.Type = mIdUser
+
+	return &r.Login, func(context.Context, *util.JSONResponse) {}, nil
+}

clientapi/auth/login_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/ratelimit"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
@@ -68,8 +69,11 @@ func TestLoginFromJSONReader(t *testing.T) {
 				Matrix: &config.Global{
 					ServerName: serverName,
 				},
+				RtFailedLogin: ratelimit.RtFailedLoginConfig{
+					Enabled: false,
+				},
 			}
-			login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
+			login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, cfg, nil)
 			if err != nil {
 				t.Fatalf("LoginFromJSONReader failed: %+v", err)
 			}
@@ -147,7 +151,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 					ServerName: serverName,
 				},
 			}
-			_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
+			_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, cfg, nil)
 			if errRes == nil {
 				cleanup(ctx, nil)
 				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
@@ -159,6 +163,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 }
 
 type fakeUserInternalAPI struct {
+	uapi.ClientUserAPI
 	UserInternalAPIForLogin
 	DeletedTokens []string
 }

clientapi/auth/login_token.go

@@ -58,7 +58,7 @@ func (t *LoginTypeToken) LoginFromJSON(ctx context.Context, reqBytes []byte) (*L
 		}
 	}
 
-	r.Login.Identifier.Type = "m.id.user"
+	r.Login.Identifier.Type = mIdUser
 	r.Login.Identifier.User = res.Data.UserID
 
 	cleanup := func(ctx context.Context, authRes *util.JSONResponse) {

clientapi/auth/password.go

@@ -22,6 +22,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/ratelimit"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
@@ -33,12 +34,18 @@ type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPassw
 type PasswordRequest struct {
 	Login
 	Password string `json:"password"`
+	Address  string `json:"address"`
+	Medium   string `json:"medium"`
 }
 
+const email = "email"
+
 // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based
 type LoginTypePassword struct {
-	GetAccountByPassword GetAccountByPassword
+	UserApi       api.ClientUserAPI
-	Config               *config.ClientAPI
+	Config        *config.ClientAPI
+	Rt            *ratelimit.RtFailedLogin
+	InhibitDevice bool
 }
 
 func (t *LoginTypePassword) Name() string {
@@ -55,13 +62,42 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
 	if err != nil {
 		return nil, nil, err
 	}
+	login.InhibitDevice = t.InhibitDevice
 
 	return login, func(context.Context, *util.JSONResponse) {}, nil
 }
 
 func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
 	r := req.(*PasswordRequest)
-	username := strings.ToLower(r.Username())
+	if r.Identifier.Address != "" {
+		r.Address = r.Identifier.Address
+	}
+	if r.Identifier.Medium != "" {
+		r.Medium = r.Identifier.Medium
+	}
+	var username string
+	if r.Medium == email && r.Address != "" {
+		r.Address = strings.ToLower(r.Address)
+		res := api.QueryLocalpartForThreePIDResponse{}
+		err := t.UserApi.QueryLocalpartForThreePID(ctx, &api.QueryLocalpartForThreePIDRequest{
+			ThreePID: r.Address,
+			Medium:   email,
+		}, &res)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("userApi.QueryLocalpartForThreePID failed")
+			resp := jsonerror.InternalServerError()
+			return nil, &resp
+		}
+		username = res.Localpart
+		if username == "" {
+			return nil, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: jsonerror.Forbidden("Invalid username or password"),
+			}
+		}
+	} else {
+		username = strings.ToLower(r.Username())
+	}
 	if username == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
@@ -83,7 +119,17 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 	}
 	// Squash username to all lowercase letters
 	res := &api.QueryAccountByPasswordResponse{}
-	err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: strings.ToLower(localpart), PlaintextPassword: r.Password}, res)
+	localpart = strings.ToLower(localpart)
+	if t.Rt != nil {
+		ok, retryIn := t.Rt.CanAct(localpart)
+		if !ok {
+			return nil, &util.JSONResponse{
+				Code: http.StatusTooManyRequests,
+				JSON: jsonerror.LimitExceeded("Too Many Requests", retryIn.Milliseconds()),
+			}
+		}
+	}
+	err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: localpart, PlaintextPassword: r.Password}, res)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -92,7 +138,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 	}
 
 	if !res.Exists {
-		err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+		err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
 			Localpart:         localpart,
 			PlaintextPassword: r.Password,
 		}, res)
@@ -105,11 +151,15 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
 		// but that would leak the existence of the user.
 		if !res.Exists {
+			if t.Rt != nil {
+				t.Rt.Act(localpart)
+			}
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."),
+				JSON: jsonerror.Forbidden("Invalid username or password"),
 			}
 		}
 	}
+	r.Login.User = username
 	return &r.Login, nil
 }

clientapi/auth/user_interactive.go

@@ -66,6 +66,7 @@ type LoginIdentifier struct {
 type Login struct {
 	LoginIdentifier                 // Flat fields deprecated in favour of `identifier`.
 	Identifier      LoginIdentifier `json:"identifier"`
+	InhibitDevice   bool            `json:"inhibit_device,omitempty"`
 
 	// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
 	// Thus a pointer is needed to differentiate between the two
@@ -75,7 +76,7 @@ type Login struct {
 
 // Username returns the user localpart/user_id in this request, if it exists.
 func (r *Login) Username() string {
-	if r.Identifier.Type == "m.id.user" {
+	if r.Identifier.Type == mIdUser {
 		return r.Identifier.User
 	}
 	// deprecated but without it Element iOS won't log in
@@ -88,8 +89,8 @@ func (r *Login) ThirdPartyID() (medium, address string) {
 		return r.Identifier.Medium, r.Identifier.Address
 	}
 	// deprecated
-	if r.Medium == "email" {
+	if r.Medium == email {
-		return "email", r.Address
+		return email, r.Address
 	}
 	return "", ""
 }
@@ -111,10 +112,10 @@ type UserInteractive struct {
 	Sessions map[string][]string
 }
 
-func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
+func NewUserInteractive(userAccountAPI api.ClientUserAPI, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
-		GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
+		UserApi: userAccountAPI,
-		Config:               cfg,
+		Config:  cfg,
 	}
 	return &UserInteractive{
 		Flows: []userInteractiveFlow{

clientapi/auth/user_interactive_test.go

@@ -24,7 +24,9 @@ var (
 	}
 )
 
-type fakeAccountDatabase struct{}
+type fakeAccountDatabase struct {
+	api.ClientUserAPI
+}
 
 func (d *fakeAccountDatabase) PerformPasswordUpdate(ctx context.Context, req *api.PerformPasswordUpdateRequest, res *api.PerformPasswordUpdateResponse) error {
 	return nil

clientapi/clientapi.go

@@ -52,6 +52,7 @@ func AddPublicRoutes(
 		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
 		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
 		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		TopicMultiRoomCast:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputMultiRoomCast),
 		UserAPI:                userAPI,
 		ServerName:             cfg.Matrix.ServerName,
 	}

clientapi/producers/syncapi.go

@@ -36,6 +36,7 @@ type SyncAPIProducer struct {
 	TopicSendToDeviceEvent string
 	TopicTypingEvent       string
 	TopicPresenceEvent     string
+	TopicMultiRoomCast     string
 	JetStream              nats.JetStreamContext
 	ServerName             gomatrixserverlib.ServerName
 	UserAPI                userapi.ClientUserAPI
@@ -159,3 +160,14 @@ func (p *SyncAPIProducer) SendPresence(
 	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
 	return err
 }
+
+func (p *SyncAPIProducer) SendMultiroom(
+	ctx context.Context, userID string, dataType string, message []byte,
+) error {
+	m := nats.NewMsg(p.TopicMultiRoomCast)
+	m.Header.Set(jetstream.UserID, userID)
+	m.Header.Set("type", dataType)
+	m.Data = message
+	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}

clientapi/ratelimit/rt_failed_login.go

@@ -0,0 +1,117 @@
+package ratelimit
+
+import (
+	"container/list"
+	"sync"
+	"time"
+)
+
+type rateLimit struct {
+	cfg   *RtFailedLoginConfig
+	times *list.List
+}
+
+type RtFailedLogin struct {
+	cfg *RtFailedLoginConfig
+	mtx sync.RWMutex
+	rts map[string]*rateLimit
+}
+
+type RtFailedLoginConfig struct {
+	Enabled  bool          `yaml:"enabled"`
+	Limit    int           `yaml:"burst"`
+	Interval time.Duration `yaml:"interval"`
+}
+
+// New creates a new rate limiter for the limit and interval.
+func NewRtFailedLogin(cfg *RtFailedLoginConfig) *RtFailedLogin {
+	if !cfg.Enabled {
+		return nil
+	}
+	rt := &RtFailedLogin{
+		cfg: cfg,
+		mtx: sync.RWMutex{},
+		rts: make(map[string]*rateLimit),
+	}
+	go rt.clean()
+	return rt
+}
+
+// CanAct is expected to be called before Act
+func (r *RtFailedLogin) CanAct(key string) (ok bool, remaining time.Duration) {
+	r.mtx.RLock()
+	rt, ok := r.rts[key]
+	if !ok {
+		r.mtx.RUnlock()
+		return true, 0
+	}
+	ok, remaining = rt.canAct()
+	r.mtx.RUnlock()
+	return
+}
+
+// Act can be called after CanAct returns true.
+func (r *RtFailedLogin) Act(key string) {
+	r.mtx.Lock()
+	rt, ok := r.rts[key]
+	if !ok {
+		rt = &rateLimit{
+			cfg:   r.cfg,
+			times: list.New(),
+		}
+		r.rts[key] = rt
+	}
+	rt.act()
+	r.mtx.Unlock()
+}
+
+func (r *RtFailedLogin) clean() {
+	for {
+		r.mtx.Lock()
+		for k, v := range r.rts {
+			if v.empty() {
+				delete(r.rts, k)
+			}
+		}
+		r.mtx.Unlock()
+		time.Sleep(time.Hour)
+	}
+}
+
+func (r *rateLimit) empty() bool {
+	back := r.times.Back()
+	if back == nil {
+		return true
+	}
+	v := back.Value
+	b := v.(time.Time)
+	now := time.Now()
+	return now.Sub(b) > r.cfg.Interval
+}
+
+func (r *rateLimit) canAct() (ok bool, remaining time.Duration) {
+	now := time.Now()
+	l := r.times.Len()
+	if l < r.cfg.Limit {
+		return true, 0
+	}
+	frnt := r.times.Front()
+	t := frnt.Value.(time.Time)
+	diff := now.Sub(t)
+	if diff < r.cfg.Interval {
+		return false, r.cfg.Interval - diff
+	}
+	return true, 0
+}
+
+func (r *rateLimit) act() {
+	now := time.Now()
+	l := r.times.Len()
+	if l < r.cfg.Limit {
+		r.times.PushBack(now)
+		return
+	}
+	frnt := r.times.Front()
+	frnt.Value = now
+	r.times.MoveToBack(frnt)
+}

clientapi/ratelimit/rt_failed_login_test.go

@@ -0,0 +1,40 @@
+package ratelimit
+
+import (
+	"testing"
+	"time"
+
+	"github.com/matryer/is"
+)
+
+func TestRtFailedLogin(t *testing.T) {
+	is := is.New(t)
+	rtfl := NewRtFailedLogin(&RtFailedLoginConfig{
+		Enabled:  true,
+		Limit:    3,
+		Interval: 10 * time.Millisecond,
+	})
+	var (
+		can        bool
+		remaining  time.Duration
+		remainingB time.Duration
+	)
+	for i := 0; i < 3; i++ {
+		can, remaining = rtfl.CanAct("foo")
+		is.True(can)
+		is.Equal(remaining, time.Duration(0))
+		rtfl.Act("foo")
+	}
+	can, remaining = rtfl.CanAct("foo")
+	is.True(!can)
+	is.True(remaining > time.Millisecond*9)
+	can, remainingB = rtfl.CanAct("bar")
+	is.True(can)
+	is.Equal(remainingB, time.Duration(0))
+	rtfl.Act("bar")
+	rtfl.Act("bar")
+	time.Sleep(remaining + time.Millisecond)
+	can, remaining = rtfl.CanAct("foo")
+	is.True(can)
+	is.Equal(remaining, time.Duration(0))
+}

clientapi/routing/deactivate.go

@@ -27,13 +27,17 @@ func Deactivate(
 			JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
 		}
 	}
-
+	var userId string
-	login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI)
+	if deviceAPI.AccountType != api.AccountTypeAppService {
-	if errRes != nil {
+		login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI)
-		return *errRes
+		if errRes != nil {
+			return *errRes
+		}
+		userId = login.Username()
+	} else {
+		userId = deviceAPI.UserID
 	}
-
+	localpart, _, err := gomatrixserverlib.SplitID('@', userId)
-	localpart, _, err := gomatrixserverlib.SplitID('@', login.Username())
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
 		return jsonerror.InternalServerError()

clientapi/routing/key_crosssigning.go

@@ -63,8 +63,8 @@ func UploadCrossSigningDeviceKeys(
 		}
 	}
 	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: accountAPI.QueryAccountByPassword,
+		UserApi: accountAPI,
-		Config:               cfg,
+		Config:  cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {
 		return *authErr

clientapi/routing/login.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/ratelimit"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -55,6 +56,7 @@ func passwordLogin() flows {
 func Login(
 	req *http.Request, userAPI userapi.ClientUserAPI,
 	cfg *config.ClientAPI,
+	rt *ratelimit.RtFailedLogin,
 ) util.JSONResponse {
 	if req.Method == http.MethodGet {
 		// TODO: support other forms of login other than password, depending on config options
@@ -63,10 +65,21 @@ func Login(
 			JSON: passwordLogin(),
 		}
 	} else if req.Method == http.MethodPost {
-		login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg)
+		login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, cfg, rt)
 		if authErr != nil {
 			return *authErr
 		}
+		if login.InhibitDevice {
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: loginResponse{
+					UserID:      userutil.MakeUserID(login.Username(), cfg.Matrix.ServerName),
+					AccessToken: "",
+					HomeServer:  cfg.Matrix.ServerName,
+					DeviceID:    "",
+				},
+			}
+		}
 		// make a device/access token
 		authErr2 := completeAuth(req.Context(), cfg.Matrix, userAPI, login, req.RemoteAddr, req.UserAgent())
 		cleanup(req.Context(), &authErr2)

clientapi/routing/multiroom.go

@@ -0,0 +1,48 @@
+package routing
+
+import (
+	"io"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	log "github.com/sirupsen/logrus"
+)
+
+func PostMultiroom(
+	req *http.Request,
+	device *api.Device,
+	producer *producers.SyncAPIProducer,
+	dataType string,
+) util.JSONResponse {
+	b, err := io.ReadAll(req.Body)
+	if err != nil {
+		log.WithError(err).Errorf("failed to read request body")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+	canonicalB, err := gomatrixserverlib.CanonicalJSON(b)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The request body is not valid canonical JSON." + err.Error()),
+		}
+	}
+	err = producer.SendMultiroom(req.Context(), device.UserID, dataType, canonicalB)
+	if err != nil {
+		log.WithError(err).Errorf("failed to send multiroomcast")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/password.go

@@ -1,12 +1,14 @@
 package routing
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -24,6 +26,7 @@ type newPasswordAuth struct {
 	Type    string `json:"type"`
 	Session string `json:"session"`
 	auth.PasswordRequest
+	ThreePidCreds threepid.Credentials `json:"threepid_creds"`
 }
 
 func Password(
@@ -33,13 +36,17 @@ func Password(
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	// Check that the existing password is right.
+	var fields logrus.Fields
+	if device != nil {
+		fields = logrus.Fields{
+			"sessionId": device.SessionID,
+			"userId":    device.UserID,
+		}
+	}
 	var r newPasswordRequest
 	r.LogoutDevices = true
 
-	logrus.WithFields(logrus.Fields{
+	logrus.WithFields(fields).Debug("Changing password")
-		"sessionId": device.SessionID,
-		"userId":    device.UserID,
-	}).Debug("Changing password")
 
 	// Unmarshal the request.
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
@@ -53,45 +60,95 @@ func Password(
 		// Generate a new, random session ID
 		sessionID = util.RandomString(sessionIDLength)
 	}
-
+	var localpart string
-	// Require password auth to change the password.
+	switch r.Auth.Type {
-	if r.Auth.Type != authtypes.LoginTypePassword {
+	case authtypes.LoginTypePassword:
-		return util.JSONResponse{
+		// Check if the existing password is correct.
-			Code: http.StatusUnauthorized,
+		typePassword := auth.LoginTypePassword{
-			JSON: newUserInteractiveResponse(
+			UserApi: userAPI,
-				sessionID,
+			Config:  cfg,
-				[]authtypes.Flow{
+		}
-					{
+		if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {
-						Stages: []authtypes.LoginType{authtypes.LoginTypePassword},
+			return *authErr
-					},
+		}
+		// Get the local part.
+		var err error
+		localpart, _, err = gomatrixserverlib.SplitID('@', device.UserID)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
+			return jsonerror.InternalServerError()
+		}
+		sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
+	case authtypes.LoginTypeEmail:
+		threePid := &authtypes.ThreePID{}
+		r.Auth.ThreePidCreds.IDServer = cfg.ThreePidDelegate
+		var (
+			bound bool
+			err   error
+		)
+		bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
+			return jsonerror.InternalServerError()
+		}
+		if !bound {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.MatrixError{
+					ErrCode: "M_THREEPID_AUTH_FAILED",
+					Err:     "Failed to auth 3pid",
 				},
-				nil,
+			}
-			),
+		}
+		var res api.QueryLocalpartForThreePIDResponse
+		err = userAPI.QueryLocalpartForThreePID(req.Context(), &api.QueryLocalpartForThreePIDRequest{
+			Medium:   threePid.Medium,
+			ThreePID: threePid.Address,
+		}, &res)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryLocalpartForThreePID failed")
+			return jsonerror.InternalServerError()
+		}
+		if res.Localpart == "" {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.MatrixError{
+					ErrCode: "M_THREEPID_NOT_FOUND",
+					Err:     "3pid is not bound to any account",
+				},
+			}
+		}
+		localpart = res.Localpart
+		sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeEmail)
+	default:
+		flows := []authtypes.Flow{
+			{
+				Stages: []authtypes.LoginType{authtypes.LoginTypePassword},
+			},
+		}
+		if cfg.ThreePidDelegate != "" {
+			flows = append(flows, authtypes.Flow{
+				Stages: []authtypes.LoginType{authtypes.LoginTypeEmail},
+			})
+		}
+		// Require password auth to change the password.
+		if r.Auth.Type == authtypes.LoginTypePassword {
+			return util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: newUserInteractiveResponse(
+					sessionID,
+					flows,
+					nil,
+				),
+			}
 		}
 	}
 
-	// Check if the existing password is correct.
-	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: userAPI.QueryAccountByPassword,
-		Config:               cfg,
-	}
-	if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {
-		return *authErr
-	}
-	sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
-
 	// Check the new password strength.
 	if resErr = validatePassword(r.NewPassword); resErr != nil {
 		return *resErr
 	}
 
-	// Get the local part.
-	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
-	}
-
 	// Ask the user API to perform the password change.
 	passwordReq := &api.PerformPasswordUpdateRequest{
 		Localpart: localpart,
@@ -109,11 +166,23 @@ func Password(
 
 	// If the request asks us to log out all other devices then
 	// ask the user API to do that.
+
 	if r.LogoutDevices {
-		logoutReq := &api.PerformDeviceDeletionRequest{
+		var logoutReq *api.PerformDeviceDeletionRequest
-			UserID:         device.UserID,
+		var sessionId int64
-			DeviceIDs:      nil,
+		if device == nil {
-			ExceptDeviceID: device.ID,
+			logoutReq = &api.PerformDeviceDeletionRequest{
+				UserID:    fmt.Sprintf("@%s:%s", localpart, cfg.Matrix.ServerName),
+				DeviceIDs: []string{},
+			}
+			sessionId = 0
+		} else {
+			logoutReq = &api.PerformDeviceDeletionRequest{
+				UserID:         device.UserID,
+				DeviceIDs:      nil,
+				ExceptDeviceID: device.ID,
+			}
+			sessionId = device.SessionID
 		}
 		logoutRes := &api.PerformDeviceDeletionResponse{}
 		if err := userAPI.PerformDeviceDeletion(req.Context(), logoutReq, logoutRes); err != nil {
@@ -123,7 +192,7 @@ func Password(
 
 		pushersReq := &api.PerformPusherDeletionRequest{
 			Localpart: localpart,
-			SessionID: device.SessionID,
+			SessionID: sessionId,
 		}
 		if err := userAPI.PerformPusherDeletion(req.Context(), pushersReq, &struct{}{}); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("PerformPusherDeletion failed")

clientapi/routing/profile.go

@@ -106,12 +106,6 @@ func SetAvatarURL(
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
-	if r.AvatarURL == "" {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("'avatar_url' must be supplied."),
-		}
-	}
 
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {

clientapi/routing/pusher.go

@@ -86,8 +86,8 @@ func SetPusher(
 			if err != nil {
 				return invalidParam("malformed url passed")
 			}
-			if pushUrl.Scheme != "https" {
+			if pushUrl.Scheme != "https" && pushUrl.Scheme != "http" {
-				return invalidParam("only https scheme is allowed")
+				return invalidParam("only https and http schemes are allowed")
 			}
 		}
 

clientapi/routing/register.go

@@ -45,6 +45,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
@@ -238,6 +239,7 @@ type authDict struct {
 	// Recaptcha
 	Response string `json:"response"`
 	// TODO: Lots of custom keys depending on the type
+	ThreePidCreds threepid.Credentials `json:"threepid_creds"`
 }
 
 // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
@@ -747,6 +749,7 @@ func handleRegistrationFlow(
 		}
 	}
 
+	var threePid *authtypes.ThreePID
 	switch r.Auth.Type {
 	case authtypes.LoginTypeRecaptcha:
 		// Check given captcha response
@@ -763,6 +766,29 @@ func handleRegistrationFlow(
 		// Add Dummy to the list of completed registration stages
 		sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeDummy)
 
+	case authtypes.LoginTypeEmail:
+		threePid = &authtypes.ThreePID{}
+		r.Auth.ThreePidCreds.IDServer = cfg.ThreePidDelegate
+		var (
+			bound bool
+			err   error
+		)
+		bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
+			return jsonerror.InternalServerError()
+		}
+		if !bound {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.MatrixError{
+					ErrCode: "M_THREEPID_AUTH_FAILED",
+					Err:     "Failed to auth 3pid",
+				},
+			}
+		}
+		sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeEmail)
+
 	case "":
 		// An empty auth type means that we want to fetch the available
 		// flows. It can also mean that we want to register as an appservice
@@ -778,7 +804,7 @@ func handleRegistrationFlow(
 	// A response with current registration flow and remaining available methods
 	// will be returned if a flow has not been successfully completed yet
 	return checkAndCompleteFlow(sessions.getCompletedStages(sessionID),
-		req, r, sessionID, cfg, userAPI)
+		req, r, sessionID, cfg, userAPI, threePid)
 }
 
 // handleApplicationServiceRegistration handles the registration of an
@@ -820,7 +846,7 @@ func handleApplicationServiceRegistration(
 	// application service registration is entirely separate.
 	return completeRegistration(
 		req.Context(), userAPI, r.Username, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session,
-		r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService,
+		r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService, nil,
 	)
 }
 
@@ -834,12 +860,13 @@ func checkAndCompleteFlow(
 	sessionID string,
 	cfg *config.ClientAPI,
 	userAPI userapi.ClientUserAPI,
+	threePid *authtypes.ThreePID,
 ) util.JSONResponse {
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
 		return completeRegistration(
 			req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID,
-			r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser,
+			r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser, threePid,
 		)
 	}
 	sessions.addParams(sessionID, r)
@@ -865,6 +892,7 @@ func completeRegistration(
 	inhibitLogin eventutil.WeakBoolean,
 	displayName, deviceID *string,
 	accType userapi.AccountType,
+	threePid *authtypes.ThreePID,
 ) util.JSONResponse {
 	if username == "" {
 		return util.JSONResponse{
@@ -903,6 +931,21 @@ func completeRegistration(
 	// Increment prometheus counter for created users
 	amtRegUsers.Inc()
 
+	// TODO-entry refuse register if threepid is already bound to account.
+	if threePid != nil {
+		err = userAPI.PerformSaveThreePIDAssociation(ctx, &userapi.PerformSaveThreePIDAssociationRequest{
+			Medium:    threePid.Medium,
+			ThreePID:  threePid.Address,
+			Localpart: accRes.Account.Localpart,
+		}, &struct{}{})
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: jsonerror.Unknown("Failed to save 3PID association: " + err.Error()),
+			}
+		}
+	}
+
 	// Check whether inhibit_login option is set. If so, don't create an access
 	// token or a device for this user
 	if inhibitLogin {
@@ -1094,5 +1137,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
 	if ssrr.Admin {
 		accType = userapi.AccountTypeAdmin
 	}
-	return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
+	return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType, nil)
 }

clientapi/routing/routing.go

@@ -32,6 +32,7 @@ import (
 	clientutil "github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/clientapi/ratelimit"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/transactions"
@@ -66,6 +67,7 @@ func Setup(
 	prometheus.MustRegister(amtRegUsers, sendEventDuration)
 
 	rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
+	rateLimitsFailedLogin := ratelimit.NewRtFailedLogin(&cfg.RtFailedLogin)
 	userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg)
 
 	unstableFeatures := map[string]bool{
@@ -434,6 +436,17 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
+	v3mux.Handle("/multiroom/{dataType}",
+		httputil.MakeAuthAPI("send_multiroom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			dataType := vars["dataType"]
+			return PostMultiroom(req, device, syncProducer, dataType)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
 		if r := rateLimits.Limit(req, nil); r != nil {
 			return *r
@@ -602,7 +615,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	v3mux.Handle("/account/password",
-		httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		httputil.MakeConditionalAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
@@ -626,7 +639,7 @@ func Setup(
 			if r := rateLimits.Limit(req, nil); r != nil {
 				return *r
 			}
-			return Login(req, userAPI, cfg)
+			return Login(req, userAPI, cfg, rateLimitsFailedLogin)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 

clientapi/routing/state.go

@@ -101,7 +101,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		}
 		// If the user has never been in the room then stop at this point.
 		// We won't tell the user about a room they have never joined.
-		if !membershipRes.HasBeenInRoom {
+		if !membershipRes.HasBeenInRoom && membershipRes.Membership != gomatrixserverlib.Invite {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
@@ -241,7 +241,7 @@ func OnIncomingStateTypeRequest(
 		}
 		// If the user has never been in the room then stop at this point.
 		// We won't tell the user about a room they have never joined.
-		if !membershipRes.HasBeenInRoom || membershipRes.Membership == gomatrixserverlib.Ban {
+		if !membershipRes.HasBeenInRoom && membershipRes.Membership != gomatrixserverlib.Invite || membershipRes.Membership == gomatrixserverlib.Ban {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),

clientapi/threepid/threepid.go

@@ -103,11 +103,8 @@ func CreateSession(
 func CheckAssociation(
 	ctx context.Context, creds Credentials, cfg *config.ClientAPI,
 ) (bool, string, string, error) {
-	if err := isTrusted(creds.IDServer, cfg); err != nil {
-		return false, "", "", err
-	}
 
-	requestURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", creds.IDServer, creds.SID, creds.Secret)
+	requestURL := fmt.Sprintf("%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", cfg.ThreePidDelegate, creds.SID, creds.Secret)
 	req, err := http.NewRequest(http.MethodGet, requestURL, nil)
 	if err != nil {
 		return false, "", "", err

cmd/dendrite-monolith-server/Dockerfile.dev

@@ -0,0 +1,8 @@
+FROM alpine:latest
+
+COPY dendrite-monolith-server /usr/bin/
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]

cmd/dendrite-monolith-server/build_dev.sh

@@ -0,0 +1,12 @@
+set -xe
+if [ -z "$(git status --porcelain)" ]; then 
+    CGO_ENABLED=0 go build .
+    TAG=$(git rev-parse --short HEAD)
+    docker build -f Dockerfile.dev -t gcr.io/globekeeper-development/dendrite-monolith:$TAG -t gcr.io/globekeeper-development/dendrite-monolith  -t gcr.io/globekeeper-production/dendrite-monolith:$TAG .
+    docker push gcr.io/globekeeper-development/dendrite-monolith:$TAG
+    docker push gcr.io/globekeeper-production/dendrite-monolith:$TAG
+    docker push gcr.io/globekeeper-development/dendrite-monolith
+else 
+    echo "Please commit changes"
+    exit 0
+fi

cmd/dendrite-monolith-server/main.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"flag"
+	"log"
 	"os"
 
 	"github.com/matrix-org/dendrite/appservice"
@@ -45,6 +46,16 @@ var (
 func main() {
 	cfg := setup.ParseFlags(true)
 	httpAddr := config.HTTPAddress("http://" + *httpBindAddr)
+	for _, logging := range cfg.Logging {
+		if logging.Type == "std" {
+			level, err := logrus.ParseLevel(logging.Level)
+			if err != nil {
+				log.Fatal(err)
+			}
+			logrus.SetLevel(level)
+			logrus.SetFormatter(&logrus.JSONFormatter{})
+		}
+	}
 	httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr)
 	httpAPIAddr := httpAddr
 	options := []basepkg.BaseDendriteOptions{}

docs/installation/10_optimisation.md

@@ -0,0 +1,71 @@
+---
+title: Optimise your installation
+parent: Installation
+has_toc: true
+nav_order: 10
+permalink: /installation/start/optimisation
+---
+
+# Optimise your installation
+
+Now that you have Dendrite running, the following tweaks will improve the reliability
+and performance of your installation.
+
+## File descriptor limit
+
+Most platforms have a limit on how many file descriptors a single process can open. All
+connections made by Dendrite consume file descriptors — this includes database connections
+and network requests to remote homeservers. When participating in large federated rooms
+where Dendrite must talk to many remote servers, it is often very easy to exhaust default
+limits which are quite low.
+
+We currently recommend setting the file descriptor limit to 65535 to avoid such
+issues. Dendrite will log immediately after startup if the file descriptor limit is too low:
+
+```
+level=warning msg="IMPORTANT: Process file descriptor limit is currently 1024, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues"
+```
+
+UNIX systems have two limits: a hard limit and a soft limit. You can view the soft limit
+by running `ulimit -Sn` and the hard limit with `ulimit -Hn`:
+
+```bash
+$ ulimit -Hn
+1048576
+
+$ ulimit -Sn
+1024
+```
+
+Increase the soft limit before starting Dendrite:
+
+```bash
+ulimit -Sn 65535
+```
+
+The log line at startup should no longer appear if the limit is sufficient.
+
+If you are running under a systemd service, you can instead add `LimitNOFILE=65535` option
+to the `[Service]` section of your service unit file.
+
+## DNS caching
+
+Dendrite has a built-in DNS cache which significantly reduces the load that Dendrite will
+place on your DNS resolver. This may also speed up outbound federation.
+
+Consider enabling the DNS cache by modifying the `global` section of your configuration file:
+
+```yaml
+  dns_cache:
+    enabled: true
+    cache_size: 4096
+    cache_lifetime: 600s
+```
+
+## Time synchronisation
+
+Matrix relies heavily on TLS which requires the system time to be correct. If the clock
+drifts then you may find that federation no works reliably (or at all) and clients may
+struggle to connect to your Dendrite server.
+
+Ensure that the time is synchronised on your system by enabling NTP sync.

go.mod

@@ -4,7 +4,6 @@ require (
 	github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2
 	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
 	github.com/DATA-DOG/go-sqlmock v1.5.0
-	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/blevesearch/bleve/v2 v2.3.4
 	github.com/codeclysm/extract v2.2.0+incompatible
@@ -12,6 +11,7 @@ require (
 	github.com/docker/docker v20.10.19+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.14.0
+	github.com/golang-jwt/jwt/v4 v4.4.1
 	github.com/gologme/log v1.3.0
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
@@ -55,6 +55,8 @@ require (
 	nhooyr.io/websocket v1.8.7
 )
 
+require github.com/matryer/is v1.4.0
+
 require (
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
@@ -82,7 +84,7 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/gogo/protobuf v1.1.1 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/mock v1.6.0 // indirect

go.sum

@@ -43,8 +43,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
 github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
-github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc=
-github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
@@ -202,9 +200,10 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
 github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
+github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
 github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
@@ -316,7 +315,6 @@ github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5l
 github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
@@ -354,6 +352,8 @@ github.com/matrix-org/pinecone v0.0.0-20221103125849-37f2e9b9ba37 h1:CQWFrgH9TJO
 github.com/matrix-org/pinecone v0.0.0-20221103125849-37f2e9b9ba37/go.mod h1:F3GHppRuHCTDeoOmmgjZMeJdbql91+RSGGsATWfC7oc=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
+github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
+github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -757,11 +757,9 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

internal/httputil/httpapi.go

@@ -87,6 +87,57 @@ func MakeAuthAPI(
 	return MakeExternalAPI(metricsName, h)
 }
 
+// MakeConditionalAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request.
+// It passes nil device if header is not provided.
+func MakeConditionalAuthAPI(
+	metricsName string, userAPI userapi.QueryAcccessTokenAPI,
+	f func(*http.Request, *userapi.Device) util.JSONResponse,
+) http.Handler {
+	h := func(req *http.Request) util.JSONResponse {
+		var (
+			jsonRes util.JSONResponse
+			dev     *userapi.Device
+		)
+		if _, err := auth.ExtractAccessToken(req); err != nil {
+			dev = nil
+		} else {
+			logger := util.GetLogger(req.Context())
+			var err *util.JSONResponse
+			dev, err = auth.VerifyUserFromRequest(req, userAPI)
+			if err != nil {
+				logger.Debugf("VerifyUserFromRequest %s -> HTTP %d", req.RemoteAddr, err.Code)
+				return *err
+			}
+			// add the user ID to the logger
+			logger = logger.WithField("user_id", dev.UserID)
+			req = req.WithContext(util.ContextWithLogger(req.Context(), logger))
+		}
+		// add the user to Sentry, if enabled
+		hub := sentry.GetHubFromContext(req.Context())
+		if hub != nil {
+			hub.Scope().SetTag("user_id", dev.UserID)
+			hub.Scope().SetTag("device_id", dev.ID)
+		}
+		defer func() {
+			if r := recover(); r != nil {
+				if hub != nil {
+					hub.CaptureException(fmt.Errorf("%s panicked", req.URL.Path))
+				}
+				// re-panic to return the 500
+				panic(r)
+			}
+		}()
+		jsonRes = f(req, dev)
+		// do not log 4xx as errors as they are client fails, not server fails
+		if hub != nil && jsonRes.Code >= 500 {
+			hub.Scope().SetExtra("response", jsonRes)
+			hub.CaptureException(fmt.Errorf("%s returned HTTP %d", req.URL.Path, jsonRes.Code))
+		}
+		return jsonRes
+	}
+	return MakeExternalAPI(metricsName, h)
+}
+
 // MakeAdminAPI is a wrapper around MakeAuthAPI which enforces that the request can only be
 // completed by a user that is a server administrator.
 func MakeAdminAPI(

internal/log_unix.go

@@ -18,10 +18,8 @@
 package internal
 
 import (
-	"io"
 	"log/syslog"
 
-	"github.com/MFAshby/stdemuxerhook"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/sirupsen/logrus"
 	lSyslog "github.com/sirupsen/logrus/hooks/syslog"
@@ -31,7 +29,6 @@ import (
 // If something fails here it means that the logging was improperly configured,
 // so we just exit with the error
 func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
-	stdLogAdded := false
 	for _, hook := range hooks {
 		// Check we received a proper logging level
 		level, err := logrus.ParseLevel(hook.Level)
@@ -39,12 +36,6 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
 			logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err)
 		}
 
-		// Perform a first filter on the logs according to the lowest level of all
-		// (Eg: If we have hook for info and above, prevent logrus from processing debug logs)
-		if logrus.GetLevel() < level {
-			logrus.SetLevel(level)
-		}
-
 		switch hook.Type {
 		case "file":
 			checkFileHookParams(hook.Params)
@@ -53,17 +44,10 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
 			checkSyslogHookParams(hook.Params)
 			setupSyslogHook(hook, level, componentName)
 		case "std":
-			setupStdLogHook(level)
-			stdLogAdded = true
 		default:
 			logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type)
 		}
 	}
-	if !stdLogAdded {
-		setupStdLogHook(logrus.InfoLevel)
-	}
-	// Hooks are now configured for stdout/err, so throw away the default logger output
-	logrus.SetOutput(io.Discard)
 }
 
 func checkSyslogHookParams(params map[string]interface{}) {
@@ -87,10 +71,6 @@ func checkSyslogHookParams(params map[string]interface{}) {
 
 }
 
-func setupStdLogHook(level logrus.Level) {
-	logrus.AddHook(&logLevelHook{level, stdemuxerhook.New(logrus.StandardLogger())})
-}
-
 func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) {
 	syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName)
 	if err == nil {

roomserver/internal/perform/perform_invite.go

@@ -140,7 +140,7 @@ func (r *Inviter) PerformInvite(
 
 	var isAlreadyJoined bool
 	if info != nil {
-		_, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey())
+		_, _, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey())
 		if err != nil {
 			return nil, fmt.Errorf("r.DB.GetMembership: %w", err)
 		}

roomserver/internal/query/query.go

@@ -32,6 +32,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/roomserver/version"
 )
@@ -180,11 +181,16 @@ func (r *Queryer) QueryMembershipForUser(
 	}
 	response.RoomExists = true
 
-	membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID)
+	membershipEventNID, membershipState, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID)
 	if err != nil {
 		return err
 	}
 
+	if membershipState == tables.MembershipStateInvite {
+		response.Membership = gomatrixserverlib.Invite
+		response.IsInRoom = true
+	}
+
 	response.IsRoomForgotten = isRoomforgotten
 
 	if membershipEventNID == 0 {
@@ -327,7 +333,7 @@ func (r *Queryer) QueryMembershipsForRoom(
 		return nil
 	}
 
-	membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender)
+	membershipEventNID, _, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender)
 	if err != nil {
 		return err
 	}
@@ -942,7 +948,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
 		}
 		// At this point we're happy that we are in the room, so now let's
 		// see if the target user is in the room.
-		_, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
+		_, _, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
 		if err != nil {
 			continue
 		}

roomserver/storage/interface.go

@@ -129,7 +129,7 @@ type Database interface {
 	// in this room, along a boolean set to true if the user is still in this room,
 	// false if not.
 	// Returns an error if there was a problem talking to the database.
-	GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomForgotten bool, err error)
+	GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, membershipNID tables.MembershipState, stillInRoom, isRoomForgotten bool, err error)
 	// Lookup the membership event numeric IDs for all user that are or have
 	// been members of a given room. Only lookup events of "join" membership if
 	// joinOnly is set to true.

roomserver/storage/shared/storage.go

@@ -400,14 +400,14 @@ func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error {
 	})
 }
 
-func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomforgotten bool, err error) {
+func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, membershipState tables.MembershipState, stillInRoom, isRoomforgotten bool, err error) {
 	var requestSenderUserNID types.EventStateKeyNID
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		requestSenderUserNID, err = d.assignStateKeyNID(ctx, txn, requestSenderUserID)
 		return err
 	})
 	if err != nil {
-		return 0, false, false, fmt.Errorf("d.assignStateKeyNID: %w", err)
+		return 0, 0, false, false, fmt.Errorf("d.assignStateKeyNID: %w", err)
 	}
 
 	senderMembershipEventNID, senderMembership, isRoomforgotten, err :=
@@ -416,12 +416,12 @@ func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, req
 		)
 	if err == sql.ErrNoRows {
 		// The user has never been a member of that room
-		return 0, false, false, nil
+		return 0, 0, false, false, nil
 	} else if err != nil {
 		return
 	}
 
-	return senderMembershipEventNID, senderMembership == tables.MembershipStateJoin, isRoomforgotten, nil
+	return senderMembershipEventNID, senderMembership, senderMembership == tables.MembershipStateJoin, isRoomforgotten, nil
 }
 
 func (d *Database) GetMembershipEventNIDsForRoom(

setup/base/base.go

@@ -137,7 +137,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		logrus.Fatalf("Failed to start due to configuration errors")
 	}
 
-	internal.SetupStdLogging()
 	internal.SetupHookLogging(cfg.Logging, componentName)
 	internal.SetupPprof()
 

setup/config/config.go

@@ -16,6 +16,7 @@ package config
 
 import (
 	"bytes"
+	"crypto/x509"
 	"encoding/pem"
 	"fmt"
 	"io"
@@ -266,6 +267,15 @@ func loadConfig(
 			return nil, fmt.Errorf("either specify a 'private_key' path or supply both 'public_key' and 'key_id'")
 		}
 	}
+	if c.ClientAPI.JwtConfig.Enabled {
+		pubPki, _ := pem.Decode([]byte(c.ClientAPI.JwtConfig.Secret))
+		var pub interface{}
+		pub, err = x509.ParsePKIXPublicKey(pubPki.Bytes)
+		if err != nil {
+			return nil, err
+		}
+		c.ClientAPI.JwtConfig.SecretKey = pub.(ed25519.PublicKey)
+	}
 
 	c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath))
 
@@ -305,7 +315,10 @@ func (config *Dendrite) Derive() error {
 		config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
 			authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeDummy}})
 	}
-
+	if config.ClientAPI.ThreePidDelegate != "" {
+		config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
+			authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeEmail}})
+	}
 	// Load application service configuration files
 	if err := loadAppServices(&config.AppServiceAPI, &config.Derived); err != nil {
 		return err

setup/config/config_appservice.go

@@ -21,7 +21,6 @@ import (
 	"regexp"
 	"strings"
 
-	log "github.com/sirupsen/logrus"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -346,11 +345,11 @@ func checkErrors(config *AppServiceAPI, derived *Derived) (err error) {
 
 		// TODO: Remove once rate_limited is implemented
 		if appservice.RateLimited {
-			log.Warn("WARNING: Application service option rate_limited is currently unimplemented")
+			// log.Warn("WARNING: Application service option rate_limited is currently unimplemented")
 		}
 		// TODO: Remove once protocols is implemented
 		if len(appservice.Protocols) > 0 {
-			log.Warn("WARNING: Application service option protocols is currently unimplemented")
+			// log.Warn("WARNING: Application service option protocols is currently unimplemented")
 		}
 	}
 
@@ -376,7 +375,7 @@ func validateNamespace(
 	// Check if GroupID for the users namespace is in the correct format
 	if key == "users" && namespace.GroupID != "" {
 		// TODO: Remove once group_id is implemented
-		log.Warn("WARNING: Application service option group_id is currently unimplemented")
+		// log.Warn("WARNING: Application service option group_id is currently unimplemented")
 
 		correctFormat := groupIDRegexp.MatchString(namespace.GroupID)
 		if !correctFormat {

setup/config/config_clientapi.go

@@ -3,6 +3,9 @@ package config
 import (
 	"fmt"
 	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/ratelimit"
+	"golang.org/x/crypto/ed25519"
 )
 
 type ClientAPI struct {
@@ -52,9 +55,23 @@ type ClientAPI struct {
 	TURN TURN `yaml:"turn"`
 
 	// Rate-limiting options
-	RateLimiting RateLimiting `yaml:"rate_limiting"`
+	RateLimiting  RateLimiting                  `yaml:"rate_limiting"`
+	RtFailedLogin ratelimit.RtFailedLoginConfig `yaml:"rate_limiting_failed_login"`
 
 	MSCs *MSCs `yaml:"-"`
+
+	ThreePidDelegate string `yaml:"three_pid_delegate"`
+
+	JwtConfig JwtConfig `yaml:"jwt_config"`
+}
+
+type JwtConfig struct {
+	Enabled   bool   `yaml:"enabled"`
+	Algorithm string `yaml:"algorithm"`
+	Issuer    string `yaml:"issuer"`
+	Secret    string `yaml:"secret"`
+	SecretKey ed25519.PublicKey
+	Audiences []string `yaml:"audiences"`
 }
 
 func (c *ClientAPI) Defaults(opts DefaultOpts) {

setup/jetstream/nats.go

@@ -8,13 +8,13 @@ import (
 	"sync"
 	"time"
 
-	"github.com/getsentry/sentry-go"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
 
 	natsserver "github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
 	natsclient "github.com/nats-io/nats.go"
 )
 
@@ -36,7 +36,7 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
 	defer natsLock.Unlock()
 	// check if we need an in-process NATS Server
 	if len(cfg.Addresses) != 0 {
-		return setupNATS(process, cfg, nil)
+		return setupNATS(cfg, nil)
 	}
 	if s.Server == nil {
 		var err error
@@ -72,13 +72,34 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
 	if err != nil {
 		logrus.Fatalln("Failed to create NATS client")
 	}
-	return setupNATS(process, cfg, nc)
+	return setupNATS(cfg, nc)
 }
 
-func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
+func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
+	var s nats.JetStreamContext
+	var err error
 	if nc == nil {
-		var err error
+		opts := []natsclient.Option{
-		opts := []natsclient.Option{}
+			natsclient.DisconnectErrHandler(func(c *natsclient.Conn, err error) {
+				logrus.WithError(err).Error("nats connection: disconnected")
+			}),
+			natsclient.ReconnectHandler(func(_ *natsclient.Conn) {
+				logrus.Info("nats connection: client reconnected")
+				for _, stream := range []*nats.StreamConfig{
+					streams[6],
+					streams[10],
+				} {
+					err = configureStream(stream, cfg, s)
+					if err != nil {
+						logrus.WithError(err).WithField("stream", stream.Name).Error("unable to configure a stream")
+					}
+
+				}
+			}),
+			natsclient.ClosedHandler(func(_ *natsclient.Conn) {
+				logrus.Info("nats connection: client closed")
+			}),
+		}
 		if cfg.DisableTLSValidation {
 			opts = append(opts, natsclient.Secure(&tls.Config{
 				InsecureSkipVerify: true,
@@ -91,89 +112,16 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
 		}
 	}
 
-	s, err := nc.JetStream()
+	s, err = nc.JetStream()
 	if err != nil {
 		logrus.WithError(err).Panic("Unable to get JetStream context")
 		return nil, nil
 	}
 
 	for _, stream := range streams { // streams are defined in streams.go
-		name := cfg.Prefixed(stream.Name)
+		err = configureStream(stream, cfg, s)
-		info, err := s.StreamInfo(name)
+		if err != nil {
-		if err != nil && err != natsclient.ErrStreamNotFound {
+			logrus.WithError(err).WithField("stream", stream.Name).Fatal("unable to configure a stream")
-			logrus.WithError(err).Fatal("Unable to get stream info")
-		}
-		subjects := stream.Subjects
-		if len(subjects) == 0 {
-			// By default we want each stream to listen for the subjects
-			// that are either an exact match for the stream name, or where
-			// the first part of the subject is the stream name. ">" is a
-			// wildcard in NATS for one or more subject tokens. In the case
-			// that the stream is called "Foo", this will match any message
-			// with the subject "Foo", "Foo.Bar" or "Foo.Bar.Baz" etc.
-			subjects = []string{name, name + ".>"}
-		}
-		if info != nil {
-			switch {
-			case !reflect.DeepEqual(info.Config.Subjects, subjects):
-				fallthrough
-			case info.Config.Retention != stream.Retention:
-				fallthrough
-			case info.Config.Storage != stream.Storage:
-				if err = s.DeleteStream(name); err != nil {
-					logrus.WithError(err).Fatal("Unable to delete stream")
-				}
-				info = nil
-			}
-		}
-		if info == nil {
-			// If we're trying to keep everything in memory (e.g. unit tests)
-			// then overwrite the storage policy.
-			if cfg.InMemory {
-				stream.Storage = natsclient.MemoryStorage
-			}
-
-			// Namespace the streams without modifying the original streams
-			// array, otherwise we end up with namespaces on namespaces.
-			namespaced := *stream
-			namespaced.Name = name
-			namespaced.Subjects = subjects
-			if _, err = s.AddStream(&namespaced); err != nil {
-				logger := logrus.WithError(err).WithFields(logrus.Fields{
-					"stream":   namespaced.Name,
-					"subjects": namespaced.Subjects,
-				})
-
-				// If the stream was supposed to be in-memory to begin with
-				// then an error here is fatal so we'll give up.
-				if namespaced.Storage == natsclient.MemoryStorage {
-					logger.WithError(err).Fatal("Unable to add in-memory stream")
-				}
-
-				// The stream was supposed to be on disk. Let's try starting
-				// Dendrite with the stream in-memory instead. That'll mean that
-				// we can't recover anything that was queued on the disk but we
-				// will still be able to start and run hopefully in the meantime.
-				logger.WithError(err).Error("Unable to add stream")
-				sentry.CaptureException(fmt.Errorf("Unable to add stream %q: %w", namespaced.Name, err))
-
-				namespaced.Storage = natsclient.MemoryStorage
-				if _, err = s.AddStream(&namespaced); err != nil {
-					// We tried to add the stream in-memory instead but something
-					// went wrong. That's an unrecoverable situation so we will
-					// give up at this point.
-					logger.WithError(err).Fatal("Unable to add in-memory stream")
-				}
-
-				if stream.Storage != namespaced.Storage {
-					// We've managed to add the stream in memory.  What's on the
-					// disk will be left alone, but our ability to recover from a
-					// future crash will be limited. Yell about it.
-					err := fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory", namespaced.Name)
-					sentry.CaptureException(err)
-					process.Degraded(err)
-				}
-			}
 		}
 	}
 
@@ -203,3 +151,52 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
 
 	return s, nc
 }
+
+func configureStream(stream *nats.StreamConfig, cfg *config.JetStream, s nats.JetStreamContext) error {
+	name := cfg.Prefixed(stream.Name)
+	info, err := s.StreamInfo(name)
+	if err != nil && err != natsclient.ErrStreamNotFound {
+		return fmt.Errorf("get stream info: %w", err)
+	}
+	subjects := stream.Subjects
+	if len(subjects) == 0 {
+		// By default we want each stream to listen for the subjects
+		// that are either an exact match for the stream name, or where
+		// the first part of the subject is the stream name. ">" is a
+		// wildcard in NATS for one or more subject tokens. In the case
+		// that the stream is called "Foo", this will match any message
+		// with the subject "Foo", "Foo.Bar" or "Foo.Bar.Baz" etc.
+		subjects = []string{name, name + ".>"}
+	}
+	if info != nil {
+		switch {
+		case !reflect.DeepEqual(info.Config.Subjects, subjects):
+			fallthrough
+		case info.Config.Retention != stream.Retention:
+			fallthrough
+		case info.Config.Storage != stream.Storage:
+			if err = s.DeleteStream(name); err != nil {
+				return fmt.Errorf("delete stream: %w", err)
+			}
+			info = nil
+		}
+	}
+	if info == nil {
+		// If we're trying to keep everything in memory (e.g. unit tests)
+		// then overwrite the storage policy.
+		if cfg.InMemory {
+			stream.Storage = natsclient.MemoryStorage
+		}
+
+		// Namespace the streams without modifying the original streams
+		// array, otherwise we end up with namespaces on namespaces.
+		namespaced := *stream
+		namespaced.Name = name
+		namespaced.Subjects = subjects
+		if _, err = s.AddStream(&namespaced); err != nil {
+			return fmt.Errorf("add stream: %w", err)
+		}
+		logrus.Infof("stream created: %s", stream.Name)
+	}
+	return nil
+}

setup/jetstream/streams.go

@@ -31,6 +31,7 @@ var (
 	RequestPresence         = "GetPresence"
 	OutputPresenceEvent     = "OutputPresenceEvent"
 	InputFulltextReindex    = "InputFulltextReindex"
+	OutputMultiRoomCast     = "OutputMultiRoomCast"
 )
 
 var safeCharacters = regexp.MustCompile("[^A-Za-z0-9$]+")
@@ -101,4 +102,9 @@ var streams = []*nats.StreamConfig{
 		Storage:   nats.MemoryStorage,
 		MaxAge:    time.Minute * 5,
 	},
+	{
+		Name:      OutputMultiRoomCast,
+		Retention: nats.InterestPolicy,
+		Storage:   nats.FileStorage,
+	},
 }

syncapi/consumers/multiroomdata.go

@@ -0,0 +1,113 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"time"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/syncapi/notifier"
+	"github.com/matrix-org/dendrite/syncapi/storage/mrd"
+	"github.com/matrix-org/dendrite/syncapi/streams"
+	"github.com/matrix-org/dendrite/syncapi/types"
+)
+
+// OutputMultiRoomDataConsumer consumes events that originated in the client API server.
+type OutputMultiRoomDataConsumer struct {
+	ctx       context.Context
+	jetstream nats.JetStreamContext
+	durable   string
+	topic     string
+	db        *mrd.Queries
+	stream    streams.StreamProvider
+	notifier  *notifier.Notifier
+}
+
+// NewOutputMultiRoomDataConsumer creates a new OutputMultiRoomDataConsumer consumer. Call Start() to begin consuming from room servers.
+func NewOutputMultiRoomDataConsumer(
+	process *process.ProcessContext,
+	cfg *config.SyncAPI,
+	js nats.JetStreamContext,
+	q *mrd.Queries,
+	notifier *notifier.Notifier,
+	stream streams.StreamProvider,
+) *OutputMultiRoomDataConsumer {
+	return &OutputMultiRoomDataConsumer{
+		ctx:       process.Context(),
+		jetstream: js,
+		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputMultiRoomCast),
+		durable:   cfg.Matrix.JetStream.Durable("SyncAPIMultiRoomDataConsumer"),
+		db:        q,
+		notifier:  notifier,
+		stream:    stream,
+	}
+}
+
+func (s *OutputMultiRoomDataConsumer) Start() error {
+	return jetstream.JetStreamConsumer(
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
+	)
+}
+
+func (s *OutputMultiRoomDataConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0]
+	userID := msg.Header.Get(jetstream.UserID)
+	dataType := msg.Header.Get("type")
+
+	log.WithFields(log.Fields{
+		"type":    dataType,
+		"user_id": userID,
+	}).Debug("Received multiroom data from client API server")
+
+	pos, err := s.db.InsertMultiRoomData(ctx, mrd.InsertMultiRoomDataParams{
+		UserID: userID,
+		Type:   dataType,
+		Data:   msg.Data,
+	})
+	if err != nil {
+		sentry.CaptureException(err)
+		log.WithFields(log.Fields{
+			"type":    dataType,
+			"user_id": userID,
+		}).WithError(err).Errorf("could not insert multi room data")
+		return false
+	}
+
+	rooms, err := s.db.SelectMultiRoomVisibilityRooms(ctx, mrd.SelectMultiRoomVisibilityRoomsParams{
+		UserID:   userID,
+		ExpireTs: time.Now().Unix(),
+	})
+	if err != nil {
+		sentry.CaptureException(err)
+		log.WithFields(log.Fields{
+			"type":    dataType,
+			"user_id": userID,
+		}).WithError(err).Errorf("failed to select multi room visibility")
+		return false
+	}
+
+	s.stream.Advance(types.StreamPosition(pos))
+	s.notifier.OnNewMultiRoomData(types.StreamingToken{MultiRoomDataPosition: types.StreamPosition(pos)}, rooms)
+
+	return true
+}

syncapi/notifier/notifier.go

@@ -280,6 +280,32 @@ func (n *Notifier) _sharedUsers(userID string) []string {
 	return sharedUsers
 }
 
+func (n *Notifier) OnNewMultiRoomData(
+	posUpdate types.StreamingToken, roomIds []string,
+) {
+	n.lock.Lock()
+	defer n.lock.Unlock()
+
+	n.currPos.ApplyUpdates(posUpdate)
+	usersInRoom := n._usersInRooms(roomIds)
+
+	n._wakeupUsers(usersInRoom, nil, n.currPos)
+}
+
+func (n *Notifier) _usersInRooms(roomIds []string) []string {
+	for i := range roomIds {
+		for _, userID := range n._joinedUsers(roomIds[i]) {
+			n._sharedUserMap[userID] = struct{}{}
+		}
+	}
+	usersInRooms := make([]string, 0, len(n._sharedUserMap)+1)
+	for userID := range n._sharedUserMap {
+		usersInRooms = append(usersInRooms, userID)
+		delete(n._sharedUserMap, userID)
+	}
+	return usersInRooms
+}
+
 func (n *Notifier) IsSharedUser(userA, userB string) bool {
 	n.lock.RLock()
 	defer n.lock.RUnlock()

syncapi/storage/interface.go

@@ -109,6 +109,7 @@ type DatabaseTransaction interface {
 	GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
 	PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
 	RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error)
+	SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string) (types.MultiRoom, error)
 }
 
 type Database interface {
@@ -188,6 +189,10 @@ type Database interface {
 type Presence interface {
 	GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
 	UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error)
+	PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
+	MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error)
+	ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error)
+	UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error
 }
 
 type SharedUsers interface {

syncapi/storage/mrd/README.md

@@ -0,0 +1,3 @@
+## Multiroom storage
+
+please install `sqlc`: `go install github.com/kyleconroy/sqlc/cmd/sqlc@latest`. Then run `sqlc -f sqlc.yaml generate` in this directory after changing `queries.sql` or `../postgres/schema.sql` files.

syncapi/storage/mrd/db.go

@@ -0,0 +1,138 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.15.0
+
+package mrd
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+)
+
+type DBTX interface {
+	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
+	PrepareContext(context.Context, string) (*sql.Stmt, error)
+	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
+	QueryRowContext(context.Context, string, ...interface{}) *sql.Row
+}
+
+func New(db DBTX) *Queries {
+	return &Queries{db: db}
+}
+
+func Prepare(ctx context.Context, db DBTX) (*Queries, error) {
+	q := Queries{db: db}
+	var err error
+	if q.deleteMultiRoomVisibilityStmt, err = db.PrepareContext(ctx, deleteMultiRoomVisibility); err != nil {
+		return nil, fmt.Errorf("error preparing query DeleteMultiRoomVisibility: %w", err)
+	}
+	if q.deleteMultiRoomVisibilityByExpireTSStmt, err = db.PrepareContext(ctx, deleteMultiRoomVisibilityByExpireTS); err != nil {
+		return nil, fmt.Errorf("error preparing query DeleteMultiRoomVisibilityByExpireTS: %w", err)
+	}
+	if q.insertMultiRoomDataStmt, err = db.PrepareContext(ctx, insertMultiRoomData); err != nil {
+		return nil, fmt.Errorf("error preparing query InsertMultiRoomData: %w", err)
+	}
+	if q.insertMultiRoomVisibilityStmt, err = db.PrepareContext(ctx, insertMultiRoomVisibility); err != nil {
+		return nil, fmt.Errorf("error preparing query InsertMultiRoomVisibility: %w", err)
+	}
+	if q.selectMaxIdStmt, err = db.PrepareContext(ctx, selectMaxId); err != nil {
+		return nil, fmt.Errorf("error preparing query SelectMaxId: %w", err)
+	}
+	if q.selectMultiRoomVisibilityRoomsStmt, err = db.PrepareContext(ctx, selectMultiRoomVisibilityRooms); err != nil {
+		return nil, fmt.Errorf("error preparing query SelectMultiRoomVisibilityRooms: %w", err)
+	}
+	return &q, nil
+}
+
+func (q *Queries) Close() error {
+	var err error
+	if q.deleteMultiRoomVisibilityStmt != nil {
+		if cerr := q.deleteMultiRoomVisibilityStmt.Close(); cerr != nil {
+			err = fmt.Errorf("error closing deleteMultiRoomVisibilityStmt: %w", cerr)
+		}
+	}
+	if q.deleteMultiRoomVisibilityByExpireTSStmt != nil {
+		if cerr := q.deleteMultiRoomVisibilityByExpireTSStmt.Close(); cerr != nil {
+			err = fmt.Errorf("error closing deleteMultiRoomVisibilityByExpireTSStmt: %w", cerr)
+		}
+	}
+	if q.insertMultiRoomDataStmt != nil {
+		if cerr := q.insertMultiRoomDataStmt.Close(); cerr != nil {
+			err = fmt.Errorf("error closing insertMultiRoomDataStmt: %w", cerr)
+		}
+	}
+	if q.insertMultiRoomVisibilityStmt != nil {
+		if cerr := q.insertMultiRoomVisibilityStmt.Close(); cerr != nil {
+			err = fmt.Errorf("error closing insertMultiRoomVisibilityStmt: %w", cerr)
+		}
+	}
+	if q.selectMaxIdStmt != nil {
+		if cerr := q.selectMaxIdStmt.Close(); cerr != nil {
+			err = fmt.Errorf("error closing selectMaxIdStmt: %w", cerr)
+		}
+	}
+	if q.selectMultiRoomVisibilityRoomsStmt != nil {
+		if cerr := q.selectMultiRoomVisibilityRoomsStmt.Close(); cerr != nil {
+			err = fmt.Errorf("error closing selectMultiRoomVisibilityRoomsStmt: %w", cerr)
+		}
+	}
+	return err
+}
+
+func (q *Queries) exec(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (sql.Result, error) {
+	switch {
+	case stmt != nil && q.tx != nil:
+		return q.tx.StmtContext(ctx, stmt).ExecContext(ctx, args...)
+	case stmt != nil:
+		return stmt.ExecContext(ctx, args...)
+	default:
+		return q.db.ExecContext(ctx, query, args...)
+	}
+}
+
+func (q *Queries) query(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (*sql.Rows, error) {
+	switch {
+	case stmt != nil && q.tx != nil:
+		return q.tx.StmtContext(ctx, stmt).QueryContext(ctx, args...)
+	case stmt != nil:
+		return stmt.QueryContext(ctx, args...)
+	default:
+		return q.db.QueryContext(ctx, query, args...)
+	}
+}
+
+func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) *sql.Row {
+	switch {
+	case stmt != nil && q.tx != nil:
+		return q.tx.StmtContext(ctx, stmt).QueryRowContext(ctx, args...)
+	case stmt != nil:
+		return stmt.QueryRowContext(ctx, args...)
+	default:
+		return q.db.QueryRowContext(ctx, query, args...)
+	}
+}
+
+type Queries struct {
+	db                                      DBTX
+	tx                                      *sql.Tx
+	deleteMultiRoomVisibilityStmt           *sql.Stmt
+	deleteMultiRoomVisibilityByExpireTSStmt *sql.Stmt
+	insertMultiRoomDataStmt                 *sql.Stmt
+	insertMultiRoomVisibilityStmt           *sql.Stmt
+	selectMaxIdStmt                         *sql.Stmt
+	selectMultiRoomVisibilityRoomsStmt      *sql.Stmt
+}
+
+func (q *Queries) WithTx(tx *sql.Tx) *Queries {
+	return &Queries{
+		db:                                      tx,
+		tx:                                      tx,
+		deleteMultiRoomVisibilityStmt:           q.deleteMultiRoomVisibilityStmt,
+		deleteMultiRoomVisibilityByExpireTSStmt: q.deleteMultiRoomVisibilityByExpireTSStmt,
+		insertMultiRoomDataStmt:                 q.insertMultiRoomDataStmt,
+		insertMultiRoomVisibilityStmt:           q.insertMultiRoomVisibilityStmt,
+		selectMaxIdStmt:                         q.selectMaxIdStmt,
+		selectMultiRoomVisibilityRoomsStmt:      q.selectMultiRoomVisibilityRoomsStmt,
+	}
+}

syncapi/storage/mrd/models.go

@@ -0,0 +1,24 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.15.0
+
+package mrd
+
+import (
+	"time"
+)
+
+type SyncapiMultiroomDatum struct {
+	ID     int64     `json:"id"`
+	UserID string    `json:"user_id"`
+	Type   string    `json:"type"`
+	Data   []byte    `json:"data"`
+	Ts     time.Time `json:"ts"`
+}
+
+type SyncapiMultiroomVisibility struct {
+	UserID   string `json:"user_id"`
+	Type     string `json:"type"`
+	RoomID   string `json:"room_id"`
+	ExpireTs int64  `json:"expire_ts"`
+}

syncapi/storage/mrd/queries.sql

@@ -0,0 +1,44 @@
+-- name: InsertMultiRoomData :one
+INSERT INTO syncapi_multiroom_data (
+    user_id,
+    type,
+    data
+) VALUES (
+    $1,
+    $2,
+    $3
+) ON CONFLICT (user_id, type) DO UPDATE SET id = nextval('syncapi_multiroom_id'), data = $3, ts = current_timestamp
+RETURNING id;
+
+
+-- name: InsertMultiRoomVisibility :exec
+INSERT INTO syncapi_multiroom_visibility (
+    user_id,
+    type,
+    room_id,
+    expire_ts
+) VALUES (
+    $1,
+    $2,
+    $3,
+    $4
+) ON CONFLICT (user_id, type, room_id) DO UPDATE SET expire_ts = $4;
+
+-- name: SelectMultiRoomVisibilityRooms :many
+SELECT room_id FROM syncapi_multiroom_visibility
+WHERE user_id = $1 
+AND expire_ts > $2;
+
+
+-- name: SelectMaxId :one
+SELECT MAX(id) FROM syncapi_multiroom_data;
+
+-- name: DeleteMultiRoomVisibility :exec
+DELETE FROM syncapi_multiroom_visibility
+WHERE user_id = $1
+AND type = $2
+AND room_id = $3;
+
+-- name: DeleteMultiRoomVisibilityByExpireTS :execrows
+DELETE FROM syncapi_multiroom_visibility
+WHERE expire_ts <= $1;

syncapi/storage/mrd/queries.sql.go

@@ -0,0 +1,143 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.15.0
+// source: queries.sql
+
+package mrd
+
+import (
+	"context"
+)
+
+const deleteMultiRoomVisibility = `-- name: DeleteMultiRoomVisibility :exec
+DELETE FROM syncapi_multiroom_visibility
+WHERE user_id = $1
+AND type = $2
+AND room_id = $3
+`
+
+type DeleteMultiRoomVisibilityParams struct {
+	UserID string `json:"user_id"`
+	Type   string `json:"type"`
+	RoomID string `json:"room_id"`
+}
+
+func (q *Queries) DeleteMultiRoomVisibility(ctx context.Context, arg DeleteMultiRoomVisibilityParams) error {
+	_, err := q.exec(ctx, q.deleteMultiRoomVisibilityStmt, deleteMultiRoomVisibility, arg.UserID, arg.Type, arg.RoomID)
+	return err
+}
+
+const deleteMultiRoomVisibilityByExpireTS = `-- name: DeleteMultiRoomVisibilityByExpireTS :execrows
+DELETE FROM syncapi_multiroom_visibility
+WHERE expire_ts <= $1
+`
+
+func (q *Queries) DeleteMultiRoomVisibilityByExpireTS(ctx context.Context, expireTs int64) (int64, error) {
+	result, err := q.exec(ctx, q.deleteMultiRoomVisibilityByExpireTSStmt, deleteMultiRoomVisibilityByExpireTS, expireTs)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected()
+}
+
+const insertMultiRoomData = `-- name: InsertMultiRoomData :one
+INSERT INTO syncapi_multiroom_data (
+    user_id,
+    type,
+    data
+) VALUES (
+    $1,
+    $2,
+    $3
+) ON CONFLICT (user_id, type) DO UPDATE SET id = nextval('syncapi_multiroom_id'), data = $3, ts = current_timestamp
+RETURNING id
+`
+
+type InsertMultiRoomDataParams struct {
+	UserID string `json:"user_id"`
+	Type   string `json:"type"`
+	Data   []byte `json:"data"`
+}
+
+func (q *Queries) InsertMultiRoomData(ctx context.Context, arg InsertMultiRoomDataParams) (int64, error) {
+	row := q.queryRow(ctx, q.insertMultiRoomDataStmt, insertMultiRoomData, arg.UserID, arg.Type, arg.Data)
+	var id int64
+	err := row.Scan(&id)
+	return id, err
+}
+
+const insertMultiRoomVisibility = `-- name: InsertMultiRoomVisibility :exec
+INSERT INTO syncapi_multiroom_visibility (
+    user_id,
+    type,
+    room_id,
+    expire_ts
+) VALUES (
+    $1,
+    $2,
+    $3,
+    $4
+) ON CONFLICT (user_id, type, room_id) DO UPDATE SET expire_ts = $4
+`
+
+type InsertMultiRoomVisibilityParams struct {
+	UserID   string `json:"user_id"`
+	Type     string `json:"type"`
+	RoomID   string `json:"room_id"`
+	ExpireTs int64  `json:"expire_ts"`
+}
+
+func (q *Queries) InsertMultiRoomVisibility(ctx context.Context, arg InsertMultiRoomVisibilityParams) error {
+	_, err := q.exec(ctx, q.insertMultiRoomVisibilityStmt, insertMultiRoomVisibility,
+		arg.UserID,
+		arg.Type,
+		arg.RoomID,
+		arg.ExpireTs,
+	)
+	return err
+}
+
+const selectMaxId = `-- name: SelectMaxId :one
+SELECT MAX(id) FROM syncapi_multiroom_data
+`
+
+func (q *Queries) SelectMaxId(ctx context.Context) (interface{}, error) {
+	row := q.queryRow(ctx, q.selectMaxIdStmt, selectMaxId)
+	var max interface{}
+	err := row.Scan(&max)
+	return max, err
+}
+
+const selectMultiRoomVisibilityRooms = `-- name: SelectMultiRoomVisibilityRooms :many
+SELECT room_id FROM syncapi_multiroom_visibility
+WHERE user_id = $1 
+AND expire_ts > $2
+`
+
+type SelectMultiRoomVisibilityRoomsParams struct {
+	UserID   string `json:"user_id"`
+	ExpireTs int64  `json:"expire_ts"`
+}
+
+func (q *Queries) SelectMultiRoomVisibilityRooms(ctx context.Context, arg SelectMultiRoomVisibilityRoomsParams) ([]string, error) {
+	rows, err := q.query(ctx, q.selectMultiRoomVisibilityRoomsStmt, selectMultiRoomVisibilityRooms, arg.UserID, arg.ExpireTs)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []string
+	for rows.Next() {
+		var room_id string
+		if err := rows.Scan(&room_id); err != nil {
+			return nil, err
+		}
+		items = append(items, room_id)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}

syncapi/storage/mrd/sqlc.yaml

@@ -0,0 +1,8 @@
+version: 1
+packages:
+  - path: ../mrd
+    engine: postgresql
+    schema: ../postgres/schema.sql
+    queries: queries.sql
+    emit_json_tags: true
+    emit_prepared_queries: true

syncapi/storage/mrd/types.go

@@ -0,0 +1,6 @@
+package mrd
+
+type StateEvent struct {
+	Hidden   bool `json:"hidden"`
+	ExpireTs int  `json:"expire_ts"`
+}

syncapi/storage/postgres/multiroomcast_table.go

@@ -0,0 +1,61 @@
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	_ "embed"
+	"fmt"
+	"time"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
+	"github.com/matrix-org/dendrite/syncapi/types"
+)
+
+//go:embed schema.sql
+var schema string
+
+var selectMultiRoomCastSQL = `SELECT d.user_id, d.type, d.data, d.ts FROM syncapi_multiroom_data AS d
+JOIN syncapi_multiroom_visibility AS v
+ON d.user_id = v.user_id
+AND d.type = v.type
+WHERE v.room_id = ANY($1)
+AND id > $2
+AND id <= $3`
+
+type multiRoomStatements struct {
+	selectMultiRoomCast *sql.Stmt
+}
+
+func NewPostgresMultiRoomCastTable(db *sql.DB) (tables.MultiRoom, error) {
+	r := &multiRoomStatements{}
+	_, err := db.Exec(schema)
+	if err != nil {
+		return nil, err
+	}
+	return r, sqlutil.StatementList{
+		{&r.selectMultiRoomCast, selectMultiRoomCastSQL},
+	}.Prepare(db)
+}
+
+func (s *multiRoomStatements) SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error) {
+	rows, err := sqlutil.TxStmt(txn, s.selectMultiRoomCast).QueryContext(ctx, pq.StringArray(joinedRooms), r.Low(), r.High())
+	if err != nil {
+		return nil, err
+	}
+	data := make([]*types.MultiRoomDataRow, 0)
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectMultiRoomData: rows.close() failed")
+	var t time.Time
+	for rows.Next() {
+		r := types.MultiRoomDataRow{}
+		err = rows.Scan(&r.UserId, &r.Type, &r.Data, &t)
+		r.Timestamp = t.Unix()
+		if err != nil {
+			return nil, fmt.Errorf("rows scan: %w", err)
+		}
+		data = append(data, &r)
+	}
+	return data, rows.Err()
+}

syncapi/storage/postgres/presence_table.go

@@ -62,6 +62,10 @@ const upsertPresenceFromSyncSQL = "" +
 	" presence = $2, last_active_ts = $3" +
 	" RETURNING id"
 
+const updateLastActiveSQL = `UPDATE syncapi_presence
+SET last_active_ts = $1
+WHERE user_id = $2`
+
 const selectPresenceForUserSQL = "" +
 	"SELECT presence, status_msg, last_active_ts" +
 	" FROM syncapi_presence" +
@@ -76,12 +80,24 @@ const selectPresenceAfter = "" +
 	" WHERE id > $1 AND last_active_ts >= $2" +
 	" ORDER BY id ASC LIMIT $3"
 
+const expirePresenceSQL = `UPDATE syncapi_presence SET 
+	id = nextval('syncapi_presence_id'), 
+	presence = 3
+WHERE
+	to_timestamp(last_active_ts / 1000) < NOW() - INTERVAL` + types.PresenceExpire + `
+AND 
+	presence != 3
+RETURNING id, user_id
+`
+
 type presenceStatements struct {
 	upsertPresenceStmt         *sql.Stmt
 	upsertPresenceFromSyncStmt *sql.Stmt
 	selectPresenceForUsersStmt *sql.Stmt
 	selectMaxPresenceStmt      *sql.Stmt
 	selectPresenceAfterStmt    *sql.Stmt
+	expirePresenceStmt         *sql.Stmt
+	updateLastActiveStmt       *sql.Stmt
 }
 
 func NewPostgresPresenceTable(db *sql.DB) (*presenceStatements, error) {
@@ -96,6 +112,8 @@ func NewPostgresPresenceTable(db *sql.DB) (*presenceStatements, error) {
 		{&s.selectPresenceForUsersStmt, selectPresenceForUserSQL},
 		{&s.selectMaxPresenceStmt, selectMaxPresenceSQL},
 		{&s.selectPresenceAfterStmt, selectPresenceAfter},
+		{&s.expirePresenceStmt, expirePresenceSQL},
+		{&s.updateLastActiveStmt, updateLastActiveSQL},
 	}.Prepare(db)
 }
 
@@ -166,3 +184,28 @@ func (p *presenceStatements) GetPresenceAfter(
 	}
 	return presences, rows.Err()
 }
+
+func (p *presenceStatements) ExpirePresence(
+	ctx context.Context,
+) ([]types.PresenceNotify, error) {
+	rows, err := p.expirePresenceStmt.QueryContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+	presences := make([]types.PresenceNotify, 0)
+	i := 0
+	for rows.Next() {
+		presences = append(presences, types.PresenceNotify{})
+		err = rows.Scan(&presences[i].StreamPos, &presences[i].UserID)
+		if err != nil {
+			return nil, err
+		}
+		i++
+	}
+	return presences, err
+}
+
+func (p *presenceStatements) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error {
+	_, err := p.updateLastActiveStmt.Exec(&lastActiveTs, &userId)
+	return err
+}

syncapi/storage/postgres/schema.sql

@@ -0,0 +1,19 @@
+CREATE SEQUENCE IF NOT EXISTS syncapi_multiroom_id;
+
+CREATE TABLE IF NOT EXISTS syncapi_multiroom_data (
+	id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_multiroom_id'),
+	user_id TEXT NOT NULL,
+	type TEXT NOT NULL,
+	data BYTEA NOT NULL,
+	ts TIMESTAMP NOT NULL DEFAULT current_timestamp
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS syncapi_multiroom_data_user_id_type_idx ON syncapi_multiroom_data(user_id, type);
+
+CREATE TABLE IF NOT EXISTS syncapi_multiroom_visibility (
+	user_id TEXT NOT NULL,
+	type TEXT NOT NULL,
+	room_id TEXT NOT NULL,
+	expire_ts BIGINT NOT NULL DEFAULT 0,
+	PRIMARY KEY(user_id, type, room_id)
+)

syncapi/storage/postgres/syncserver.go

@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/storage/mrd"
 	"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/shared"
 )
@@ -102,6 +103,11 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	if err != nil {
 		return nil, err
 	}
+	mr, err := NewPostgresMultiRoomCastTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	mrq := mrd.New(d.db)
 
 	// apply migrations which need multiple tables
 	m := sqlutil.NewMigrator(d.db)
@@ -134,6 +140,8 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 		Ignores:             ignores,
 		Presence:            presence,
 		Relations:           relations,
+		MultiRoom:           mr,
+		MultiRoomQ:          mrq,
 	}
 	return &d, nil
 }

syncapi/storage/shared/storage_consumer.go

@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/tidwall/gjson"
 
@@ -30,6 +31,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage/mrd"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
@@ -54,6 +56,8 @@ type Database struct {
 	Ignores             tables.Ignores
 	Presence            tables.Presence
 	Relations           tables.Relations
+	MultiRoomQ          *mrd.Queries
+	MultiRoom           tables.MultiRoom
 }
 
 func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) {
@@ -336,6 +340,13 @@ func (d *Database) updateRoomState(
 			}
 		}
 
+		if strings.HasPrefix(event.Type(), "connect.mrd") {
+			err := d.UpdateMultiRoomVisibility(ctx, event)
+			if err != nil {
+				logrus.WithError(err).WithField("event_id", event.EventID()).Error("failed to update multi room visibility")
+			}
+		}
+
 		if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil {
 			return fmt.Errorf("d.CurrentRoomState.UpsertRoomState: %w", err)
 		}
@@ -625,3 +636,49 @@ func (d *Database) SelectMemberships(
 ) (eventIDs []string, err error) {
 	return d.Memberships.SelectMemberships(ctx, nil, roomID, pos, membership, notMembership)
 }
+
+func (s *Database) ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error) {
+	return s.Presence.ExpirePresence(ctx)
+}
+
+func (d *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) {
+	return d.Presence.GetMaxPresenceID(ctx, nil)
+}
+
+func (d *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {
+	return d.Presence.GetPresenceAfter(ctx, nil, after, filter)
+}
+
+func (s *Database) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error {
+	return s.Presence.UpdateLastActive(ctx, userId, lastActiveTs)
+}
+
+func (d *Database) UpdateMultiRoomVisibility(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
+	var mrdEv mrd.StateEvent
+	err := json.Unmarshal(event.Content(), &mrdEv)
+	if err != nil {
+		return fmt.Errorf("unmarshal multiroom visibility failed: %w", err)
+	}
+	if mrdEv.Hidden {
+		err = d.MultiRoomQ.DeleteMultiRoomVisibility(ctx, mrd.DeleteMultiRoomVisibilityParams{
+			UserID: event.Sender(),
+			Type:   event.Type(),
+			RoomID: event.RoomID(),
+		})
+		if err != nil {
+			return fmt.Errorf("delete multiroom visibility failed: %w", err)
+		}
+	}
+	if mrdEv.ExpireTs > 0 {
+		err = d.MultiRoomQ.InsertMultiRoomVisibility(ctx, mrd.InsertMultiRoomVisibilityParams{
+			UserID:   event.Sender(),
+			Type:     event.Type(),
+			RoomID:   event.RoomID(),
+			ExpireTs: int64(mrdEv.ExpireTs),
+		})
+		if err != nil {
+			return fmt.Errorf("insert multiroom visibility failed: %w", err)
+		}
+	}
+	return nil
+}

syncapi/storage/shared/storage_sync.go

@@ -688,3 +688,22 @@ func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID,
 
 	return events, prevBatch, nextBatch, nil
 }
+
+func (d *DatabaseTransaction) SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string) (types.MultiRoom, error) {
+	rows, err := d.MultiRoom.SelectMultiRoomData(ctx, r, joinedRooms, d.txn)
+	if err != nil {
+		return nil, fmt.Errorf("select multi room data: %w", err)
+	}
+	mr := make(types.MultiRoom, 3)
+	for _, row := range rows {
+		if mr[row.UserId] == nil {
+			mr[row.UserId] = make(map[string]types.MultiRoomData)
+		}
+		mr[row.UserId][row.Type] = types.MultiRoomData{
+			Content:   row.Data,
+			Timestamp: row.Timestamp,
+		}
+	}
+	return mr, nil
+
+}

syncapi/storage/sqlite3/presence_table.go

@@ -180,3 +180,15 @@ func (p *presenceStatements) GetPresenceAfter(
 	}
 	return presences, rows.Err()
 }
+
+func (p *presenceStatements) ExpirePresence(
+	ctx context.Context,
+) ([]types.PresenceNotify, error) {
+	// TODO implement
+	return nil, nil
+}
+
+func (p *presenceStatements) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error {
+	// TODO implement
+	return nil
+}

syncapi/storage/storage.go

@@ -22,18 +22,26 @@ import (
 
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/storage/mrd"
 	"github.com/matrix-org/dendrite/syncapi/storage/postgres"
 	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
 )
 
 // NewSyncServerDatasource opens a database connection.
-func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
+func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, *mrd.Queries, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(base, dbProperties)
+		ds, err := sqlite3.NewDatabase(base, dbProperties)
+		return ds, nil, err
+
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(base, dbProperties)
+		ds, err := postgres.NewDatabase(base, dbProperties)
+		if err != nil {
+			return nil, nil, err
+		}
+		mrq := mrd.New(ds.DB)
+		return ds, mrq, nil
 	default:
-		return nil, fmt.Errorf("unexpected database type")
+		return nil, nil, fmt.Errorf("unexpected database type")
 	}
 }

syncapi/storage/storage_test.go

@@ -21,7 +21,7 @@ var ctx = context.Background()
 func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func(), func()) {
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
 	base, closeBase := testrig.CreateBaseDendrite(t, dbType)
-	db, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{
+	db, _, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	})
 	if err != nil {

syncapi/storage/tables/interface.go

@@ -210,6 +210,8 @@ type Presence interface {
 	GetPresenceForUser(ctx context.Context, txn *sql.Tx, userID string) (presence *types.PresenceInternal, err error)
 	GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error)
 	GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error)
+	ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error)
+	UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error
 }
 
 type Relations interface {
@@ -230,3 +232,7 @@ type Relations interface {
 	// "from" or want to work forwards and don't have a "to").
 	SelectMaxRelationID(ctx context.Context, txn *sql.Tx) (id int64, err error)
 }
+
+type MultiRoom interface {
+	SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error)
+}

syncapi/streams/stream_multiroomdata.go

@@ -0,0 +1,72 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package streams
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/syncapi/notifier"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/storage/mrd"
+	"github.com/matrix-org/dendrite/syncapi/types"
+)
+
+type MultiRoomDataStreamProvider struct {
+	DefaultStreamProvider
+	notifier *notifier.Notifier
+	mrdDb    *mrd.Queries
+}
+
+func (p *MultiRoomDataStreamProvider) Setup(ctx context.Context, snapshot storage.DatabaseTransaction) {
+	p.DefaultStreamProvider.Setup(ctx, snapshot)
+
+	id, err := p.mrdDb.SelectMaxId(context.Background())
+	if err != nil && err != sql.ErrNoRows {
+		panic(err)
+	}
+	p.latestMutex.Lock()
+	defer p.latestMutex.Unlock()
+	if id == nil {
+		p.latest = types.StreamPosition(0)
+	} else {
+		p.latest = types.StreamPosition(id.(int64))
+	}
+}
+
+func (p *MultiRoomDataStreamProvider) CompleteSync(
+	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
+	req *types.SyncRequest,
+) types.StreamPosition {
+	return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
+}
+
+func (p *MultiRoomDataStreamProvider) IncrementalSync(
+	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
+	req *types.SyncRequest,
+	from, to types.StreamPosition,
+) types.StreamPosition {
+	mr, err := snapshot.SelectMultiRoomData(ctx, &types.Range{From: from, To: to}, req.JoinedRooms)
+	if err != nil {
+		req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed")
+		return from
+	}
+	req.Log.Tracef("MultiRoomDataStreamProvider IncrementalSync: %+v", mr)
+	req.Response.MultiRoom = mr
+	return to
+
+}

syncapi/streams/stream_pdu.go

@@ -77,6 +77,7 @@ func (p *PDUStreamProvider) CompleteSync(
 		req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed")
 		return from
 	}
+	req.JoinedRooms = joinedRoomIDs
 
 	stateFilter := req.Filter.Room.State
 	eventFilter := req.Filter.Room.Timeline
@@ -170,6 +171,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 	for _, roomID := range syncJoinedRooms {
 		req.Rooms[roomID] = gomatrixserverlib.Join
 	}
+	req.JoinedRooms = syncJoinedRooms
 
 	if len(stateDeltas) == 0 {
 		return to

syncapi/streams/stream_presence.go

@@ -17,7 +17,6 @@ package streams
 import (
 	"context"
 	"encoding/json"
-	"sync"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/tidwall/gjson"
@@ -29,8 +28,6 @@ import (
 
 type PresenceStreamProvider struct {
 	DefaultStreamProvider
-	// cache contains previously sent presence updates to avoid unneeded updates
-	cache    sync.Map
 	notifier *notifier.Notifier
 }
 
@@ -114,19 +111,6 @@ func (p *PresenceStreamProvider) IncrementalSync(
 		if req.Device.UserID != presence.UserID && !p.notifier.IsSharedUser(req.Device.UserID, presence.UserID) {
 			continue
 		}
-		cacheKey := req.Device.UserID + req.Device.ID + presence.UserID
-		pres, ok := p.cache.Load(cacheKey)
-		if ok {
-			// skip already sent presence
-			prevPresence := pres.(*types.PresenceInternal)
-			currentlyActive := prevPresence.CurrentlyActive()
-			skip := prevPresence.Equals(presence) && currentlyActive && req.Device.UserID != presence.UserID
-			_, membershipChange := req.MembershipChanges[presence.UserID]
-			if skip && !membershipChange {
-				req.Log.Tracef("Skipping presence, no change (%s)", presence.UserID)
-				continue
-			}
-		}
 
 		if _, known := types.PresenceFromString(presence.ClientFields.Presence); known {
 			presence.ClientFields.LastActiveAgo = presence.LastActiveAgo()
@@ -154,7 +138,6 @@ func (p *PresenceStreamProvider) IncrementalSync(
 		if len(req.Response.Presence.Events) == req.Filter.Presence.Limit {
 			break
 		}
-		p.cache.Store(cacheKey, presence)
 	}
 
 	if len(req.Response.Presence.Events) == 0 {

syncapi/streams/streams.go

@@ -9,6 +9,7 @@ import (
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/storage/mrd"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
@@ -23,12 +24,14 @@ type Streams struct {
 	DeviceListStreamProvider       StreamProvider
 	NotificationDataStreamProvider StreamProvider
 	PresenceStreamProvider         StreamProvider
+	MultiRoomStreamProvider        StreamProvider
 }
 
 func NewSyncStreamProviders(
 	d storage.Database, userAPI userapi.SyncUserAPI,
 	rsAPI rsapi.SyncRoomserverAPI, keyAPI keyapi.SyncKeyAPI,
 	eduCache *caching.EDUCache, lazyLoadCache caching.LazyLoadCache, notifier *notifier.Notifier,
+	mrdb *mrd.Queries,
 ) *Streams {
 	streams := &Streams{
 		PDUStreamProvider: &PDUStreamProvider{
@@ -66,6 +69,11 @@ func NewSyncStreamProviders(
 			DefaultStreamProvider: DefaultStreamProvider{DB: d},
 			notifier:              notifier,
 		},
+		MultiRoomStreamProvider: &MultiRoomDataStreamProvider{
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
+			notifier:              notifier,
+			mrdDb:                 mrdb,
+		},
 	}
 
 	ctx := context.TODO()
@@ -85,6 +93,7 @@ func NewSyncStreamProviders(
 	streams.NotificationDataStreamProvider.Setup(ctx, snapshot)
 	streams.DeviceListStreamProvider.Setup(ctx, snapshot)
 	streams.PresenceStreamProvider.Setup(ctx, snapshot)
+	streams.MultiRoomStreamProvider.Setup(ctx, snapshot)
 
 	succeeded = true
 	return streams

syncapi/sync/request.go

@@ -80,6 +80,8 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
 		filter.AccountData.Limit = math.MaxInt32
 		filter.Room.AccountData.Limit = math.MaxInt32
 	}
+	// Ignore state limit filter so that e.g. correct room name appears on clients.
+	filter.Room.State.Limit = math.MaxInt32
 
 	logger := util.GetLogger(req.Context()).WithFields(logrus.Fields{
 		"user_id":   device.UserID,

syncapi/sync/requestpool.go

@@ -51,7 +51,7 @@ type RequestPool struct {
 	keyAPI   keyapi.SyncKeyAPI
 	rsAPI    roomserverAPI.SyncRoomserverAPI
 	lastseen *sync.Map
-	presence *sync.Map
+	Presence *sync.Map
 	streams  *streams.Streams
 	Notifier *notifier.Notifier
 	producer PresencePublisher
@@ -86,14 +86,14 @@ func NewRequestPool(
 		keyAPI:   keyAPI,
 		rsAPI:    rsAPI,
 		lastseen: &sync.Map{},
-		presence: &sync.Map{},
+		Presence: &sync.Map{},
 		streams:  streams,
 		Notifier: notifier,
 		producer: producer,
 		consumer: consumer,
 	}
 	go rp.cleanLastSeen()
-	go rp.cleanPresence(db, time.Minute*5)
+	// go rp.cleanPresence(db, time.Minute*5)
 	return rp
 }
 
@@ -112,11 +112,11 @@ func (rp *RequestPool) cleanPresence(db storage.Presence, cleanupTime time.Durat
 		return
 	}
 	for {
-		rp.presence.Range(func(key interface{}, v interface{}) bool {
+		rp.Presence.Range(func(key interface{}, v interface{}) bool {
 			p := v.(types.PresenceInternal)
 			if time.Since(p.LastActiveTS.Time()) > cleanupTime {
 				rp.updatePresence(db, types.PresenceUnavailable.String(), p.UserID)
-				rp.presence.Delete(key)
+				rp.Presence.Delete(key)
 			}
 			return true
 		})
@@ -154,13 +154,22 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
 	}
 	newPresence.ClientFields.Presence = presenceID.String()
 
-	defer rp.presence.Store(userID, newPresence)
+	defer rp.Presence.Store(userID, newPresence)
 	// avoid spamming presence updates when syncing
-	existingPresence, ok := rp.presence.LoadOrStore(userID, newPresence)
+	existingPresence, ok := rp.Presence.LoadOrStore(userID, newPresence)
 	if ok {
 		p := existingPresence.(types.PresenceInternal)
-		if p.ClientFields.Presence == newPresence.ClientFields.Presence {
+		if dbPresence != nil {
-			return
+			if p.Presence == newPresence.Presence && newPresence.LastActiveTS-dbPresence.LastActiveTS < types.PresenceNoOpMs {
+				return
+			}
+			if dbPresence.Presence == types.PresenceOnline && presenceID == types.PresenceOnline && newPresence.LastActiveTS-dbPresence.LastActiveTS >= types.PresenceNoOpMs {
+				err := db.UpdateLastActive(context.Background(), userID, uint64(newPresence.LastActiveTS))
+				if err != nil {
+					logrus.WithError(err).Error("failed to update last active")
+				}
+				return
+			}
 		}
 	}
 
@@ -248,7 +257,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 	defer activeSyncRequests.Dec()
 
 	rp.updateLastSeen(req, device)
-	rp.updatePresence(rp.db, req.FormValue("set_presence"), device.UserID)
+	rp.updatePresence(rp.db, "", device.UserID)
 
 	waitingSyncRequests.Inc()
 	defer waitingSyncRequests.Dec()
@@ -398,6 +407,14 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 						)
 					},
 				),
+				MultiRoomDataPosition: withTransaction(
+					syncReq.Since.MultiRoomDataPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.MultiRoomStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
+				),
 			}
 		} else {
 			// Incremental sync
@@ -483,6 +500,15 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 						)
 					},
 				),
+				MultiRoomDataPosition: withTransaction(
+					syncReq.Since.MultiRoomDataPosition,
+					func(snapshot storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.MultiRoomStreamProvider.IncrementalSync(
+							syncReq.Context, snapshot, syncReq,
+							syncReq.Since.MultiRoomDataPosition, rp.Notifier.CurrentPosition().MultiRoomDataPosition,
+						)
+					},
+				),
 			}
 			// it's possible for there to be no updates for this user even though since < current pos,
 			// e.g busy servers with a quiet user. In this scenario, we don't want to return a no-op

syncapi/sync/requestpool_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -23,7 +24,9 @@ func (d *dummyPublisher) SendPresence(userID string, presence types.Presence, st
 	return nil
 }
 
-type dummyDB struct{}
+type dummyDB struct {
+	storage.Database
+}
 
 func (d dummyDB) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) {
 	return 0, nil
@@ -109,7 +112,7 @@ func TestRequestPool_updatePresence(t *testing.T) {
 		},
 	}
 	rp := &RequestPool{
-		presence: &syncMap,
+		Presence: &syncMap,
 		producer: publisher,
 		consumer: consumer,
 		cfg: &config.SyncAPI{

syncapi/syncapi.go

@@ -16,6 +16,7 @@ package syncapi
 
 import (
 	"context"
+	"time"
 
 	"github.com/sirupsen/logrus"
 
@@ -34,6 +35,7 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/sync"
+	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 // AddPublicRoutes sets up and registers HTTP handlers for the SyncAPI
@@ -48,14 +50,26 @@ func AddPublicRoutes(
 
 	js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
-	syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database)
+	syncDB, mrq, err := storage.NewSyncServerDatasource(base, &cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to sync db")
 	}
 
+	go func() {
+		var affected int64
+		for {
+			affected, err = mrq.DeleteMultiRoomVisibilityByExpireTS(context.Background(), time.Now().Unix())
+			if err != nil {
+				logrus.WithError(err).Error("failed to expire multiroom visibility")
+			}
+			logrus.WithField("rows", affected).Info("expired multiroom visibility")
+			time.Sleep(time.Minute)
+		}
+	}()
+
 	eduCache := caching.NewTypingCache()
 	notifier := notifier.NewNotifier()
-	streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier)
+	streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier, mrq)
 	notifier.SetCurrentPosition(streams.Latest(context.Background()))
 	if err = notifier.Load(context.Background(), syncDB); err != nil {
 		logrus.WithError(err).Panicf("failed to load notifier ")
@@ -130,8 +144,35 @@ func AddPublicRoutes(
 		logrus.WithError(err).Panicf("failed to start receipts consumer")
 	}
 
+	multiRoomConsumer := consumers.NewOutputMultiRoomDataConsumer(
+		base.ProcessContext, cfg, js, mrq, notifier, streams.MultiRoomStreamProvider,
+	)
+	if err = multiRoomConsumer.Start(); err != nil {
+		logrus.WithError(err).Panicf("failed to start multiroom consumer")
+	}
+
 	routing.Setup(
 		base.PublicClientAPIMux, requestPool, syncDB, userAPI,
 		rsAPI, cfg, base.Caches, base.Fulltext,
 	)
+
+	go func() {
+		ctx := context.Background()
+		for {
+			notify, err := syncDB.ExpirePresence(ctx)
+			if err != nil {
+				logrus.WithError(err).Error("failed to expire presence")
+			}
+			for i := range notify {
+				requestPool.Presence.Store(notify[i].UserID, types.PresenceInternal{
+					Presence: types.PresenceOffline,
+				})
+				notifier.OnNewPresence(types.StreamingToken{
+					PresencePosition: notify[i].StreamPos,
+				}, notify[i].UserID)
+
+			}
+			time.Sleep(types.PresenceExpireInterval)
+		}
+	}()
 }

syncapi/types/multiroom.go

@@ -0,0 +1,21 @@
+package types
+
+type MultiRoom map[string]map[string]MultiRoomData
+
+type MultiRoomContent []byte
+
+type MultiRoomData struct {
+	Content   MultiRoomContent `json:"content"`
+	Timestamp int64            `json:"timestamp"`
+}
+
+func (d MultiRoomContent) MarshalJSON() ([]byte, error) {
+	return d, nil
+}
+
+type MultiRoomDataRow struct {
+	Data      []byte
+	Type      string
+	UserId    string
+	Timestamp int64
+}

syncapi/types/multiroom_test.go

@@ -0,0 +1,21 @@
+package types
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/matryer/is"
+)
+
+func TestMarshallMultiRoom(t *testing.T) {
+	is := is.New(t)
+	m, err := json.Marshal(
+		MultiRoom{
+			"@3:example.com": map[string]MultiRoomData{
+				"location": {
+					Content:   MultiRoomContent(`{"foo":"bar"}`),
+					Timestamp: 123,
+				}}})
+	is.NoErr(err)
+	is.Equal(m, []byte(`{"@3:example.com":{"location":{"content":{"foo":"bar"},"timestamp":123}}}`))
+}

syncapi/types/presence.go

@@ -21,6 +21,12 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
+const (
+	PresenceNoOpMs         = 60_000
+	PresenceExpire         = "'4 minutes'"
+	PresenceExpireInterval = time.Second * 30
+)
+
 type Presence uint8
 
 const (
@@ -66,6 +72,11 @@ type PresenceInternal struct {
 	Presence     Presence                    `json:"-"`
 }
 
+type PresenceNotify struct {
+	StreamPos StreamPosition
+	UserID    string
+}
+
 // Equals compares p1 with p2.
 func (p1 *PresenceInternal) Equals(p2 *PresenceInternal) bool {
 	return p1.ClientFields.Presence == p2.ClientFields.Presence &&

syncapi/types/provider.go

@@ -21,7 +21,8 @@ type SyncRequest struct {
 	WantFullState bool
 
 	// Updated by the PDU stream.
-	Rooms map[string]string
+	Rooms       map[string]string
+	JoinedRooms []string
 	// Updated by the PDU stream.
 	MembershipChanges map[string]struct{}
 	// Updated by the PDU stream.

syncapi/types/types.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -114,6 +115,7 @@ type StreamingToken struct {
 	DeviceListPosition       StreamPosition
 	NotificationDataPosition StreamPosition
 	PresencePosition         StreamPosition
+	MultiRoomDataPosition    StreamPosition
 }
 
 // This will be used as a fallback by json.Marshal.
@@ -129,12 +131,12 @@ func (s *StreamingToken) UnmarshalText(text []byte) (err error) {
 
 func (t StreamingToken) String() string {
 	posStr := fmt.Sprintf(
-		"s%d_%d_%d_%d_%d_%d_%d_%d_%d",
+		"s%d_%d_%d_%d_%d_%d_%d_%d_%d_%d",
 		t.PDUPosition, t.TypingPosition,
 		t.ReceiptPosition, t.SendToDevicePosition,
 		t.InvitePosition, t.AccountDataPosition,
 		t.DeviceListPosition, t.NotificationDataPosition,
-		t.PresencePosition,
+		t.PresencePosition, t.MultiRoomDataPosition,
 	)
 	return posStr
 }
@@ -160,12 +162,14 @@ func (t *StreamingToken) IsAfter(other StreamingToken) bool {
 		return true
 	case t.PresencePosition > other.PresencePosition:
 		return true
+	case t.MultiRoomDataPosition > other.MultiRoomDataPosition:
+		return true
 	}
 	return false
 }
 
 func (t *StreamingToken) IsEmpty() bool {
-	return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition+t.PresencePosition == 0
+	return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition+t.PresencePosition+t.MultiRoomDataPosition == 0
 }
 
 // WithUpdates returns a copy of the StreamingToken with updates applied from another StreamingToken.
@@ -209,6 +213,9 @@ func (t *StreamingToken) ApplyUpdates(other StreamingToken) {
 	if other.PresencePosition > t.PresencePosition {
 		t.PresencePosition = other.PresencePosition
 	}
+	if other.MultiRoomDataPosition > t.MultiRoomDataPosition {
+		t.MultiRoomDataPosition = other.MultiRoomDataPosition
+	}
 }
 
 type TopologyToken struct {
@@ -294,9 +301,11 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) {
 func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 	if len(tok) < 1 {
 		err = ErrMalformedSyncToken
+		logrus.WithField("token", tok).Info("invalid stream token: bad length")
 		return
 	}
 	if tok[0] != SyncTokenTypeStream[0] {
+		logrus.WithField("token", tok).Info("invalid stream token: not starting from s")
 		err = ErrMalformedSyncToken
 		return
 	}
@@ -304,7 +313,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 	// s478_0_0_0_0_13.dl-0-2 but we have now removed partitioned stream positions
 	tok = strings.Split(tok, ".")[0]
 	parts := strings.Split(tok[1:], "_")
-	var positions [9]StreamPosition
+	var positions [10]StreamPosition
 	for i, p := range parts {
 		if i >= len(positions) {
 			break
@@ -312,6 +321,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 		var pos int
 		pos, err = strconv.Atoi(p)
 		if err != nil {
+			logrus.WithField("token", tok).Info("invalid stream token: strconv")
 			err = ErrMalformedSyncToken
 			return
 		}
@@ -327,6 +337,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 		DeviceListPosition:       positions[6],
 		NotificationDataPosition: positions[7],
 		PresencePosition:         positions[8],
+		MultiRoomDataPosition:    positions[9],
 	}
 	return token, nil
 }
@@ -363,6 +374,7 @@ type Response struct {
 	ToDevice            *ToDeviceResponse `json:"to_device,omitempty"`
 	DeviceLists         *DeviceLists      `json:"device_lists,omitempty"`
 	DeviceListsOTKCount map[string]int    `json:"device_one_time_keys_count,omitempty"`
+	MultiRoom           MultiRoom         `json:"multiroom,omitempty"`
 }
 
 func (r Response) MarshalJSON() ([]byte, error) {
@@ -401,7 +413,8 @@ func (r *Response) HasUpdates() bool {
 		len(r.Rooms.Peek) > 0 ||
 		len(r.ToDevice.Events) > 0 ||
 		len(r.DeviceLists.Changed) > 0 ||
-		len(r.DeviceLists.Left) > 0)
+		len(r.DeviceLists.Left) > 0) ||
+		len(r.MultiRoom) > 0
 }
 
 // NewResponse creates an empty response with initialised maps.

syncapi/types/types_test.go

@@ -9,10 +9,10 @@ import (
 
 func TestSyncTokens(t *testing.T) {
 	shouldPass := map[string]string{
-		"s4_0_0_0_0_0_0_0_3": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3}.String(),
+		"s4_0_0_0_0_0_0_0_3_0": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3, 0}.String(),
-		"s3_1_0_0_0_0_2_0_5": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5}.String(),
+		"s3_1_0_0_0_0_2_0_5_1": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5, 1}.String(),
-		"s3_1_2_3_5_0_0_0_6": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6}.String(),
+		"s3_1_2_3_5_0_0_0_6_2": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6, 2}.String(),
-		"t3_1":               TopologyToken{3, 1}.String(),
+		"t3_1":                 TopologyToken{3, 1}.String(),
 	}
 
 	for a, b := range shouldPass {

sytest-blacklist

@@ -49,3 +49,18 @@ Leaves are present in non-gapped incremental syncs
 
 # Below test was passing for the wrong reason, failing correctly since #2858
 New federated private chats get full presence information (SYN-115)
+If a device list update goes missing, the server resyncs on the next one
+# You'll be shocked to discover this is flakey too
+
+Inbound /v1/send_join rejects joins from other servers
+
+# For notifications extension on iOS
+
+/event/ does not allow access to events before the user joined
+
+# Failing after recent updates with presence
+
+Newly joined room includes presence in incremental sync
+User sees their own presence in a sync
+User is offline if they set_presence=offline in their sync
+User sees updates to presence from other users in the incremental sync.

sytest-whitelist

@@ -204,7 +204,6 @@ Deleted tags appear in an incremental v2 /sync
 /event/ on non world readable room does not work
 Outbound federation can query profile data
 /event/ on joined room works
-/event/ does not allow access to events before the user joined
 Federation key API allows unsigned requests for keys
 GET /publicRooms lists rooms
 GET /publicRooms includes avatar URLs

test/db.go

@@ -171,7 +171,6 @@ func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, clo
 func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) {
 	dbs := map[string]DBType{
 		"postgres": DBTypePostgres,
-		"sqlite":   DBTypeSQLite,
 	}
 	for dbName, dbType := range dbs {
 		dbt := dbType

userapi/api/api.go

@@ -533,7 +533,7 @@ type PerformPusherSetRequest struct {
 
 type PerformPusherDeletionRequest struct {
 	Localpart string
-	SessionID int64
+	SessionID int64 // Pusher corresponding to this SessionID will not be deleted
 }
 
 // Pusher represents a push notification subscriber

userapi/api/api_multicast.go

@@ -0,0 +1,6 @@
+package api
+
+type MulticastMetadata struct {
+	ExpireMs       int
+	ExcludeRoomIds []string
+}

userapi/internal/api.go

@@ -613,14 +613,25 @@ func (a *UserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *a
 		return err
 	}
 
+	threepids, err := a.DB.GetThreePIDsForLocalpart(ctx, req.Localpart)
+	if err != nil {
+		return err
+	}
+	for i := 0; i < len(threepids); i++ {
+		err = a.DB.RemoveThreePIDAssociation(ctx, threepids[i].Address, threepids[i].Medium)
+		if err != nil {
+			return err
+		}
+	}
+
 	pusherReq := &api.PerformPusherDeletionRequest{
 		Localpart: req.Localpart,
 	}
-	if err := a.PerformPusherDeletion(ctx, pusherReq, &struct{}{}); err != nil {
+	if err = a.PerformPusherDeletion(ctx, pusherReq, &struct{}{}); err != nil {
 		return err
 	}
 
-	err := a.DB.DeactivateAccount(ctx, req.Localpart)
+	err = a.DB.DeactivateAccount(ctx, req.Localpart)
 	res.AccountDeactivated = err == nil
 	return err
 }

userapi/storage/postgres/accounts_table.go

@@ -98,6 +98,11 @@ func NewPostgresAccountsTable(db *sql.DB, serverName gomatrixserverlib.ServerNam
 			Up:      deltas.UpAddAccountType,
 			Down:    deltas.DownAddAccountType,
 		},
+		{
+			Version: "userapi: no guests",
+			Up:      deltas.UpNoGuests,
+			Down:    deltas.DownNoGuests,
+		},
 	}...)
 	err = m.Up(context.Background())
 	if err != nil {

userapi/storage/postgres/deltas/2022080800000000_no_guests.go

@@ -0,0 +1,20 @@
+package deltas
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+)
+
+func UpNoGuests(ctx context.Context, tx *sql.Tx) error {
+	// AddAccountType introduced a bug where each user that had was registered as a regular user, but without user_id, became a guest.
+	_, err := tx.ExecContext(ctx, "UPDATE userapi_accounts SET account_type = 1 WHERE account_type = 2;")
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownNoGuests(ctx context.Context, tx *sql.Tx) error {
+	return nil
+}

userapi/storage/postgres/notifications_table.go

@@ -72,7 +72,7 @@ const selectNotificationSQL = "" +
 	") AND NOT read ORDER BY localpart, id LIMIT $4"
 
 const selectNotificationCountSQL = "" +
-	"SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND (" +
+	"SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND (" +
 	"(($2 & 1) <> 0 AND highlight) OR (($2 & 2) <> 0 AND NOT highlight)" +
 	") AND NOT read"
 

userapi/storage/shared/storage.go

@@ -565,7 +565,7 @@ func (d *Database) CreateDevice(
 	ctx context.Context, localpart string, deviceID *string, accessToken string,
 	displayName *string, ipAddr, userAgent string,
 ) (dev *api.Device, returnErr error) {
-	if deviceID != nil {
+	if deviceID != nil && *deviceID != "" {
 		returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 			var err error
 			// Revoke existing tokens for this device

userapi/storage/sqlite3/notifications_table.go

@@ -72,7 +72,7 @@ const selectNotificationSQL = "" +
 	") AND NOT read ORDER BY localpart, id LIMIT $4"
 
 const selectNotificationCountSQL = "" +
-	"SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND (" +
+	"SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND (" +
 	"(($2 & 1) <> 0 AND highlight) OR (($2 & 2) <> 0 AND NOT highlight)" +
 	") AND NOT read"
 

userapi/storage/storage_test.go

@@ -128,6 +128,11 @@ func Test_Accounts(t *testing.T) {
 		_, err = db.GetAccountByPassword(ctx, aliceLocalpart, "newPassword")
 		assert.Error(t, err, "expected an error, got none")
 
+		// This should return an empty slice, as the account is deactivated and the 3pid is unbound
+		threepids, err := db.GetThreePIDsForLocalpart(ctx, aliceLocalpart)
+		assert.NoError(t, err, "failed to get 3pid for account")
+		assert.Equal(t, len(threepids), 0)
+
 		_, err = db.GetAccountByLocalpart(ctx, "unusename")
 		assert.Error(t, err, "expected an error for non existent localpart")
 
@@ -533,7 +538,7 @@ func Test_Notification(t *testing.T) {
 		// get notifications
 		count, err := db.GetNotificationCount(ctx, aliceLocalpart, tables.AllNotifications)
 		assert.NoError(t, err, "unable to get notification count")
-		assert.Equal(t, int64(10), count)
+		assert.Equal(t, int64(2), count)
 		notifs, count, err := db.GetNotifications(ctx, aliceLocalpart, 0, 15, tables.AllNotifications)
 		assert.NoError(t, err, "unable to get notifications")
 		assert.Equal(t, int64(10), count)