From e6c992ba8bcbc59706d6dd55db3b237ebbdc8e7e Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 4 Oct 2022 11:41:06 +0200
Subject: [PATCH 01/42] Update Dockerfile (#2342)

Updates/adds a new multistage (build-kit) Dockerfile. (if accepted,
could make `Dockerfile.monolith` and `Dockerfile.polylith` in
`build/docker` obsolete)
There's no huge difference between the dockerfiles, except this uses a
non-root user when running the container, also doesn't copy the working
directory to the image when building.
Also adds vulnerabilities scans using
[Trivy](https://github.com/aquasecurity/trivy) for the created docker
images. (untested)

Building images is done using
```
docker build . --target image-monolith -t dendrite-monolith
docker build . --target image-polylith -t dendrite-polylith
```

As noted in the comments, only adds `dendrite-polylith-multi` to the
polylith image and all required binaries to the monolith image.
Probably needs some docs updating, if this is accepted.

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 .github/workflows/dendrite.yml                |   6 +-
 .github/workflows/docker.yml                  | 106 +++++++++++++-----
 Dockerfile                                    |  99 ++++++++++++++++
 .../dendritejs-pinecone/jsServer.go           |   0
 {cmd => build}/dendritejs-pinecone/main.go    |   0
 .../dendritejs-pinecone/main_noop.go          |   0
 .../dendritejs-pinecone/main_test.go          |   0
 build/docker/Dockerfile.demo-pinecone         |  25 -----
 build/docker/Dockerfile.monolith              |  25 -----
 build/docker/Dockerfile.polylith              |  25 -----
 build/docker/README.md                        |  14 ++-
 build/docker/crossbuild.sh                    |  67 +++++++++++
 build/docker/images-build.sh                  |   5 +-
 13 files changed, 257 insertions(+), 115 deletions(-)
 create mode 100644 Dockerfile
 rename {cmd => build}/dendritejs-pinecone/jsServer.go (100%)
 rename {cmd => build}/dendritejs-pinecone/main.go (100%)
 rename {cmd => build}/dendritejs-pinecone/main_noop.go (100%)
 rename {cmd => build}/dendritejs-pinecone/main_test.go (100%)
 delete mode 100644 build/docker/Dockerfile.demo-pinecone
 delete mode 100644 build/docker/Dockerfile.monolith
 delete mode 100644 build/docker/Dockerfile.polylith
 create mode 100644 build/docker/crossbuild.sh

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index be3c7c173..524d36039 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -284,7 +284,7 @@ jobs:
         API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Run Sytest
         run: /bootstrap.sh dendrite
         working-directory: /src
@@ -344,8 +344,8 @@ jobs:
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
 
-      - name: Run actions/checkout@v2 for dendrite
-        uses: actions/checkout@v2
+      - name: Run actions/checkout@v3 for dendrite
+        uses: actions/checkout@v3
         with:
           path: dendrite
 
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b4e24e52f..358037c02 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -21,26 +21,32 @@ jobs:
   monolith:
     name: Monolith image
     runs-on: ubuntu-latest
+    needs: build-flags
     permissions:
       contents: read
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get release tag
+        uses: actions/checkout@v3
+      - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -49,27 +55,41 @@ jobs:
       - name: Build main monolith image
         if: github.ref_name == 'main'
         id: docker_build_monolith
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.monolith
+          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
+          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
             ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
 
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: "trivy-results.sarif"
+
       - name: Build release monolith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_monolith_release
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.monolith
+          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
+          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -81,26 +101,32 @@ jobs:
   polylith:
     name: Polylith image
     runs-on: ubuntu-latest
+    needs: build-flags
     permissions:
       contents: read
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get release tag
+        uses: actions/checkout@v3
+      - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -109,27 +135,40 @@ jobs:
       - name: Build main polylith image
         if: github.ref_name == 'main'
         id: docker_build_polylith
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.polylith
+          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
+          target: polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
             ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
 
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: "trivy-results.sarif"
+
       - name: Build release polylith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_polylith_release
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.polylith
+          target: polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -141,26 +180,32 @@ jobs:
   demo-pinecone:
     name: Pinecone demo image
     runs-on: ubuntu-latest
+    needs: build-flags
     permissions:
       contents: read
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get release tag
+        uses: actions/checkout@v3
+      - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -169,12 +214,12 @@ jobs:
       - name: Build main pinecone demo image
         if: github.ref_name == 'main'
         id: docker_build_demo_pinecone
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.demo-pinecone
+          target: demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -184,12 +229,13 @@ jobs:
       - name: Build release pinecone demo image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_demo_pinecone_release
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.demo-pinecone
+          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
+          target: demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 000000000..bf5831832
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,99 @@
+#syntax=docker/dockerfile:1.2
+
+#
+# base installs required dependencies and runs go mod download to cache dependencies
+#
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base
+RUN apk --update --no-cache add bash build-base curl
+
+#
+# build creates all needed binaries
+#
+FROM base AS build
+WORKDIR /src
+ARG TARGETOS
+ARG TARGETARCH
+ARG FLAGS
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cache/go-build \
+    sh ./build/docker/crossbuild.sh
+
+#
+# The dendrite base image; mainly creates a user and switches to it
+#
+FROM alpine:latest AS dendrite-base
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
+LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
+RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D
+USER dendrite
+WORKDIR /home/dendrite
+
+#
+# Builds the polylith image and only contains the polylith binary
+#
+FROM dendrite-base AS polylith
+LABEL org.opencontainers.image.title="Dendrite (Polylith)"
+
+COPY --from=build /out/dendrite-polylith-multi /usr/bin/
+
+ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
+
+#
+# Builds the monolith image and contains all required binaries
+#
+FROM dendrite-base AS monolith
+LABEL org.opencontainers.image.title="Dendrite (Monolith)"
+
+COPY --from=build /out/create-account /usr/bin/create-account
+COPY --from=build /out/generate-config /usr/bin/generate-config
+COPY --from=build /out/generate-keys /usr/bin/generate-keys
+COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
+
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
+EXPOSE 8008 8448
+
+#
+# Builds the P2P demo image and contains all required binaries
+#
+FROM dendrite-base AS demo-pinecone
+LABEL org.opencontainers.image.title="Dendrite (P2P Demo)"
+
+COPY --from=build /out/create-account /usr/bin/create-account
+COPY --from=build /out/generate-config /usr/bin/generate-config
+COPY --from=build /out/generate-keys /usr/bin/generate-keys
+COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
+EXPOSE 8008 8448
+
+#
+# Builds the Complement image, used for integration tests
+#
+FROM base AS complement
+LABEL org.opencontainers.image.title="Dendrite (Complement)"
+RUN apk add --no-cache sqlite openssl ca-certificates
+COPY --from=build /out/* /usr/bin/
+RUN rm /usr/bin/dendrite-polylith-multi /usr/bin/dendrite-demo* /usr/bin/dendritejs-pinecone
+
+WORKDIR /dendrite
+RUN /usr/bin/generate-keys --private-key matrix_key.pem && \
+    mkdir /ca && \
+    openssl genrsa -out /ca/ca.key 2048 && \
+    openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt
+
+ENV SERVER_NAME=localhost
+ENV API=0
+EXPOSE 8008 8448
+
+# At runtime, generate TLS cert based on the CA now mounted at /ca
+# At runtime, replace the SERVER_NAME with what we are told
+CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \
+    /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
+    cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
+    /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
diff --git a/cmd/dendritejs-pinecone/jsServer.go b/build/dendritejs-pinecone/jsServer.go
similarity index 100%
rename from cmd/dendritejs-pinecone/jsServer.go
rename to build/dendritejs-pinecone/jsServer.go
diff --git a/cmd/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go
similarity index 100%
rename from cmd/dendritejs-pinecone/main.go
rename to build/dendritejs-pinecone/main.go
diff --git a/cmd/dendritejs-pinecone/main_noop.go b/build/dendritejs-pinecone/main_noop.go
similarity index 100%
rename from cmd/dendritejs-pinecone/main_noop.go
rename to build/dendritejs-pinecone/main_noop.go
diff --git a/cmd/dendritejs-pinecone/main_test.go b/build/dendritejs-pinecone/main_test.go
similarity index 100%
rename from cmd/dendritejs-pinecone/main_test.go
rename to build/dendritejs-pinecone/main_test.go
diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone
deleted file mode 100644
index 133c63c53..000000000
--- a/build/docker/Dockerfile.demo-pinecone
+++ /dev/null
@@ -1,25 +0,0 @@
-FROM docker.io/golang:1.19-alpine AS base
-
-RUN apk --update --no-cache add bash build-base
-
-WORKDIR /build
-
-COPY . /build
-
-RUN mkdir -p bin
-RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone
-RUN go build -trimpath -o bin/ ./cmd/create-account
-RUN go build -trimpath -o bin/ ./cmd/generate-keys
-
-FROM alpine:latest
-LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
-LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
-LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-
-COPY --from=base /build/bin/* /usr/bin/
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith
deleted file mode 100644
index 3180e9626..000000000
--- a/build/docker/Dockerfile.monolith
+++ /dev/null
@@ -1,25 +0,0 @@
-FROM docker.io/golang:1.19-alpine AS base
-
-RUN apk --update --no-cache add bash build-base
-
-WORKDIR /build
-
-COPY . /build
-
-RUN mkdir -p bin
-RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
-RUN go build -trimpath -o bin/ ./cmd/create-account
-RUN go build -trimpath -o bin/ ./cmd/generate-keys
-
-FROM alpine:latest
-LABEL org.opencontainers.image.title="Dendrite (Monolith)"
-LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
-LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-
-COPY --from=base /build/bin/* /usr/bin/
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith
deleted file mode 100644
index 79f8a5f23..000000000
--- a/build/docker/Dockerfile.polylith
+++ /dev/null
@@ -1,25 +0,0 @@
-FROM docker.io/golang:1.19-alpine AS base
-
-RUN apk --update --no-cache add bash build-base
-
-WORKDIR /build
-
-COPY . /build
-
-RUN mkdir -p bin
-RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
-RUN go build -trimpath -o bin/ ./cmd/create-account
-RUN go build -trimpath -o bin/ ./cmd/generate-keys
-
-FROM alpine:latest
-LABEL org.opencontainers.image.title="Dendrite (Polylith)"
-LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
-LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-
-COPY --from=base /build/bin/* /usr/bin/
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
diff --git a/build/docker/README.md b/build/docker/README.md
index 261519fde..14a9c8594 100644
--- a/build/docker/README.md
+++ b/build/docker/README.md
@@ -9,11 +9,15 @@ They can be found on Docker Hub:
 
 ## Dockerfiles
 
-The `Dockerfile` builds the base image which contains all of the Dendrite
-components. The `Dockerfile.component` file takes the given component, as
-specified with `--buildarg component=` from the base image and produce
-smaller component-specific images, which are substantially smaller and do
-not contain the Go toolchain etc.
+The `Dockerfile` is a multistage file which can build all three Dendrite
+images depending on the supplied `--target`. From the root of the Dendrite
+repository, run:
+
+```
+docker build . --target monolith -t matrixdotorg/dendrite-monolith
+docker build . --target polylith -t matrixdotorg/dendrite-monolith
+docker build . --target demo-pinecone -t matrixdotorg/dendrite-monolith
+```
 
 ## Compose files
 
diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh
new file mode 100644
index 000000000..46e5d7e9b
--- /dev/null
+++ b/build/docker/crossbuild.sh
@@ -0,0 +1,67 @@
+#!/bin/sh
+set -e
+
+# In order to cross-compile with the multi-stage Docker builds, we need to
+# ensure that the suitable toolchain for cross-compiling is installed. Since
+# the images are Alpine-based, we will use musl. Download and install the
+# toolchain inside the build container.
+
+USERARCH=`go env GOARCH`
+GOARCH="$TARGETARCH"
+GOOS="linux"
+
+echo "Target arch: $TARGETARCH"
+echo "User arch:   $USERARCH"
+
+if [ "$TARGETARCH" != "$USERARCH" ]; then
+    if [ "$USERARCH" != "amd64" ]; then
+      echo "Cross-compiling only supported on amd64"
+      exit 1
+    fi
+
+    echo "Cross compile"
+    case $GOARCH in
+    arm64)
+        curl -s https://more.musl.cc/x86_64-linux-musl/aarch64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=aarch64-linux-musl-gcc
+        ;;
+
+    amd64)
+        curl -s https://more.musl.cc/x86_64-linux-musl/x86_64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=x86_64-linux-musl-gcc
+        ;;
+
+    386)
+        curl -s https://more.musl.cc/x86_64-linux-musl/i686-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=i686-linux-musl-gcc
+        ;;
+
+    arm)
+        curl -s https://more.musl.cc/x86_64-linux-musl/armv7l-linux-musleabihf-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=armv7l-linux-musleabihf-gcc
+        ;;
+
+    s390x)
+        curl -s https://more.musl.cc/x86_64-linux-musl/s390x-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=s390x-linux-musl-gcc
+        ;;
+
+    ppc64le)
+        curl -s https://more.musl.cc/x86_64-linux-musl/powerpc64le-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=powerpc64le-linux-musl-gcc
+        ;;
+
+    *)
+        echo "Unsupported GOARCH=${GOARCH}"
+        exit 1
+        ;;
+    esac
+else
+    echo "Native compile"
+fi
+
+# Output the go environment just in case it is useful for debugging.
+go env
+
+# Build Dendrite and tools, statically linking them.
+CGO_ENABLED=1 go build -v -ldflags="-linkmode external -extldflags -static ${FLAGS}" -trimpath -o /out/ ./cmd/...
diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh
index c2c140685..1a8326151 100755
--- a/build/docker/images-build.sh
+++ b/build/docker/images-build.sh
@@ -6,5 +6,6 @@ TAG=${1:-latest}
 
 echo "Building tag '${TAG}'"
 
-docker build -t matrixdotorg/dendrite-monolith:${TAG}  -f build/docker/Dockerfile.monolith .
-docker build -t matrixdotorg/dendrite-polylith:${TAG}  -f build/docker/Dockerfile.polylith .
\ No newline at end of file
+docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG}
+docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG}
+docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG}
\ No newline at end of file

From ede4632835929bf93d0cb57df58c06efd94fa7e0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 10:43:58 +0100
Subject: [PATCH 02/42] Fix Docker GHA

---
 .github/workflows/docker.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 358037c02..c15b6c1a4 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -21,7 +21,6 @@ jobs:
   monolith:
     name: Monolith image
     runs-on: ubuntu-latest
-    needs: build-flags
     permissions:
       contents: read
       packages: write
@@ -101,7 +100,6 @@ jobs:
   polylith:
     name: Polylith image
     runs-on: ubuntu-latest
-    needs: build-flags
     permissions:
       contents: read
       packages: write
@@ -180,7 +178,6 @@ jobs:
   demo-pinecone:
     name: Pinecone demo image
     runs-on: ubuntu-latest
-    needs: build-flags
     permissions:
       contents: read
       packages: write

From 98b73652e00edb8304aa4a3471aa29151f025297 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 11:07:54 +0100
Subject: [PATCH 03/42] Try to populate `-ldflags` in Docker builds

---
 .github/workflows/docker.yml | 6 +++---
 build/docker/crossbuild.sh   | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index c15b6c1a4..e0db280fa 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -35,6 +35,7 @@ jobs:
           BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
           [ ${BRANCH} == "main" ] && BRANCH=""
           echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
+          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -59,7 +60,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
           target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -114,6 +114,7 @@ jobs:
           BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
           [ ${BRANCH} == "main" ] && BRANCH=""
           echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
+          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -138,7 +139,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
           target: polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -192,6 +192,7 @@ jobs:
           BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
           [ ${BRANCH} == "main" ] && BRANCH=""
           echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
+          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -231,7 +232,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
           target: demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh
index 46e5d7e9b..957893a0a 100644
--- a/build/docker/crossbuild.sh
+++ b/build/docker/crossbuild.sh
@@ -12,6 +12,7 @@ GOOS="linux"
 
 echo "Target arch: $TARGETARCH"
 echo "User arch:   $USERARCH"
+echo "Link flags:  $FLAGS"
 
 if [ "$TARGETARCH" != "$USERARCH" ]; then
     if [ "$USERARCH" != "amd64" ]; then

From 085bf5e28b49948209b464c5b12c02775e6c1d70 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 11:33:05 +0100
Subject: [PATCH 04/42] Revert Docker changes

---
 .github/workflows/dendrite.yml                |   6 +-
 .github/workflows/docker.yml                  | 103 +++++-------------
 Dockerfile                                    |  99 -----------------
 build/docker/Dockerfile.demo-pinecone         |  25 +++++
 build/docker/Dockerfile.monolith              |  25 +++++
 build/docker/Dockerfile.polylith              |  25 +++++
 build/docker/README.md                        |  14 +--
 build/docker/crossbuild.sh                    |  68 ------------
 build/docker/images-build.sh                  |   5 +-
 .../dendritejs-pinecone/jsServer.go           |   0
 {build => cmd}/dendritejs-pinecone/main.go    |   0
 .../dendritejs-pinecone/main_noop.go          |   0
 .../dendritejs-pinecone/main_test.go          |   0
 13 files changed, 115 insertions(+), 255 deletions(-)
 delete mode 100644 Dockerfile
 create mode 100644 build/docker/Dockerfile.demo-pinecone
 create mode 100644 build/docker/Dockerfile.monolith
 create mode 100644 build/docker/Dockerfile.polylith
 delete mode 100644 build/docker/crossbuild.sh
 rename {build => cmd}/dendritejs-pinecone/jsServer.go (100%)
 rename {build => cmd}/dendritejs-pinecone/main.go (100%)
 rename {build => cmd}/dendritejs-pinecone/main_noop.go (100%)
 rename {build => cmd}/dendritejs-pinecone/main_test.go (100%)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 524d36039..be3c7c173 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -284,7 +284,7 @@ jobs:
         API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Run Sytest
         run: /bootstrap.sh dendrite
         working-directory: /src
@@ -344,8 +344,8 @@ jobs:
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
 
-      - name: Run actions/checkout@v3 for dendrite
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v2 for dendrite
+        uses: actions/checkout@v2
         with:
           path: dendrite
 
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index e0db280fa..b4e24e52f 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -26,27 +26,21 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
+        uses: actions/checkout@v2
+      - name: Get release tag
         if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -55,40 +49,27 @@ jobs:
       - name: Build main monolith image
         if: github.ref_name == 'main'
         id: docker_build_monolith
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: monolith
+          file: ./build/docker/Dockerfile.monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
             ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v1
-        with:
-          sarif_file: "trivy-results.sarif"
-
       - name: Build release monolith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_monolith_release
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
-          target: monolith
+          file: ./build/docker/Dockerfile.monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -105,27 +86,21 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
+        uses: actions/checkout@v2
+      - name: Get release tag
         if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -134,39 +109,27 @@ jobs:
       - name: Build main polylith image
         if: github.ref_name == 'main'
         id: docker_build_polylith
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: polylith
+          file: ./build/docker/Dockerfile.polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
             ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v1
-        with:
-          sarif_file: "trivy-results.sarif"
-
       - name: Build release polylith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_polylith_release
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: polylith
+          file: ./build/docker/Dockerfile.polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -183,27 +146,21 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
+        uses: actions/checkout@v2
+      - name: Get release tag
         if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -212,12 +169,12 @@ jobs:
       - name: Build main pinecone demo image
         if: github.ref_name == 'main'
         id: docker_build_demo_pinecone
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: demo-pinecone
+          file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -227,12 +184,12 @@ jobs:
       - name: Build release pinecone demo image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_demo_pinecone_release
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: demo-pinecone
+          file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index bf5831832..000000000
--- a/Dockerfile
+++ /dev/null
@@ -1,99 +0,0 @@
-#syntax=docker/dockerfile:1.2
-
-#
-# base installs required dependencies and runs go mod download to cache dependencies
-#
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base
-RUN apk --update --no-cache add bash build-base curl
-
-#
-# build creates all needed binaries
-#
-FROM base AS build
-WORKDIR /src
-ARG TARGETOS
-ARG TARGETARCH
-ARG FLAGS
-RUN --mount=target=. \
-    --mount=type=cache,target=/root/.cache/go-build \
-    sh ./build/docker/crossbuild.sh
-
-#
-# The dendrite base image; mainly creates a user and switches to it
-#
-FROM alpine:latest AS dendrite-base
-LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
-LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
-LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
-RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D
-USER dendrite
-WORKDIR /home/dendrite
-
-#
-# Builds the polylith image and only contains the polylith binary
-#
-FROM dendrite-base AS polylith
-LABEL org.opencontainers.image.title="Dendrite (Polylith)"
-
-COPY --from=build /out/dendrite-polylith-multi /usr/bin/
-
-ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
-
-#
-# Builds the monolith image and contains all required binaries
-#
-FROM dendrite-base AS monolith
-LABEL org.opencontainers.image.title="Dendrite (Monolith)"
-
-COPY --from=build /out/create-account /usr/bin/create-account
-COPY --from=build /out/generate-config /usr/bin/generate-config
-COPY --from=build /out/generate-keys /usr/bin/generate-keys
-COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
-
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
-EXPOSE 8008 8448
-
-#
-# Builds the P2P demo image and contains all required binaries
-#
-FROM dendrite-base AS demo-pinecone
-LABEL org.opencontainers.image.title="Dendrite (P2P Demo)"
-
-COPY --from=build /out/create-account /usr/bin/create-account
-COPY --from=build /out/generate-config /usr/bin/generate-config
-COPY --from=build /out/generate-keys /usr/bin/generate-keys
-COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
-EXPOSE 8008 8448
-
-#
-# Builds the Complement image, used for integration tests
-#
-FROM base AS complement
-LABEL org.opencontainers.image.title="Dendrite (Complement)"
-RUN apk add --no-cache sqlite openssl ca-certificates
-COPY --from=build /out/* /usr/bin/
-RUN rm /usr/bin/dendrite-polylith-multi /usr/bin/dendrite-demo* /usr/bin/dendritejs-pinecone
-
-WORKDIR /dendrite
-RUN /usr/bin/generate-keys --private-key matrix_key.pem && \
-    mkdir /ca && \
-    openssl genrsa -out /ca/ca.key 2048 && \
-    openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt
-
-ENV SERVER_NAME=localhost
-ENV API=0
-EXPOSE 8008 8448
-
-# At runtime, generate TLS cert based on the CA now mounted at /ca
-# At runtime, replace the SERVER_NAME with what we are told
-CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \
-    /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
-    cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone
new file mode 100644
index 000000000..133c63c53
--- /dev/null
+++ b/build/docker/Dockerfile.demo-pinecone
@@ -0,0 +1,25 @@
+FROM docker.io/golang:1.19-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
+
+FROM alpine:latest
+LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+
+COPY --from=base /build/bin/* /usr/bin/
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith
new file mode 100644
index 000000000..3180e9626
--- /dev/null
+++ b/build/docker/Dockerfile.monolith
@@ -0,0 +1,25 @@
+FROM docker.io/golang:1.19-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
+
+FROM alpine:latest
+LABEL org.opencontainers.image.title="Dendrite (Monolith)"
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+
+COPY --from=base /build/bin/* /usr/bin/
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith
new file mode 100644
index 000000000..79f8a5f23
--- /dev/null
+++ b/build/docker/Dockerfile.polylith
@@ -0,0 +1,25 @@
+FROM docker.io/golang:1.19-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
+
+FROM alpine:latest
+LABEL org.opencontainers.image.title="Dendrite (Polylith)"
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+
+COPY --from=base /build/bin/* /usr/bin/
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
diff --git a/build/docker/README.md b/build/docker/README.md
index 14a9c8594..261519fde 100644
--- a/build/docker/README.md
+++ b/build/docker/README.md
@@ -9,15 +9,11 @@ They can be found on Docker Hub:
 
 ## Dockerfiles
 
-The `Dockerfile` is a multistage file which can build all three Dendrite
-images depending on the supplied `--target`. From the root of the Dendrite
-repository, run:
-
-```
-docker build . --target monolith -t matrixdotorg/dendrite-monolith
-docker build . --target polylith -t matrixdotorg/dendrite-monolith
-docker build . --target demo-pinecone -t matrixdotorg/dendrite-monolith
-```
+The `Dockerfile` builds the base image which contains all of the Dendrite
+components. The `Dockerfile.component` file takes the given component, as
+specified with `--buildarg component=` from the base image and produce
+smaller component-specific images, which are substantially smaller and do
+not contain the Go toolchain etc.
 
 ## Compose files
 
diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh
deleted file mode 100644
index 957893a0a..000000000
--- a/build/docker/crossbuild.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/sh
-set -e
-
-# In order to cross-compile with the multi-stage Docker builds, we need to
-# ensure that the suitable toolchain for cross-compiling is installed. Since
-# the images are Alpine-based, we will use musl. Download and install the
-# toolchain inside the build container.
-
-USERARCH=`go env GOARCH`
-GOARCH="$TARGETARCH"
-GOOS="linux"
-
-echo "Target arch: $TARGETARCH"
-echo "User arch:   $USERARCH"
-echo "Link flags:  $FLAGS"
-
-if [ "$TARGETARCH" != "$USERARCH" ]; then
-    if [ "$USERARCH" != "amd64" ]; then
-      echo "Cross-compiling only supported on amd64"
-      exit 1
-    fi
-
-    echo "Cross compile"
-    case $GOARCH in
-    arm64)
-        curl -s https://more.musl.cc/x86_64-linux-musl/aarch64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=aarch64-linux-musl-gcc
-        ;;
-
-    amd64)
-        curl -s https://more.musl.cc/x86_64-linux-musl/x86_64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=x86_64-linux-musl-gcc
-        ;;
-
-    386)
-        curl -s https://more.musl.cc/x86_64-linux-musl/i686-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=i686-linux-musl-gcc
-        ;;
-
-    arm)
-        curl -s https://more.musl.cc/x86_64-linux-musl/armv7l-linux-musleabihf-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=armv7l-linux-musleabihf-gcc
-        ;;
-
-    s390x)
-        curl -s https://more.musl.cc/x86_64-linux-musl/s390x-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=s390x-linux-musl-gcc
-        ;;
-
-    ppc64le)
-        curl -s https://more.musl.cc/x86_64-linux-musl/powerpc64le-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=powerpc64le-linux-musl-gcc
-        ;;
-
-    *)
-        echo "Unsupported GOARCH=${GOARCH}"
-        exit 1
-        ;;
-    esac
-else
-    echo "Native compile"
-fi
-
-# Output the go environment just in case it is useful for debugging.
-go env
-
-# Build Dendrite and tools, statically linking them.
-CGO_ENABLED=1 go build -v -ldflags="-linkmode external -extldflags -static ${FLAGS}" -trimpath -o /out/ ./cmd/...
diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh
index 1a8326151..c2c140685 100755
--- a/build/docker/images-build.sh
+++ b/build/docker/images-build.sh
@@ -6,6 +6,5 @@ TAG=${1:-latest}
 
 echo "Building tag '${TAG}'"
 
-docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG}
-docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG}
-docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG}
\ No newline at end of file
+docker build -t matrixdotorg/dendrite-monolith:${TAG}  -f build/docker/Dockerfile.monolith .
+docker build -t matrixdotorg/dendrite-polylith:${TAG}  -f build/docker/Dockerfile.polylith .
\ No newline at end of file
diff --git a/build/dendritejs-pinecone/jsServer.go b/cmd/dendritejs-pinecone/jsServer.go
similarity index 100%
rename from build/dendritejs-pinecone/jsServer.go
rename to cmd/dendritejs-pinecone/jsServer.go
diff --git a/build/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go
similarity index 100%
rename from build/dendritejs-pinecone/main.go
rename to cmd/dendritejs-pinecone/main.go
diff --git a/build/dendritejs-pinecone/main_noop.go b/cmd/dendritejs-pinecone/main_noop.go
similarity index 100%
rename from build/dendritejs-pinecone/main_noop.go
rename to cmd/dendritejs-pinecone/main_noop.go
diff --git a/build/dendritejs-pinecone/main_test.go b/cmd/dendritejs-pinecone/main_test.go
similarity index 100%
rename from build/dendritejs-pinecone/main_test.go
rename to cmd/dendritejs-pinecone/main_test.go

From a767102f8a74031b5d0612d985c7589dfe0e0821 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 11:34:52 +0100
Subject: [PATCH 05/42] Reduce `max_open_conns` in monolith sample config

---
 dendrite-sample.monolith.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index e41e83d7c..f0fa386d1 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -37,7 +37,7 @@ global:
   # you must configure the "database" block for each component instead.
   database:
     connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 90
     max_idle_conns: 5
     conn_max_lifetime: -1
 

From 3da182212e86daca1d7019efbd424b2bd38b8b3c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 13:02:41 +0100
Subject: [PATCH 06/42] Track reasons why the process is in a degraded state

---
 setup/base/base.go       |  8 +++++++-
 setup/jetstream/nats.go  |  6 +++---
 setup/process/process.go | 35 +++++++++++++++++++++++------------
 3 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/setup/base/base.go b/setup/base/base.go
index 0636c7b8d..2e3a3a195 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"database/sql"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -467,8 +468,13 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		w.WriteHeader(200)
 	})
 	b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) {
-		if b.ProcessContext.IsDegraded() {
+		if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded {
 			w.WriteHeader(503)
+			_ = json.NewEncoder(w).Encode(struct {
+				Warnings []string `json:"warnings"`
+			}{
+				Warnings: reasons,
+			})
 			return
 		}
 		w.WriteHeader(200)
diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index 7409fd6c8..af4eb2949 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -169,9 +169,9 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
 					// We've managed to add the stream in memory.  What's on the
 					// disk will be left alone, but our ability to recover from a
 					// future crash will be limited. Yell about it.
-					sentry.CaptureException(fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible", namespaced.Name))
-					logrus.Warn("Stream is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible")
-					process.Degraded()
+					err := fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory", namespaced.Name)
+					sentry.CaptureException(err)
+					process.Degraded(err)
 				}
 			}
 		}
diff --git a/setup/process/process.go b/setup/process/process.go
index 06ef60217..b2d2844a8 100644
--- a/setup/process/process.go
+++ b/setup/process/process.go
@@ -2,19 +2,18 @@ package process
 
 import (
 	"context"
-	"fmt"
 	"sync"
 
 	"github.com/getsentry/sentry-go"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 )
 
 type ProcessContext struct {
-	wg       *sync.WaitGroup    // used to wait for components to shutdown
-	ctx      context.Context    // cancelled when Stop is called
-	shutdown context.CancelFunc // shut down Dendrite
-	degraded atomic.Bool
+	mu       sync.RWMutex
+	wg       *sync.WaitGroup     // used to wait for components to shutdown
+	ctx      context.Context     // cancelled when Stop is called
+	shutdown context.CancelFunc  // shut down Dendrite
+	degraded map[string]struct{} // reasons why the process is degraded
 }
 
 func NewProcessContext() *ProcessContext {
@@ -50,13 +49,25 @@ func (b *ProcessContext) WaitForComponentsToFinish() {
 	b.wg.Wait()
 }
 
-func (b *ProcessContext) Degraded() {
-	if b.degraded.CompareAndSwap(false, true) {
-		logrus.Warn("Dendrite is running in a degraded state")
-		sentry.CaptureException(fmt.Errorf("Process is running in a degraded state"))
+func (b *ProcessContext) Degraded(err error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if _, ok := b.degraded[err.Error()]; !ok {
+		logrus.WithError(err).Warn("Dendrite has entered a degraded state")
+		sentry.CaptureException(err)
+		b.degraded[err.Error()] = struct{}{}
 	}
 }
 
-func (b *ProcessContext) IsDegraded() bool {
-	return b.degraded.Load()
+func (b *ProcessContext) IsDegraded() (bool, []string) {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+	if len(b.degraded) == 0 {
+		return false, nil
+	}
+	reasons := make([]string, 0, len(b.degraded))
+	for reason := range b.degraded {
+		reasons = append(reasons, reason)
+	}
+	return true, reasons
 }

From ae10aac456e90dc6a3ea56ca4aceb4a4e5aa8e04 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 15:40:04 +0100
Subject: [PATCH 07/42] Don't perform a federated join after invite if we are
 already joined to the room (#2762)

If we are already joined to the room then it shouldn't matter if you
were invited or not, so this looks like a bug.
---
 roomserver/internal/perform/perform_join.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index 43be54beb..167b375b7 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -237,7 +237,7 @@ func (r *Joiner) performJoinRoomByID(
 	// Force a federated join if we're dealing with a pending invite
 	// and we aren't in the room.
 	isInvitePending, inviteSender, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID)
-	if err == nil && isInvitePending {
+	if err == nil && !serverInRoom && isInvitePending {
 		_, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender)
 		if ierr != nil {
 			return "", "", fmt.Errorf("gomatrixserverlib.SplitID: %w", err)

From 21f88819858dec022dec5919d6db5554605a9b8b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 16:43:10 +0100
Subject: [PATCH 08/42] Add indexes that optimise `selectStateInRangeSQL`
 (#2764)

This gets rid of some expensive scans on `add_state_ids` and
`remove_state_ids`, turning them into much cheaper and faster index
scans instead.
---
 syncapi/storage/postgres/output_room_events_table.go | 2 ++
 syncapi/storage/sqlite3/output_room_events_table.go  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index cb092150d..b562e6804 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -76,6 +76,8 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender);
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id);
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_add_state_ids_idx ON syncapi_output_room_events ((add_state_ids IS NOT NULL));
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_remove_state_ids_idx ON syncapi_output_room_events ((remove_state_ids IS NOT NULL));
 `
 
 const insertEventSQL = "" +
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 165943027..d6a674b9c 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -55,6 +55,8 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender);
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id);
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_add_state_ids_idx ON syncapi_output_room_events ((add_state_ids IS NOT NULL));
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_remove_state_ids_idx ON syncapi_output_room_events ((remove_state_ids IS NOT NULL));
 `
 
 const insertEventSQL = "" +

From e0703522932ab4bf41b23d5229c7a4540b59d830 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 5 Oct 2022 11:14:33 +0200
Subject: [PATCH 09/42] Side effect import bleve analyzer languages (#2763)

... to actually allow different languages. Fixes #2761
Binary size increases by ~1MB.
---
 internal/fulltext/bleve.go | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go
index da8932f5c..7187861dd 100644
--- a/internal/fulltext/bleve.go
+++ b/internal/fulltext/bleve.go
@@ -21,6 +21,29 @@ import (
 	"strings"
 
 	"github.com/blevesearch/bleve/v2"
+
+	// side effect imports to allow all possible languages
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ar"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/cjk"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ckb"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/da"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/de"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/en"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/es"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/fa"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/fi"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/fr"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/hi"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/hr"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/hu"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/it"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/nl"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/no"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/pt"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ro"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ru"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/sv"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/tr"
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/matrix-org/gomatrixserverlib"
 

From ebd137cf6b2fbd767625dc5289b0bef6d1e51971 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 5 Oct 2022 11:07:17 +0100
Subject: [PATCH 10/42] Check PostgreSQL connection count (#2760)

This PR queries PostgreSQL for the `max_connections` and
`superuser_reserved_connections` settings and then ensures that
Dendrite's `max_open_conns` doesn't exceed the allowed value.

This is a really common source of configuration problems and can either
result in blocking queries or deadlocks, so it seems reasonable that we
complain as loudly as possible when it happens.
---
 internal/sqlutil/sqlutil.go | 39 +++++++++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/internal/sqlutil/sqlutil.go b/internal/sqlutil/sqlutil.go
index 0cdae6d30..789bceeac 100644
--- a/internal/sqlutil/sqlutil.go
+++ b/internal/sqlutil/sqlutil.go
@@ -2,6 +2,7 @@ package sqlutil
 
 import (
 	"database/sql"
+	"flag"
 	"fmt"
 	"regexp"
 
@@ -9,6 +10,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+var skipSanityChecks = flag.Bool("skip-db-sanity", false, "Ignore sanity checks on the database connections (NOT RECOMMENDED!)")
+
 // Open opens a database specified by its database driver name and a driver-specific data source name,
 // usually consisting of at least a database name and connection information. Includes tracing driver
 // if DENDRITE_TRACE_SQL=1
@@ -37,15 +40,39 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error)
 		return nil, err
 	}
 	if driverName != "sqlite3" {
-		logrus.WithFields(logrus.Fields{
-			"MaxOpenConns":    dbProperties.MaxOpenConns(),
-			"MaxIdleConns":    dbProperties.MaxIdleConns(),
-			"ConnMaxLifetime": dbProperties.ConnMaxLifetime(),
-			"dataSourceName":  regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
-		}).Debug("Setting DB connection limits")
+		logger := logrus.WithFields(logrus.Fields{
+			"max_open_conns":    dbProperties.MaxOpenConns(),
+			"max_idle_conns":    dbProperties.MaxIdleConns(),
+			"conn_max_lifetime": dbProperties.ConnMaxLifetime(),
+			"data_source_name":  regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
+		})
+		logger.Debug("Setting DB connection limits")
 		db.SetMaxOpenConns(dbProperties.MaxOpenConns())
 		db.SetMaxIdleConns(dbProperties.MaxIdleConns())
 		db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime())
+
+		if !*skipSanityChecks {
+			if dbProperties.MaxOpenConns() == 0 {
+				logrus.Warnf("WARNING: Configuring 'max_open_conns' to be unlimited is not recommended. This can result in bad performance or deadlocks.")
+			}
+
+			switch driverName {
+			case "postgres":
+				// Perform a quick sanity check if possible that we aren't trying to use more database
+				// connections than PostgreSQL is willing to give us.
+				var max, reserved int
+				if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='max_connections';").Scan(&max); err != nil {
+					return nil, fmt.Errorf("failed to find maximum connections: %w", err)
+				}
+				if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='superuser_reserved_connections';").Scan(&reserved); err != nil {
+					return nil, fmt.Errorf("failed to find reserved connections: %w", err)
+				}
+				if configured, allowed := dbProperties.MaxOpenConns(), max-reserved; configured > allowed {
+					logrus.Errorf("ERROR: The configured 'max_open_conns' is greater than the %d non-superuser connections that PostgreSQL is configured to allow. This can result in bad performance or deadlocks. Please pay close attention to your configured database connection counts. If you REALLY know what you are doing and want to override this error, pass the --skip-db-sanity option to Dendrite.", allowed)
+					return nil, fmt.Errorf("database sanity checks failed")
+				}
+			}
+		}
 	}
 	return db, nil
 }

From 8c0c3441d88a612ca8e9ba4f83c6ff29ca73f5d0 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 5 Oct 2022 12:12:42 +0200
Subject: [PATCH 11/42] Add `RoomEventType` nats.Header to avoid unneeded
 unmarshalling (#2765)

---
 appservice/consumers/roomserver.go    |  5 +++++
 federationapi/consumers/roomserver.go |  7 +++++++
 roomserver/producers/roomevent.go     | 13 ++++++-------
 setup/jetstream/streams.go            |  7 ++++---
 test/testrig/jetstream.go             |  9 ++++-----
 userapi/consumers/roomserver.go       |  7 ++++---
 6 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index d44f32b38..ac68f4bd4 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -101,6 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage(
 	log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs))
 	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs))
 	for _, msg := range msgs {
+		// Only handle events we care about
+		receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
+		if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInviteEvent {
+			continue
+		}
 		// Parse out the event JSON
 		var output api.OutputEvent
 		if err := json.Unmarshal(msg.Data, &output); err != nil {
diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go
index 349b50b05..a42733628 100644
--- a/federationapi/consumers/roomserver.go
+++ b/federationapi/consumers/roomserver.go
@@ -79,6 +79,13 @@ func (s *OutputRoomEventConsumer) Start() error {
 // realises that it cannot update the room state using the deltas.
 func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
 	msg := msgs[0] // Guaranteed to exist if onMessage is called
+	receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
+
+	// Only handle events we care about
+	if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInboundPeek {
+		return true
+	}
+
 	// Parse out the event JSON
 	var output api.OutputEvent
 	if err := json.Unmarshal(msg.Data, &output); err != nil {
diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go
index 987e6c942..9c4521986 100644
--- a/roomserver/producers/roomevent.go
+++ b/roomserver/producers/roomevent.go
@@ -17,12 +17,13 @@ package producers
 import (
 	"encoding/json"
 
-	"github.com/matrix-org/dendrite/roomserver/acls"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
+
+	"github.com/matrix-org/dendrite/roomserver/acls"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 )
 
 var keyContentFields = map[string]string{
@@ -40,10 +41,8 @@ type RoomEventProducer struct {
 func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.OutputEvent) error {
 	var err error
 	for _, update := range updates {
-		msg := &nats.Msg{
-			Subject: r.Topic,
-			Header:  nats.Header{},
-		}
+		msg := nats.NewMsg(r.Topic)
+		msg.Header.Set(jetstream.RoomEventType, string(update.Type))
 		msg.Header.Set(jetstream.RoomID, roomID)
 		msg.Data, err = json.Marshal(update)
 		if err != nil {
diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go
index ee9810dae..590f0cbd9 100644
--- a/setup/jetstream/streams.go
+++ b/setup/jetstream/streams.go
@@ -9,9 +9,10 @@ import (
 )
 
 const (
-	UserID  = "user_id"
-	RoomID  = "room_id"
-	EventID = "event_id"
+	UserID        = "user_id"
+	RoomID        = "room_id"
+	EventID       = "event_id"
+	RoomEventType = "output_room_event_type"
 )
 
 var (
diff --git a/test/testrig/jetstream.go b/test/testrig/jetstream.go
index 74cf95062..b880eea43 100644
--- a/test/testrig/jetstream.go
+++ b/test/testrig/jetstream.go
@@ -4,10 +4,11 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/nats-io/nats.go"
+
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/nats-io/nats.go"
 )
 
 func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Msg) {
@@ -21,10 +22,8 @@ func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Ms
 
 func NewOutputEventMsg(t *testing.T, base *base.BaseDendrite, roomID string, update api.OutputEvent) *nats.Msg {
 	t.Helper()
-	msg := &nats.Msg{
-		Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
-		Header:  nats.Header{},
-	}
+	msg := nats.NewMsg(base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent))
+	msg.Header.Set(jetstream.RoomEventType, string(update.Type))
 	msg.Header.Set(jetstream.RoomID, roomID)
 	var err error
 	msg.Data, err = json.Marshal(update)
diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go
index 952de98f7..a12876946 100644
--- a/userapi/consumers/roomserver.go
+++ b/userapi/consumers/roomserver.go
@@ -72,15 +72,16 @@ func (s *OutputRoomEventConsumer) Start() error {
 
 func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
 	msg := msgs[0] // Guaranteed to exist if onMessage is called
+	// Only handle events we care about
+	if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent {
+		return true
+	}
 	var output rsapi.OutputEvent
 	if err := json.Unmarshal(msg.Data, &output); err != nil {
 		// If the message was invalid, log it and move on to the next message in the stream
 		log.WithError(err).Errorf("roomserver output log: message parse failure")
 		return true
 	}
-	if output.Type != rsapi.OutputTypeNewRoomEvent {
-		return true
-	}
 	event := output.NewRoomEvent.Event
 	if event == nil {
 		log.Errorf("userapi consumer: expected event")

From 6f602bb0969fac6d455de1088bf3980f77a4017f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 5 Oct 2022 11:16:05 +0100
Subject: [PATCH 12/42] Demote `Failed to query device keys for some users`
 warning to `level=debug`

Many of these warnings are due to dead servers and are quite annoying when they fill up the logs.
---
 keyserver/internal/device_list_update.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index fcfcd092d..8b02f3d6c 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -424,7 +424,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 			"succeeded": successCount,
 			"failed":    len(userIDs) - successCount,
 			"wait_time": waitTime,
-		}).Warn("Failed to query device keys for some users")
+		}).Debug("Failed to query device keys for some users")
 	}
 	return waitTime, !allUsersSucceeded
 }

From c85bc3434fc930423e9e27c8bca9b31b5ad7a441 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 5 Oct 2022 12:47:53 +0100
Subject: [PATCH 13/42] Optimise `QuerySharedUsers` so that we can only work on
 local users (#2766)

Otherwise the sync API key change consumer wastes a lot of time trying
to wake up the notifiers for non-local users.
---
 roomserver/api/query.go                       |  1 +
 roomserver/internal/query/query.go            |  2 +-
 roomserver/storage/interface.go               |  2 +-
 .../storage/postgres/membership_table.go      | 17 +++++++++-----
 roomserver/storage/shared/storage.go          |  4 ++--
 .../storage/sqlite3/membership_table.go       | 23 +++++++++++--------
 roomserver/storage/tables/interface.go        |  2 +-
 .../storage/tables/membership_table_test.go   |  2 +-
 syncapi/consumers/keychange.go                |  6 +++--
 9 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index aa7dc4735..d63c24785 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -278,6 +278,7 @@ type QuerySharedUsersRequest struct {
 	OtherUserIDs   []string
 	ExcludeRoomIDs []string
 	IncludeRoomIDs []string
+	LocalOnly      bool
 }
 
 type QuerySharedUsersResponse struct {
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index b41a92e94..ee8e1cfe7 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -799,7 +799,7 @@ func (r *Queryer) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUser
 	}
 	roomIDs = roomIDs[:j]
 
-	users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs)
+	users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs, req.LocalOnly)
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 43e8da7bb..11e175f55 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -157,7 +157,7 @@ type Database interface {
 	// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
 	GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
 	// JoinedUsersSetInRooms returns how many times each of the given users appears across the given rooms.
-	JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error)
+	JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error)
 	// GetLocalServerInRoom returns true if we think we're in a given room or false otherwise.
 	GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error)
 	// GetServerInRoom returns true if we think a server is in a given room or false otherwise.
diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go
index bd3fd5592..0150534e1 100644
--- a/roomserver/storage/postgres/membership_table.go
+++ b/roomserver/storage/postgres/membership_table.go
@@ -68,14 +68,18 @@ CREATE TABLE IF NOT EXISTS roomserver_membership (
 
 var selectJoinedUsersSetForRoomsAndUserSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
-	" WHERE room_nid = ANY($1) AND target_nid = ANY($2) AND" +
-	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" WHERE (target_local OR $1 = false)" +
+	" AND room_nid = ANY($2) AND target_nid = ANY($3)" +
+	" AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
+	" AND forgotten = false" +
 	" GROUP BY target_nid"
 
 var selectJoinedUsersSetForRoomsSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
-	" WHERE room_nid = ANY($1) AND" +
-	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" WHERE (target_local OR $1 = false) " +
+	" AND room_nid = ANY($2)" +
+	" AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
+	" AND forgotten = false" +
 	" GROUP BY target_nid"
 
 // Insert a row in to membership table so that it can be locked by the
@@ -334,6 +338,7 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(
 	ctx context.Context, txn *sql.Tx,
 	roomNIDs []types.RoomNID,
 	userNIDs []types.EventStateKeyNID,
+	localOnly bool,
 ) (map[types.EventStateKeyNID]int, error) {
 	var (
 		rows *sql.Rows
@@ -342,9 +347,9 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(
 	stmt := sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsStmt)
 	if len(userNIDs) > 0 {
 		stmt = sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsAndUserStmt)
-		rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs), pq.Array(userNIDs))
+		rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs), pq.Array(userNIDs))
 	} else {
-		rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs))
+		rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs))
 	}
 
 	if err != nil {
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 593abbea1..d83a1ff74 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -1280,7 +1280,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu
 }
 
 // JoinedUsersSetInRooms returns a map of how many times the given users appear in the specified rooms.
-func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error) {
+func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error) {
 	roomNIDs, err := d.RoomsTable.BulkSelectRoomNIDs(ctx, nil, roomIDs)
 	if err != nil {
 		return nil, err
@@ -1295,7 +1295,7 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs [
 		userNIDs = append(userNIDs, nid)
 		nidToUserID[nid] = id
 	}
-	userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs)
+	userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs, localOnly)
 	if err != nil {
 		return nil, err
 	}
diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go
index f3303eb0e..cd149f0ed 100644
--- a/roomserver/storage/sqlite3/membership_table.go
+++ b/roomserver/storage/sqlite3/membership_table.go
@@ -44,14 +44,18 @@ const membershipSchema = `
 
 var selectJoinedUsersSetForRoomsAndUserSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
-	" WHERE room_nid IN ($1) AND target_nid IN ($2) AND" +
-	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" WHERE (target_local OR $1 = false)" +
+	" AND room_nid IN ($2) AND target_nid IN ($3)" +
+	" AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
+	" AND forgotten = false" +
 	" GROUP BY target_nid"
 
 var selectJoinedUsersSetForRoomsSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
-	" WHERE room_nid IN ($1) AND " +
-	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" WHERE (target_local OR $1 = false)" +
+	" AND room_nid IN ($2)" +
+	" AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
+	" AND forgotten = false" +
 	" GROUP BY target_nid"
 
 // Insert a row in to membership table so that it can be locked by the
@@ -305,8 +309,9 @@ func (s *membershipStatements) SelectRoomsWithMembership(
 	return roomNIDs, nil
 }
 
-func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error) {
-	params := make([]interface{}, 0, len(roomNIDs)+len(userNIDs))
+func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) {
+	params := make([]interface{}, 0, 1+len(roomNIDs)+len(userNIDs))
+	params = append(params, localOnly)
 	for _, v := range roomNIDs {
 		params = append(params, v)
 	}
@@ -314,10 +319,10 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context,
 		params = append(params, v)
 	}
 
-	query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1)
+	query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1)
 	if len(userNIDs) > 0 {
-		query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1)
-		query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)), 1)
+		query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1)
+		query = strings.Replace(query, "($3)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)+1), 1)
 	}
 	var rows *sql.Rows
 	var err error
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 68d30f994..d7bcc95ab 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -137,7 +137,7 @@ type Membership interface {
 	UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID, forgotten bool) (bool, error)
 	SelectRoomsWithMembership(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, membershipState MembershipState) ([]types.RoomNID, error)
 	// SelectJoinedUsersSetForRooms returns how many times each of the given users appears across the given rooms.
-	SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error)
+	SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error)
 	SelectKnownUsers(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, searchString string, limit int) ([]string, error)
 	UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error
 	SelectLocalServerInRoom(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID) (bool, error)
diff --git a/roomserver/storage/tables/membership_table_test.go b/roomserver/storage/tables/membership_table_test.go
index f789ef4ac..c9541d9d2 100644
--- a/roomserver/storage/tables/membership_table_test.go
+++ b/roomserver/storage/tables/membership_table_test.go
@@ -79,7 +79,7 @@ func TestMembershipTable(t *testing.T) {
 		assert.NoError(t, err)
 		assert.True(t, inRoom)
 
-		userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs)
+		userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs, false)
 		assert.NoError(t, err)
 		assert.Equal(t, 1, len(userJoinedToRooms))
 
diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go
index dc7d9e207..96ebba7ef 100644
--- a/syncapi/consumers/keychange.go
+++ b/syncapi/consumers/keychange.go
@@ -111,7 +111,8 @@ func (s *OutputKeyChangeEventConsumer) onDeviceKeyMessage(m api.DeviceMessage, d
 	// work out who we need to notify about the new key
 	var queryRes roomserverAPI.QuerySharedUsersResponse
 	err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{
-		UserID: output.UserID,
+		UserID:    output.UserID,
+		LocalOnly: true,
 	}, &queryRes)
 	if err != nil {
 		logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server")
@@ -135,7 +136,8 @@ func (s *OutputKeyChangeEventConsumer) onCrossSigningMessage(m api.DeviceMessage
 	// work out who we need to notify about the new key
 	var queryRes roomserverAPI.QuerySharedUsersResponse
 	err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{
-		UserID: output.UserID,
+		UserID:    output.UserID,
+		LocalOnly: true,
 	}, &queryRes)
 	if err != nil {
 		logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server")

From 0f777d421c56e386b4c483233277f2b96c4da3a0 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 5 Oct 2022 14:47:13 +0200
Subject: [PATCH 14/42] Remove empty fields from `/sync` response (#2755)

First attempt at removing empty fields from `/sync` responses. Needs
https://github.com/matrix-org/sytest/pull/1298 to keep Sytest happy.

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 syncapi/internal/keychange_test.go    |  33 +++--
 syncapi/streams/stream_accountdata.go |   6 +-
 syncapi/streams/stream_invite.go      |   4 +-
 syncapi/streams/stream_pdu.go         |  10 +-
 syncapi/streams/stream_receipt.go     |   5 +-
 syncapi/streams/stream_typing.go      |   9 +-
 syncapi/types/types.go                | 201 ++++++++++++++++++--------
 7 files changed, 178 insertions(+), 90 deletions(-)

diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index 3b9c8221c..53f3e5a40 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -170,9 +170,12 @@ func joinResponseWithRooms(syncResponse *types.Response, userID string, roomIDs
 				Content:  []byte(`{"membership":"join"}`),
 			},
 		}
-
-		jr := syncResponse.Rooms.Join[roomID]
-		jr.State.Events = roomEvents
+		jr, ok := syncResponse.Rooms.Join[roomID]
+		if !ok {
+			jr = types.NewJoinResponse()
+		}
+		jr.Timeline = &types.Timeline{}
+		jr.State = &types.ClientEvents{Events: roomEvents}
 		syncResponse.Rooms.Join[roomID] = jr
 	}
 	return syncResponse
@@ -191,8 +194,11 @@ func leaveResponseWithRooms(syncResponse *types.Response, userID string, roomIDs
 			},
 		}
 
-		lr := syncResponse.Rooms.Leave[roomID]
-		lr.Timeline.Events = roomEvents
+		lr, ok := syncResponse.Rooms.Leave[roomID]
+		if !ok {
+			lr = types.NewLeaveResponse()
+		}
+		lr.Timeline = &types.Timeline{Events: roomEvents}
 		syncResponse.Rooms.Leave[roomID] = lr
 	}
 	return syncResponse
@@ -328,9 +334,13 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) {
 		},
 	}
 
-	jr := syncResponse.Rooms.Join[roomID]
-	jr.State.Events = roomStateEvents
-	jr.Timeline.Events = roomTimelineEvents
+	jr, ok := syncResponse.Rooms.Join[roomID]
+	if !ok {
+		jr = types.NewJoinResponse()
+	}
+
+	jr.State = &types.ClientEvents{Events: roomStateEvents}
+	jr.Timeline = &types.Timeline{Events: roomTimelineEvents}
 	syncResponse.Rooms.Join[roomID] = jr
 
 	rsAPI := &mockRoomserverAPI{
@@ -442,8 +452,11 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) {
 		},
 	}
 
-	lr := syncResponse.Rooms.Leave[roomID]
-	lr.Timeline.Events = roomEvents
+	lr, ok := syncResponse.Rooms.Leave[roomID]
+	if !ok {
+		lr = types.NewLeaveResponse()
+	}
+	lr.Timeline = &types.Timeline{Events: roomEvents}
 	syncResponse.Rooms.Leave[roomID] = lr
 
 	rsAPI := &mockRoomserverAPI{
diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index 3f2f7d134..3593a6563 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -90,9 +90,9 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 				}
 			} else {
 				if roomData, ok := dataRes.RoomAccountData[roomID][dataType]; ok {
-					joinData := *types.NewJoinResponse()
-					if existing, ok := req.Response.Rooms.Join[roomID]; ok {
-						joinData = existing
+					joinData, ok := req.Response.Rooms.Join[roomID]
+					if !ok {
+						joinData = types.NewJoinResponse()
 					}
 					joinData.AccountData.Events = append(
 						joinData.AccountData.Events,
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 17b3b8434..7875ffa35 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -65,7 +65,7 @@ func (p *InviteStreamProvider) IncrementalSync(
 			continue
 		}
 		ir := types.NewInviteResponse(inviteEvent)
-		req.Response.Rooms.Invite[roomID] = *ir
+		req.Response.Rooms.Invite[roomID] = ir
 	}
 
 	// When doing an initial sync, we don't want to add retired invites, as this
@@ -87,7 +87,7 @@ func (p *InviteStreamProvider) IncrementalSync(
 				Type:           "m.room.member",
 				Content:        gomatrixserverlib.RawJSON(`{"membership":"leave"}`),
 			})
-			req.Response.Rooms.Leave[roomID] = *lr
+			req.Response.Rooms.Leave[roomID] = lr
 		}
 	}
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index d252265ff..613ac434f 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -106,7 +106,7 @@ func (p *PDUStreamProvider) CompleteSync(
 			}
 			continue
 		}
-		req.Response.Rooms.Join[roomID] = *jr
+		req.Response.Rooms.Join[roomID] = jr
 		req.Rooms[roomID] = gomatrixserverlib.Join
 	}
 
@@ -129,7 +129,7 @@ func (p *PDUStreamProvider) CompleteSync(
 				}
 				continue
 			}
-			req.Response.Rooms.Peek[peek.RoomID] = *jr
+			req.Response.Rooms.Peek[peek.RoomID] = jr
 		}
 	}
 
@@ -320,7 +320,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		// didn't "remove" events, return that the response is limited.
 		jr.Timeline.Limited = limited && len(events) == len(recentEvents)
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
-		res.Rooms.Join[delta.RoomID] = *jr
+		res.Rooms.Join[delta.RoomID] = jr
 
 	case gomatrixserverlib.Peek:
 		jr := types.NewJoinResponse()
@@ -329,7 +329,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
 		jr.Timeline.Limited = limited
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
-		res.Rooms.Peek[delta.RoomID] = *jr
+		res.Rooms.Peek[delta.RoomID] = jr
 
 	case gomatrixserverlib.Leave:
 		fallthrough // transitions to leave are the same as ban
@@ -342,7 +342,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		// didn't "remove" events, return that the response is limited.
 		lr.Timeline.Limited = limited && len(events) == len(recentEvents)
 		lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
-		res.Rooms.Leave[delta.RoomID] = *lr
+		res.Rooms.Leave[delta.RoomID] = lr
 	}
 
 	return latestPosition, nil
diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index 8818a5533..76927cc36 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -4,9 +4,10 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type ReceiptStreamProvider struct {
@@ -76,7 +77,7 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 			continue
 		}
 
-		jr := *types.NewJoinResponse()
+		jr := types.NewJoinResponse()
 		if existing, ok := req.Response.Rooms.Join[roomID]; ok {
 			jr = existing
 		}
diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go
index a6f7c7a06..84c199b39 100644
--- a/syncapi/streams/stream_typing.go
+++ b/syncapi/streams/stream_typing.go
@@ -4,10 +4,11 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type TypingStreamProvider struct {
@@ -35,9 +36,9 @@ func (p *TypingStreamProvider) IncrementalSync(
 			continue
 		}
 
-		jr := *types.NewJoinResponse()
-		if existing, ok := req.Response.Rooms.Join[roomID]; ok {
-			jr = existing
+		jr, ok := req.Response.Rooms.Join[roomID]
+		if !ok {
+			jr = types.NewJoinResponse()
 		}
 
 		if users, updated := p.EDUCache.GetTypingUsersIfUpdatedAfter(
diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index 3b85db4a4..b6d340f93 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -327,29 +327,57 @@ type PrevEventRef struct {
 	PrevSender    string          `json:"prev_sender"`
 }
 
+type DeviceLists struct {
+	Changed []string `json:"changed,omitempty"`
+	Left    []string `json:"left,omitempty"`
+}
+
+type RoomsResponse struct {
+	Join   map[string]*JoinResponse   `json:"join,omitempty"`
+	Peek   map[string]*JoinResponse   `json:"peek,omitempty"`
+	Invite map[string]*InviteResponse `json:"invite,omitempty"`
+	Leave  map[string]*LeaveResponse  `json:"leave,omitempty"`
+}
+
+type ToDeviceResponse struct {
+	Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"`
+}
+
 // Response represents a /sync API response. See https://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-client-r0-sync
 type Response struct {
-	NextBatch   StreamingToken `json:"next_batch"`
-	AccountData struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
-	} `json:"account_data,omitempty"`
-	Presence struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
-	} `json:"presence,omitempty"`
-	Rooms struct {
-		Join   map[string]JoinResponse   `json:"join,omitempty"`
-		Peek   map[string]JoinResponse   `json:"peek,omitempty"`
-		Invite map[string]InviteResponse `json:"invite,omitempty"`
-		Leave  map[string]LeaveResponse  `json:"leave,omitempty"`
-	} `json:"rooms,omitempty"`
-	ToDevice struct {
-		Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"`
-	} `json:"to_device,omitempty"`
-	DeviceLists struct {
-		Changed []string `json:"changed,omitempty"`
-		Left    []string `json:"left,omitempty"`
-	} `json:"device_lists,omitempty"`
-	DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"`
+	NextBatch           StreamingToken    `json:"next_batch"`
+	AccountData         *ClientEvents     `json:"account_data,omitempty"`
+	Presence            *ClientEvents     `json:"presence,omitempty"`
+	Rooms               *RoomsResponse    `json:"rooms,omitempty"`
+	ToDevice            *ToDeviceResponse `json:"to_device,omitempty"`
+	DeviceLists         *DeviceLists      `json:"device_lists,omitempty"`
+	DeviceListsOTKCount map[string]int    `json:"device_one_time_keys_count,omitempty"`
+}
+
+func (r Response) MarshalJSON() ([]byte, error) {
+	type alias Response
+	a := alias(r)
+	if r.AccountData != nil && len(r.AccountData.Events) == 0 {
+		a.AccountData = nil
+	}
+	if r.Presence != nil && len(r.Presence.Events) == 0 {
+		a.Presence = nil
+	}
+	if r.DeviceLists != nil {
+		if len(r.DeviceLists.Left) == 0 && len(r.DeviceLists.Changed) == 0 {
+			a.DeviceLists = nil
+		}
+	}
+	if r.Rooms != nil {
+		if len(r.Rooms.Join) == 0 && len(r.Rooms.Peek) == 0 &&
+			len(r.Rooms.Invite) == 0 && len(r.Rooms.Leave) == 0 {
+			a.Rooms = nil
+		}
+	}
+	if r.ToDevice != nil && len(r.ToDevice.Events) == 0 {
+		a.ToDevice = nil
+	}
+	return json.Marshal(a)
 }
 
 func (r *Response) HasUpdates() bool {
@@ -370,18 +398,21 @@ func NewResponse() *Response {
 	res := Response{}
 	// Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section,
 	// so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors.
-	res.Rooms.Join = map[string]JoinResponse{}
-	res.Rooms.Peek = map[string]JoinResponse{}
-	res.Rooms.Invite = map[string]InviteResponse{}
-	res.Rooms.Leave = map[string]LeaveResponse{}
+	res.Rooms = &RoomsResponse{
+		Join:   map[string]*JoinResponse{},
+		Peek:   map[string]*JoinResponse{},
+		Invite: map[string]*InviteResponse{},
+		Leave:  map[string]*LeaveResponse{},
+	}
 
 	// Also pre-intialise empty slices or else we'll insert 'null' instead of '[]' for the value.
 	// TODO: We really shouldn't have to do all this to coerce encoding/json to Do The Right Thing. We should
 	//       really be using our own Marshal/Unmarshal implementations otherwise this may prove to be a CPU bottleneck.
 	//       This also applies to NewJoinResponse, NewInviteResponse and NewLeaveResponse.
-	res.AccountData.Events = []gomatrixserverlib.ClientEvent{}
-	res.Presence.Events = []gomatrixserverlib.ClientEvent{}
-	res.ToDevice.Events = []gomatrixserverlib.SendToDeviceEvent{}
+	res.AccountData = &ClientEvents{}
+	res.Presence = &ClientEvents{}
+	res.DeviceLists = &DeviceLists{}
+	res.ToDevice = &ToDeviceResponse{}
 	res.DeviceListsOTKCount = map[string]int{}
 
 	return &res
@@ -403,38 +434,73 @@ type UnreadNotifications struct {
 	NotificationCount int `json:"notification_count"`
 }
 
+type ClientEvents struct {
+	Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
+}
+
+type Timeline struct {
+	Events    []gomatrixserverlib.ClientEvent `json:"events"`
+	Limited   bool                            `json:"limited"`
+	PrevBatch *TopologyToken                  `json:"prev_batch,omitempty"`
+}
+
+type Summary struct {
+	Heroes             []string `json:"m.heroes,omitempty"`
+	JoinedMemberCount  *int     `json:"m.joined_member_count,omitempty"`
+	InvitedMemberCount *int     `json:"m.invited_member_count,omitempty"`
+}
+
 // JoinResponse represents a /sync response for a room which is under the 'join' or 'peek' key.
 type JoinResponse struct {
-	Summary struct {
-		Heroes             []string `json:"m.heroes,omitempty"`
-		JoinedMemberCount  *int     `json:"m.joined_member_count,omitempty"`
-		InvitedMemberCount *int     `json:"m.invited_member_count,omitempty"`
-	} `json:"summary"`
-	State struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"state"`
-	Timeline struct {
-		Events    []gomatrixserverlib.ClientEvent `json:"events"`
-		Limited   bool                            `json:"limited"`
-		PrevBatch *TopologyToken                  `json:"prev_batch,omitempty"`
-	} `json:"timeline"`
-	Ephemeral struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"ephemeral"`
-	AccountData struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"account_data"`
+	Summary              *Summary      `json:"summary,omitempty"`
+	State                *ClientEvents `json:"state,omitempty"`
+	Timeline             *Timeline     `json:"timeline,omitempty"`
+	Ephemeral            *ClientEvents `json:"ephemeral,omitempty"`
+	AccountData          *ClientEvents `json:"account_data,omitempty"`
 	*UnreadNotifications `json:"unread_notifications,omitempty"`
 }
 
+func (jr JoinResponse) MarshalJSON() ([]byte, error) {
+	type alias JoinResponse
+	a := alias(jr)
+	if jr.State != nil && len(jr.State.Events) == 0 {
+		a.State = nil
+	}
+	if jr.Ephemeral != nil && len(jr.Ephemeral.Events) == 0 {
+		a.Ephemeral = nil
+	}
+	if jr.AccountData != nil && len(jr.AccountData.Events) == 0 {
+		a.AccountData = nil
+	}
+	if jr.Timeline != nil && len(jr.Timeline.Events) == 0 {
+		a.Timeline = nil
+	}
+	if jr.Summary != nil {
+		var nilPtr int
+		joinedEmpty := jr.Summary.JoinedMemberCount == nil || jr.Summary.JoinedMemberCount == &nilPtr
+		invitedEmpty := jr.Summary.InvitedMemberCount == nil || jr.Summary.InvitedMemberCount == &nilPtr
+		if joinedEmpty && invitedEmpty && len(jr.Summary.Heroes) == 0 {
+			a.Summary = nil
+		}
+
+	}
+	if jr.UnreadNotifications != nil &&
+		jr.UnreadNotifications.NotificationCount == 0 && jr.UnreadNotifications.HighlightCount == 0 {
+		a.UnreadNotifications = nil
+	}
+	return json.Marshal(a)
+}
+
 // NewJoinResponse creates an empty response with initialised arrays.
 func NewJoinResponse() *JoinResponse {
-	res := JoinResponse{}
-	res.State.Events = []gomatrixserverlib.ClientEvent{}
-	res.Timeline.Events = []gomatrixserverlib.ClientEvent{}
-	res.Ephemeral.Events = []gomatrixserverlib.ClientEvent{}
-	res.AccountData.Events = []gomatrixserverlib.ClientEvent{}
-	return &res
+	return &JoinResponse{
+		Summary:             &Summary{},
+		State:               &ClientEvents{},
+		Timeline:            &Timeline{},
+		Ephemeral:           &ClientEvents{},
+		AccountData:         &ClientEvents{},
+		UnreadNotifications: &UnreadNotifications{},
+	}
 }
 
 // InviteResponse represents a /sync response for a room which is under the 'invite' key.
@@ -469,21 +535,28 @@ func NewInviteResponse(event *gomatrixserverlib.HeaderedEvent) *InviteResponse {
 
 // LeaveResponse represents a /sync response for a room which is under the 'leave' key.
 type LeaveResponse struct {
-	State struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"state"`
-	Timeline struct {
-		Events    []gomatrixserverlib.ClientEvent `json:"events"`
-		Limited   bool                            `json:"limited"`
-		PrevBatch *TopologyToken                  `json:"prev_batch,omitempty"`
-	} `json:"timeline"`
+	State    *ClientEvents `json:"state,omitempty"`
+	Timeline *Timeline     `json:"timeline,omitempty"`
+}
+
+func (lr LeaveResponse) MarshalJSON() ([]byte, error) {
+	type alias LeaveResponse
+	a := alias(lr)
+	if lr.State != nil && len(lr.State.Events) == 0 {
+		a.State = nil
+	}
+	if lr.Timeline != nil && len(lr.Timeline.Events) == 0 {
+		a.Timeline = nil
+	}
+	return json.Marshal(a)
 }
 
 // NewLeaveResponse creates an empty response with initialised arrays.
 func NewLeaveResponse() *LeaveResponse {
-	res := LeaveResponse{}
-	res.State.Events = []gomatrixserverlib.ClientEvent{}
-	res.Timeline.Events = []gomatrixserverlib.ClientEvent{}
+	res := LeaveResponse{
+		State:    &ClientEvents{},
+		Timeline: &Timeline{},
+	}
 	return &res
 }
 

From 9ba3103f8839243ea8529bf1f563afded3e7b591 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 5 Oct 2022 14:54:19 +0100
Subject: [PATCH 15/42] Document database connection limits

---
 docs/installation/11_optimisation.md | 35 ++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/docs/installation/11_optimisation.md b/docs/installation/11_optimisation.md
index f2f67c947..686ec2eb9 100644
--- a/docs/installation/11_optimisation.md
+++ b/docs/installation/11_optimisation.md
@@ -11,6 +11,41 @@ permalink: /installation/start/optimisation
 Now that you have Dendrite running, the following tweaks will improve the reliability
 and performance of your installation.
 
+## PostgreSQL connection limit
+
+A PostgreSQL database engine is configured to allow only a certain number of connections.
+This is typically controlled by the `max_connections` and `superuser_reserved_connections`
+configuration items in `postgresql.conf`. Once these limits are violated, **PostgreSQL will
+immediately stop accepting new connections** until some of the existing connections are closed.
+This is a common source of misconfiguration and requires particular care.
+
+If your PostgreSQL `max_connections` is set to `100` and `superuser_reserved_connections` is
+set to `3` then you have an effective connection limit of 97 database connections. It is
+therefore important to ensure that Dendrite doesn't violate that limit, otherwise database
+queries will unexpectedly fail and this will cause problems both within Dendrite and for users.
+
+If you are also running other software that uses the same PostgreSQL database engine, then you
+must also take into account that some connections will be already used by your other software
+and therefore will not be available to Dendrite. Check the configuration of any other software
+using the same database engine for their configured connection limits and adjust your calculations
+accordingly.
+
+Dendrite has a `max_open_conns` configuration item in each `database` block to control how many
+connections it will open to the database.
+
+**If you are using the `global` database pool** then you only need to configure the
+`max_open_conns` setting once in the `global` section.
+
+**If you are defining a `database` config per component** then you will need to ensure that
+the **sum total** of all configured `max_open_conns` to a given database server do not exceed
+the connection limit. If you configure a total that adds up to more connections than are available
+then this will cause database queries to fail.
+
+You may wish to raise the `max_connections` limit on your PostgreSQL server to accommodate
+additional connections, in which case you should also update the `max_open_conns` in your
+Dendrite configuration accordingly. However be aware that this is only advisable on particularly
+powerful servers that can handle the concurrent load of additional queries running at one time.
+
 ## File descriptor limit
 
 Most platforms have a limit on how many file descriptors a single process can open. All

From e53dcb25a9a0258b48fd4f7afddd2c3c54d55b7a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 6 Oct 2022 10:07:13 +0100
Subject: [PATCH 16/42] Tweak logging for federated room joins

---
 federationapi/internal/perform.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 84702f4ce..4cdd3a5eb 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -259,7 +259,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	if err != nil {
 		return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err)
 	}
-	logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts")
+	logrus.WithField("room", roomID).Infof("Joined federated room with %d hosts", len(joinedHosts))
 	if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil {
 		return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err)
 	}

From 3f82bceb70050c1233b7de6d87ffa5510596d145 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 6 Oct 2022 10:51:06 +0100
Subject: [PATCH 17/42] Don't try to talk to ourselves when finding missing
 events

---
 roomserver/internal/input/input_events.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 01fd62010..d1b6bc73e 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -173,12 +173,15 @@ func (r *Inputer) processRoomEvent(
 		for _, server := range serverRes.ServerNames {
 			servers[server] = struct{}{}
 		}
+		// Don't try to talk to ourselves.
+		delete(servers, r.Cfg.Matrix.ServerName)
+		// Now build up the list of servers.
 		serverRes.ServerNames = serverRes.ServerNames[:0]
-		if input.Origin != "" {
+		if input.Origin != "" && input.Origin != r.Cfg.Matrix.ServerName {
 			serverRes.ServerNames = append(serverRes.ServerNames, input.Origin)
 			delete(servers, input.Origin)
 		}
-		if senderDomain != input.Origin {
+		if senderDomain != input.Origin && senderDomain != r.Cfg.Matrix.ServerName {
 			serverRes.ServerNames = append(serverRes.ServerNames, senderDomain)
 			delete(servers, senderDomain)
 		}

From ec5d1d681d1362f5746c5cb45e93829d6a68aa4d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 6 Oct 2022 12:30:24 +0200
Subject: [PATCH 18/42] Always return `one_time_key_counts` on `/keys/upload`
 (#2769)

The OTK count is
[required](https://spec.matrix.org/v1.4/client-server-api/#post_matrixclientv3keysupload)
in responses to `/keys/upload`, so return those.
---
 clientapi/routing/keys.go      | 4 ++--
 keyserver/internal/internal.go | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go
index b7a76b47e..5c3681382 100644
--- a/clientapi/routing/keys.go
+++ b/clientapi/routing/keys.go
@@ -19,11 +19,12 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/keyserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/util"
 )
 
 type uploadKeysRequest struct {
@@ -77,7 +78,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devi
 		}
 	}
 	keyCount := make(map[string]int)
-	// we only return key counts when the client uploads OTKs
 	if len(uploadRes.OneTimeKeyCounts) > 0 {
 		keyCount = uploadRes.OneTimeKeyCounts[0].KeyCount
 	}
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 017c29e84..a0280dff4 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -70,6 +70,11 @@ func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.Perform
 	if len(req.OneTimeKeys) > 0 {
 		a.uploadOneTimeKeys(ctx, req, res)
 	}
+	otks, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID)
+	if err != nil {
+		return err
+	}
+	res.OneTimeKeyCounts = []api.OneTimeKeysCount{*otks}
 	return nil
 }
 

From d605d928bce87b381e2f64b8835619d803e67a54 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 6 Oct 2022 11:56:00 +0100
Subject: [PATCH 19/42] Allow specifying old signing keys with the public key
 and key ID only (#2770)

If the private key file is lost, it's often possible to retrieve the
public key from another server elsewhere, so we should make it possible
to configure it in that way.
---
 dendrite-sample.monolith.yaml |  7 ++++-
 dendrite-sample.polylith.yaml |  7 ++++-
 federationapi/routing/keys.go |  2 +-
 setup/config/config.go        | 48 +++++++++++++++++++++++------------
 setup/config/config_global.go |  7 +++--
 5 files changed, 50 insertions(+), 21 deletions(-)

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index f0fa386d1..eadb74a2a 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -18,12 +18,17 @@ global:
   private_key: matrix_key.pem
 
   # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
-  # to old signing private keys that were formerly in use on this domain. These
+  # to old signing keys that were formerly in use on this domain name. These
   # keys will not be used for federation request or event signing, but will be
   # provided to any other homeserver that asks when trying to verify old events.
   old_private_keys:
+  #  If the old private key file is available:
   #  - private_key: old_matrix_key.pem
   #    expired_at: 1601024554498
+  #  If only the public key (in base64 format) and key ID are known:
+  #  - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM=
+  #    key_id: ed25519:mykeyid
+  #    expired_at: 1601024554498
 
   # How long a remote server can cache our server signing key before requesting it
   # again. Increasing this number will reduce the number of requests made by other
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 0ae4cc8fb..aa7e0cc38 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -18,12 +18,17 @@ global:
   private_key: matrix_key.pem
 
   # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
-  # to old signing private keys that were formerly in use on this domain. These
+  # to old signing keys that were formerly in use on this domain name. These
   # keys will not be used for federation request or event signing, but will be
   # provided to any other homeserver that asks when trying to verify old events.
   old_private_keys:
+  #  If the old private key file is available:
   #  - private_key: old_matrix_key.pem
   #    expired_at: 1601024554498
+  #  If only the public key (in base64 format) and key ID are known:
+  #  - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM=
+  #    key_id: ed25519:mykeyid
+  #    expired_at: 1601024554498
 
   # How long a remote server can cache our server signing key before requesting it
   # again. Increasing this number will reduce the number of requests made by other
diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go
index b03d4c1d6..8931830f3 100644
--- a/federationapi/routing/keys.go
+++ b/federationapi/routing/keys.go
@@ -160,7 +160,7 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver
 	for _, oldVerifyKey := range cfg.Matrix.OldVerifyKeys {
 		keys.OldVerifyKeys[oldVerifyKey.KeyID] = gomatrixserverlib.OldVerifyKey{
 			VerifyKey: gomatrixserverlib.VerifyKey{
-				Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PrivateKey.Public().(ed25519.PublicKey)),
+				Key: oldVerifyKey.PublicKey,
 			},
 			ExpiredTS: oldVerifyKey.ExpiredAt,
 		}
diff --git a/setup/config/config.go b/setup/config/config.go
index 5a618d671..e99852ec9 100644
--- a/setup/config/config.go
+++ b/setup/config/config.go
@@ -231,24 +231,40 @@ func loadConfig(
 		return nil, err
 	}
 
-	for i, oldPrivateKey := range c.Global.OldVerifyKeys {
-		var oldPrivateKeyData []byte
+	for _, key := range c.Global.OldVerifyKeys {
+		switch {
+		case key.PrivateKeyPath != "":
+			var oldPrivateKeyData []byte
+			oldPrivateKeyPath := absPath(basePath, key.PrivateKeyPath)
+			oldPrivateKeyData, err = readFile(oldPrivateKeyPath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read %q: %w", oldPrivateKeyPath, err)
+			}
 
-		oldPrivateKeyPath := absPath(basePath, oldPrivateKey.PrivateKeyPath)
-		oldPrivateKeyData, err = readFile(oldPrivateKeyPath)
-		if err != nil {
-			return nil, err
+			// NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are
+			// a number of private keys out there with non-compatible symbols in them due
+			// to lack of validation in Synapse, we won't enforce that for old verify keys.
+			keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false)
+			if perr != nil {
+				return nil, fmt.Errorf("failed to parse %q: %w", oldPrivateKeyPath, perr)
+			}
+
+			key.KeyID = keyID
+			key.PrivateKey = privateKey
+			key.PublicKey = gomatrixserverlib.Base64Bytes(privateKey.Public().(ed25519.PublicKey))
+
+		case key.KeyID == "":
+			return nil, fmt.Errorf("'key_id' must be specified if 'public_key' is specified")
+
+		case len(key.PublicKey) == ed25519.PublicKeySize:
+			continue
+
+		case len(key.PublicKey) > 0:
+			return nil, fmt.Errorf("the supplied 'public_key' is the wrong length")
+
+		default:
+			return nil, fmt.Errorf("either specify a 'private_key' path or supply both 'public_key' and 'key_id'")
 		}
-
-		// NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are
-		// a number of private keys out there with non-compatible symbols in them due
-		// to lack of validation in Synapse, we won't enforce that for old verify keys.
-		keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false)
-		if perr != nil {
-			return nil, perr
-		}
-
-		c.Global.OldVerifyKeys[i].KeyID, c.Global.OldVerifyKeys[i].PrivateKey = keyID, privateKey
 	}
 
 	c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath))
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index acc608dd7..2efae0d5a 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -27,7 +27,7 @@ type Global struct {
 	// Information about old private keys that used to be used to sign requests and
 	// events on this domain. They will not be used but will be advertised to other
 	// servers that ask for them to help verify old events.
-	OldVerifyKeys []OldVerifyKeys `yaml:"old_private_keys"`
+	OldVerifyKeys []*OldVerifyKeys `yaml:"old_private_keys"`
 
 	// How long a remote server can cache our server key for before requesting it again.
 	// Increasing this number will reduce the number of requests made by remote servers
@@ -127,8 +127,11 @@ type OldVerifyKeys struct {
 	// The private key itself.
 	PrivateKey ed25519.PrivateKey `yaml:"-"`
 
+	// The public key, in case only that part is known.
+	PublicKey gomatrixserverlib.Base64Bytes `yaml:"public_key"`
+
 	// The key ID of the private key.
-	KeyID gomatrixserverlib.KeyID `yaml:"-"`
+	KeyID gomatrixserverlib.KeyID `yaml:"key_id"`
 
 	// When the private key was designed as "expired", as a UNIX timestamp
 	// in millisecond precision.

From 8c5b16678475112f5a1c13d616931283421aee05 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 6 Oct 2022 12:57:13 +0200
Subject: [PATCH 20/42] Use the stream positions of the notifier (#2768)

Use the stream positions of the notifier, which might have advanced
since setting it at the beginning of the loop. This possibly helps in
reducing roundtrips to the SyncAPI, just because we didn't fetch the
latest data.
Also fixes a minor oversight in the receipts stream.
---
 syncapi/streams/stream_receipt.go |  6 +++---
 syncapi/sync/requestpool.go       | 18 +++++++++---------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index 76927cc36..bba911022 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -77,9 +77,9 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 			continue
 		}
 
-		jr := types.NewJoinResponse()
-		if existing, ok := req.Response.Rooms.Join[roomID]; ok {
-			jr = existing
+		jr, ok := req.Response.Rooms.Join[roomID]
+		if !ok {
+			jr = types.NewJoinResponse()
 		}
 
 		ev := gomatrixserverlib.ClientEvent{
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index a71d32ab8..29d92b293 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -407,7 +407,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.PDUStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.PDUPosition, currentPos.PDUPosition,
+							syncReq.Since.PDUPosition, rp.Notifier.CurrentPosition().PDUPosition,
 						)
 					},
 				),
@@ -416,7 +416,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.TypingStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.TypingPosition, currentPos.TypingPosition,
+							syncReq.Since.TypingPosition, rp.Notifier.CurrentPosition().TypingPosition,
 						)
 					},
 				),
@@ -425,7 +425,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.ReceiptStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
+							syncReq.Since.ReceiptPosition, rp.Notifier.CurrentPosition().ReceiptPosition,
 						)
 					},
 				),
@@ -434,7 +434,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.InviteStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.InvitePosition, currentPos.InvitePosition,
+							syncReq.Since.InvitePosition, rp.Notifier.CurrentPosition().InvitePosition,
 						)
 					},
 				),
@@ -443,7 +443,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.SendToDeviceStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
+							syncReq.Since.SendToDevicePosition, rp.Notifier.CurrentPosition().SendToDevicePosition,
 						)
 					},
 				),
@@ -452,7 +452,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.AccountDataStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
+							syncReq.Since.AccountDataPosition, rp.Notifier.CurrentPosition().AccountDataPosition,
 						)
 					},
 				),
@@ -461,7 +461,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.NotificationDataStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
+							syncReq.Since.NotificationDataPosition, rp.Notifier.CurrentPosition().NotificationDataPosition,
 						)
 					},
 				),
@@ -470,7 +470,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.DeviceListStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
+							syncReq.Since.DeviceListPosition, rp.Notifier.CurrentPosition().DeviceListPosition,
 						)
 					},
 				),
@@ -479,7 +479,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.PresenceStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.PresencePosition, currentPos.PresencePosition,
+							syncReq.Since.PresencePosition, rp.Notifier.CurrentPosition().PresencePosition,
 						)
 					},
 				),

From 453b50e1d3ce4b4906e050fa8ebe7bcc4c881600 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 7 Oct 2022 07:32:27 +0200
Subject: [PATCH 21/42] Update README

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3bb9a2350..dfef11bae 100644
--- a/README.md
+++ b/README.md
@@ -79,7 +79,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co
 
 # Create an user account (add -admin for an admin user).
 # Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
-$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice
+$ ./bin/create-account --config dendrite.yaml --username alice
 ```
 
 Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
@@ -90,7 +90,7 @@ We use a script called Are We Synapse Yet which checks Sytest compliance rates.
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
 updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
-servers such as matrix.org reasonably well, although there are still some missing features (like Search).
+servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs).
 
 We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather
 than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API).
@@ -112,6 +112,7 @@ This means Dendrite supports amongst others:
 - Guests
 - User Directory
 - Presence
+- Fulltext search
 
 ## Contributing
 

From b9d0e9f7ed7ce1f4be72a25c6f5185a6e809f019 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 7 Oct 2022 10:54:42 +0200
Subject: [PATCH 22/42] Add test for `QueryDeviceMessages` (#2773)

Adds tests for `QueryDeviceMessages` and also includes some
optimizations to reduce allocations in the DB layer.
---
 keyserver/internal/internal.go                |   8 +-
 keyserver/internal/internal_test.go           | 156 ++++++++++++++++++
 .../storage/postgres/device_keys_table.go     |  16 +-
 .../storage/sqlite3/device_keys_table.go      |  16 +-
 4 files changed, 172 insertions(+), 24 deletions(-)
 create mode 100644 keyserver/internal/internal_test.go

diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index a0280dff4..06fc4987c 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -212,15 +212,13 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query
 		return nil
 	}
 	maxStreamID := int64(0)
+	// remove deleted devices
+	var result []api.DeviceMessage
 	for _, m := range msgs {
 		if m.StreamID > maxStreamID {
 			maxStreamID = m.StreamID
 		}
-	}
-	// remove deleted devices
-	var result []api.DeviceMessage
-	for _, m := range msgs {
-		if m.KeyJSON == nil {
+		if m.KeyJSON == nil || len(m.KeyJSON) == 0 {
 			continue
 		}
 		result = append(result, m)
diff --git a/keyserver/internal/internal_test.go b/keyserver/internal/internal_test.go
new file mode 100644
index 000000000..8a2c9c5d9
--- /dev/null
+++ b/keyserver/internal/internal_test.go
@@ -0,0 +1,156 @@
+package internal_test
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/keyserver/internal"
+	"github.com/matrix-org/dendrite/keyserver/storage"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+)
+
+func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := storage.NewDatabase(nil, &config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	})
+	if err != nil {
+		t.Fatalf("failed to create new user db: %v", err)
+	}
+	return db, close
+}
+
+func Test_QueryDeviceMessages(t *testing.T) {
+	alice := test.NewUser(t)
+	type args struct {
+		req *api.QueryDeviceMessagesRequest
+		res *api.QueryDeviceMessagesResponse
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+		want    *api.QueryDeviceMessagesResponse
+	}{
+		{
+			name: "no existing keys",
+			args: args{
+				req: &api.QueryDeviceMessagesRequest{
+					UserID: "@doesNotExist:localhost",
+				},
+				res: &api.QueryDeviceMessagesResponse{},
+			},
+			want: &api.QueryDeviceMessagesResponse{},
+		},
+		{
+			name: "existing user returns devices",
+			args: args{
+				req: &api.QueryDeviceMessagesRequest{
+					UserID: alice.ID,
+				},
+				res: &api.QueryDeviceMessagesResponse{},
+			},
+			want: &api.QueryDeviceMessagesResponse{
+				StreamID: 6,
+				Devices: []api.DeviceMessage{
+					{
+						Type: api.TypeDeviceKeyUpdate, StreamID: 5, DeviceKeys: &api.DeviceKeys{
+							DeviceID:    "myDevice",
+							DisplayName: "first device",
+							UserID:      alice.ID,
+							KeyJSON:     []byte("ghi"),
+						},
+					},
+					{
+						Type: api.TypeDeviceKeyUpdate, StreamID: 6, DeviceKeys: &api.DeviceKeys{
+							DeviceID:    "mySecondDevice",
+							DisplayName: "second device",
+							UserID:      alice.ID,
+							KeyJSON:     []byte("jkl"),
+						}, // streamID 6
+					},
+				},
+			},
+		},
+	}
+
+	deviceMessages := []api.DeviceMessage{
+		{ // not the user we're looking for
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				UserID: "@doesNotExist:localhost",
+			},
+			// streamID 1 for this user
+		},
+		{ // empty keyJSON will be ignored
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID: "myDevice",
+				UserID:   alice.ID,
+			}, // streamID 1
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID: "myDevice",
+				UserID:   alice.ID,
+				KeyJSON:  []byte("abc"),
+			}, // streamID 2
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID: "myDevice",
+				UserID:   alice.ID,
+				KeyJSON:  []byte("def"),
+			}, // streamID 3
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID: "myDevice",
+				UserID:   alice.ID,
+				KeyJSON:  []byte(""),
+			}, // streamID 4
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID:    "myDevice",
+				DisplayName: "first device",
+				UserID:      alice.ID,
+				KeyJSON:     []byte("ghi"),
+			}, // streamID 5
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID:    "mySecondDevice",
+				UserID:      alice.ID,
+				KeyJSON:     []byte("jkl"),
+				DisplayName: "second device",
+			}, // streamID 6
+		},
+	}
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, closeDB := mustCreateDatabase(t, dbType)
+		defer closeDB()
+		if err := db.StoreLocalDeviceKeys(ctx, deviceMessages); err != nil {
+			t.Fatalf("failed to store local devicesKeys")
+		}
+
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
+				a := &internal.KeyInternalAPI{
+					DB: db,
+				}
+				if err := a.QueryDeviceMessages(ctx, tt.args.req, tt.args.res); (err != nil) != tt.wantErr {
+					t.Errorf("QueryDeviceMessages() error = %v, wantErr %v", err, tt.wantErr)
+				}
+				got := tt.args.res
+				if !reflect.DeepEqual(got, tt.want) {
+					t.Errorf("QueryDeviceMessages(): got:\n%+v, want:\n%+v", got, tt.want)
+				}
+			})
+		}
+	})
+}
diff --git a/keyserver/storage/postgres/device_keys_table.go b/keyserver/storage/postgres/device_keys_table.go
index ccd20cbd6..2aa11c520 100644
--- a/keyserver/storage/postgres/device_keys_table.go
+++ b/keyserver/storage/postgres/device_keys_table.go
@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/lib/pq"
+
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/keyserver/api"
@@ -204,20 +205,17 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID
 		deviceIDMap[d] = true
 	}
 	var result []api.DeviceMessage
+	var displayName sql.NullString
 	for rows.Next() {
 		dk := api.DeviceMessage{
-			Type:       api.TypeDeviceKeyUpdate,
-			DeviceKeys: &api.DeviceKeys{},
+			Type: api.TypeDeviceKeyUpdate,
+			DeviceKeys: &api.DeviceKeys{
+				UserID: userID,
+			},
 		}
-		dk.UserID = userID
-		var keyJSON string
-		var streamID int64
-		var displayName sql.NullString
-		if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil {
+		if err := rows.Scan(&dk.DeviceID, &dk.KeyJSON, &dk.StreamID, &displayName); err != nil {
 			return nil, err
 		}
-		dk.KeyJSON = []byte(keyJSON)
-		dk.StreamID = streamID
 		if displayName.Valid {
 			dk.DisplayName = displayName.String
 		}
diff --git a/keyserver/storage/sqlite3/device_keys_table.go b/keyserver/storage/sqlite3/device_keys_table.go
index e77b49b35..73768da5b 100644
--- a/keyserver/storage/sqlite3/device_keys_table.go
+++ b/keyserver/storage/sqlite3/device_keys_table.go
@@ -137,21 +137,17 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectBatchDeviceKeysStmt: rows.close() failed")
 	var result []api.DeviceMessage
+	var displayName sql.NullString
 	for rows.Next() {
 		dk := api.DeviceMessage{
-			Type:       api.TypeDeviceKeyUpdate,
-			DeviceKeys: &api.DeviceKeys{},
+			Type: api.TypeDeviceKeyUpdate,
+			DeviceKeys: &api.DeviceKeys{
+				UserID: userID,
+			},
 		}
-		dk.Type = api.TypeDeviceKeyUpdate
-		dk.UserID = userID
-		var keyJSON string
-		var streamID int64
-		var displayName sql.NullString
-		if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil {
+		if err := rows.Scan(&dk.DeviceID, &dk.KeyJSON, &dk.StreamID, &displayName); err != nil {
 			return nil, err
 		}
-		dk.KeyJSON = []byte(keyJSON)
-		dk.StreamID = streamID
 		if displayName.Valid {
 			dk.DisplayName = displayName.String
 		}

From 8d8f4689a0c71f79051554e69fb9113592a4b2e4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 7 Oct 2022 12:21:38 +0100
Subject: [PATCH 23/42] tDatabase transaction tweaks in roomserver

---
 roomserver/storage/shared/storage.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index d83a1ff74..8859571b7 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -598,7 +598,7 @@ func (d *Database) storeEvent(
 	}
 	// First writer is with a database-provided transaction, so that NIDs are assigned
 	// globally outside of the updater context, to help avoid races.
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+	err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error {
 		// TODO: Here we should aim to have two different code paths for new rooms
 		// vs existing ones.
 
@@ -964,9 +964,9 @@ func (d *Database) loadRedactionPair(
 	}
 
 	if isRedactionEvent {
-		redactedEvent = d.loadEvent(ctx, info.RedactsEventID)
+		redactedEvent = d.loadEvent(ctx, txn, info.RedactsEventID)
 	} else {
-		redactionEvent = d.loadEvent(ctx, info.RedactionEventID)
+		redactionEvent = d.loadEvent(ctx, txn, info.RedactionEventID)
 	}
 
 	return redactionEvent, redactedEvent, info.Validated, nil
@@ -982,15 +982,15 @@ func (d *Database) applyRedactions(events []types.Event) {
 }
 
 // loadEvent loads a single event or returns nil on any problems/missing event
-func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event {
-	nids, err := d.EventNIDs(ctx, []string{eventID})
+func (d *Database) loadEvent(ctx context.Context, txn *sql.Tx, eventID string) *types.Event {
+	nids, err := d.eventNIDs(ctx, txn, []string{eventID}, NoFilter)
 	if err != nil {
 		return nil
 	}
 	if len(nids) == 0 {
 		return nil
 	}
-	evs, err := d.Events(ctx, []types.EventNID{nids[eventID]})
+	evs, err := d.events(ctx, txn, []types.EventNID{nids[eventID]})
 	if err != nil {
 		return nil
 	}
@@ -1358,7 +1358,7 @@ func (d *Database) ForgetRoom(ctx context.Context, userID, roomID string, forget
 	}
 
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.MembershipTable.UpdateForgetMembership(ctx, nil, roomNIDs[0], stateKeyNID, forget)
+		return d.MembershipTable.UpdateForgetMembership(ctx, txn, roomNIDs[0], stateKeyNID, forget)
 	})
 }
 

From 1b5460a920d271aa03c798709f2f1b3b710e4e0e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 7 Oct 2022 13:42:35 +0100
Subject: [PATCH 24/42] Ensure we only wake up a given user once (#2775)

This ensures that the sync API notifier only wakes up a given user once
for a given stream position.
---
 syncapi/notifier/notifier.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go
index db18c6b77..27f7c37ba 100644
--- a/syncapi/notifier/notifier.go
+++ b/syncapi/notifier/notifier.go
@@ -48,6 +48,7 @@ type Notifier struct {
 	lastCleanUpTime time.Time
 	// This map is reused to prevent allocations and GC pressure in SharedUsers.
 	_sharedUserMap map[string]struct{}
+	_wakeupUserMap map[string]struct{}
 }
 
 // NewNotifier creates a new notifier set to the given sync position.
@@ -61,6 +62,7 @@ func NewNotifier() *Notifier {
 		lock:                   &sync.RWMutex{},
 		lastCleanUpTime:        time.Now(),
 		_sharedUserMap:         map[string]struct{}{},
+		_wakeupUserMap:         map[string]struct{}{},
 	}
 }
 
@@ -408,12 +410,16 @@ func (n *Notifier) setPeekingDevices(roomIDToPeekingDevices map[string][]types.P
 // specified user IDs, and also the specified peekingDevices
 func (n *Notifier) _wakeupUsers(userIDs []string, peekingDevices []types.PeekingDevice, newPos types.StreamingToken) {
 	for _, userID := range userIDs {
+		n._wakeupUserMap[userID] = struct{}{}
+	}
+	for userID := range n._wakeupUserMap {
 		for _, stream := range n._fetchUserStreams(userID) {
 			if stream == nil {
 				continue
 			}
 			stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream
 		}
+		delete(n._wakeupUserMap, userID)
 	}
 
 	for _, peekingDevice := range peekingDevices {

From 8e231130e97ce716357bbb4af8f82159dc6e684e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 7 Oct 2022 14:05:06 +0100
Subject: [PATCH 25/42] Revert "tDatabase transaction tweaks in roomserver"

This reverts commit 8d8f4689a0c71f79051554e69fb9113592a4b2e4.
---
 roomserver/storage/shared/storage.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 8859571b7..d83a1ff74 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -598,7 +598,7 @@ func (d *Database) storeEvent(
 	}
 	// First writer is with a database-provided transaction, so that NIDs are assigned
 	// globally outside of the updater context, to help avoid races.
-	err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error {
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		// TODO: Here we should aim to have two different code paths for new rooms
 		// vs existing ones.
 
@@ -964,9 +964,9 @@ func (d *Database) loadRedactionPair(
 	}
 
 	if isRedactionEvent {
-		redactedEvent = d.loadEvent(ctx, txn, info.RedactsEventID)
+		redactedEvent = d.loadEvent(ctx, info.RedactsEventID)
 	} else {
-		redactionEvent = d.loadEvent(ctx, txn, info.RedactionEventID)
+		redactionEvent = d.loadEvent(ctx, info.RedactionEventID)
 	}
 
 	return redactionEvent, redactedEvent, info.Validated, nil
@@ -982,15 +982,15 @@ func (d *Database) applyRedactions(events []types.Event) {
 }
 
 // loadEvent loads a single event or returns nil on any problems/missing event
-func (d *Database) loadEvent(ctx context.Context, txn *sql.Tx, eventID string) *types.Event {
-	nids, err := d.eventNIDs(ctx, txn, []string{eventID}, NoFilter)
+func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event {
+	nids, err := d.EventNIDs(ctx, []string{eventID})
 	if err != nil {
 		return nil
 	}
 	if len(nids) == 0 {
 		return nil
 	}
-	evs, err := d.events(ctx, txn, []types.EventNID{nids[eventID]})
+	evs, err := d.Events(ctx, []types.EventNID{nids[eventID]})
 	if err != nil {
 		return nil
 	}
@@ -1358,7 +1358,7 @@ func (d *Database) ForgetRoom(ctx context.Context, userID, roomID string, forget
 	}
 
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.MembershipTable.UpdateForgetMembership(ctx, txn, roomNIDs[0], stateKeyNID, forget)
+		return d.MembershipTable.UpdateForgetMembership(ctx, nil, roomNIDs[0], stateKeyNID, forget)
 	})
 }
 

From 1ca3f3efb512db4c1827eb571588ec31258782a5 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 7 Oct 2022 16:00:12 +0200
Subject: [PATCH 26/42] Fix issue with DMs shown as normal rooms (#2776)

Fixes #2121, test added in
https://github.com/matrix-org/complement/pull/494
---
 federationapi/api/api.go                      |  1 +
 federationapi/internal/perform.go             | 19 +++++++++---
 roomserver/api/perform.go                     |  1 +
 roomserver/internal/helpers/helpers.go        | 31 ++++++++++---------
 roomserver/internal/perform/perform_join.go   | 20 ++++++++++--
 roomserver/internal/perform/perform_leave.go  |  2 +-
 roomserver/internal/query/query.go            |  2 +-
 roomserver/storage/interface.go               |  5 +--
 roomserver/storage/postgres/invite_table.go   | 13 ++++----
 roomserver/storage/shared/storage.go          |  9 +++---
 roomserver/storage/sqlite3/invite_table.go    | 13 ++++----
 roomserver/storage/tables/interface.go        |  2 +-
 .../storage/tables/invite_table_test.go       | 11 ++++---
 13 files changed, 82 insertions(+), 47 deletions(-)

diff --git a/federationapi/api/api.go b/federationapi/api/api.go
index f25538784..362333fc9 100644
--- a/federationapi/api/api.go
+++ b/federationapi/api/api.go
@@ -159,6 +159,7 @@ type PerformJoinRequest struct {
 	// The sorted list of servers to try. Servers will be tried sequentially, after de-duplication.
 	ServerNames types.ServerNames      `json:"server_names"`
 	Content     map[string]interface{} `json:"content"`
+	Unsigned    map[string]interface{} `json:"unsigned"`
 }
 
 type PerformJoinResponse struct {
diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 4cdd3a5eb..28ec48d7b 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -7,14 +7,15 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/federationapi/consumers"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/consumers"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/version"
 )
 
 // PerformLeaveRequest implements api.FederationInternalAPI
@@ -95,6 +96,7 @@ func (r *FederationInternalAPI) PerformJoin(
 			request.Content,
 			serverName,
 			supportedVersions,
+			request.Unsigned,
 		); err != nil {
 			logrus.WithError(err).WithFields(logrus.Fields{
 				"server_name": serverName,
@@ -139,6 +141,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	content map[string]interface{},
 	serverName gomatrixserverlib.ServerName,
 	supportedVersions []gomatrixserverlib.RoomVersion,
+	unsigned map[string]interface{},
 ) error {
 	// Try to perform a make_join using the information supplied in the
 	// request.
@@ -267,6 +270,14 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	// If we successfully performed a send_join above then the other
 	// server now thinks we're a part of the room. Send the newly
 	// returned state to the roomserver to update our local view.
+	if unsigned != nil {
+		event, err = event.SetUnsigned(unsigned)
+		if err != nil {
+			// non-fatal, log and continue
+			logrus.WithError(err).Errorf("Failed to set unsigned content")
+		}
+	}
+
 	if err = roomserverAPI.SendEventWithState(
 		context.Background(),
 		r.rsAPI,
diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index 20931f807..7a362f969 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -80,6 +80,7 @@ type PerformJoinRequest struct {
 	UserID        string                         `json:"user_id"`
 	Content       map[string]interface{}         `json:"content"`
 	ServerNames   []gomatrixserverlib.ServerName `json:"server_names"`
+	Unsigned      map[string]interface{}         `json:"unsigned"`
 }
 
 type PerformJoinResponse struct {
diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index cbd1561f7..3b83a0a6d 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -7,6 +7,9 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/auth"
 	"github.com/matrix-org/dendrite/roomserver/state"
@@ -14,8 +17,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
 // TODO: temporary package which has helper functions used by both internal/perform packages.
@@ -97,35 +98,35 @@ func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverNam
 func IsInvitePending(
 	ctx context.Context, db storage.Database,
 	roomID, userID string,
-) (bool, string, string, error) {
+) (bool, string, string, *gomatrixserverlib.Event, error) {
 	// Look up the room NID for the supplied room ID.
 	info, err := db.RoomInfo(ctx, roomID)
 	if err != nil {
-		return false, "", "", fmt.Errorf("r.DB.RoomInfo: %w", err)
+		return false, "", "", nil, fmt.Errorf("r.DB.RoomInfo: %w", err)
 	}
 	if info == nil {
-		return false, "", "", fmt.Errorf("cannot get RoomInfo: unknown room ID %s", roomID)
+		return false, "", "", nil, fmt.Errorf("cannot get RoomInfo: unknown room ID %s", roomID)
 	}
 
 	// Look up the state key NID for the supplied user ID.
 	targetUserNIDs, err := db.EventStateKeyNIDs(ctx, []string{userID})
 	if err != nil {
-		return false, "", "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err)
+		return false, "", "", nil, fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err)
 	}
 	targetUserNID, targetUserFound := targetUserNIDs[userID]
 	if !targetUserFound {
-		return false, "", "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs)
+		return false, "", "", nil, fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs)
 	}
 
 	// Let's see if we have an event active for the user in the room. If
 	// we do then it will contain a server name that we can direct the
 	// send_leave to.
-	senderUserNIDs, eventIDs, err := db.GetInvitesForUser(ctx, info.RoomNID, targetUserNID)
+	senderUserNIDs, eventIDs, eventJSON, err := db.GetInvitesForUser(ctx, info.RoomNID, targetUserNID)
 	if err != nil {
-		return false, "", "", fmt.Errorf("r.DB.GetInvitesForUser: %w", err)
+		return false, "", "", nil, fmt.Errorf("r.DB.GetInvitesForUser: %w", err)
 	}
 	if len(senderUserNIDs) == 0 {
-		return false, "", "", nil
+		return false, "", "", nil, nil
 	}
 	userNIDToEventID := make(map[types.EventStateKeyNID]string)
 	for i, nid := range senderUserNIDs {
@@ -135,18 +136,20 @@ func IsInvitePending(
 	// Look up the user ID from the NID.
 	senderUsers, err := db.EventStateKeys(ctx, senderUserNIDs)
 	if err != nil {
-		return false, "", "", fmt.Errorf("r.DB.EventStateKeys: %w", err)
+		return false, "", "", nil, fmt.Errorf("r.DB.EventStateKeys: %w", err)
 	}
 	if len(senderUsers) == 0 {
-		return false, "", "", fmt.Errorf("no senderUsers")
+		return false, "", "", nil, fmt.Errorf("no senderUsers")
 	}
 
 	senderUser, senderUserFound := senderUsers[senderUserNIDs[0]]
 	if !senderUserFound {
-		return false, "", "", fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers)
+		return false, "", "", nil, fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers)
 	}
 
-	return true, senderUser, userNIDToEventID[senderUserNIDs[0]], nil
+	event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false, info.RoomVersion)
+
+	return true, senderUser, userNIDToEventID[senderUserNIDs[0]], event, err
 }
 
 // GetMembershipsAtState filters the state events to
diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index 167b375b7..262273ff5 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -22,6 +22,10 @@ import (
 	"time"
 
 	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
+
 	fsAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -32,8 +36,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
 )
 
 type Joiner struct {
@@ -236,7 +238,7 @@ func (r *Joiner) performJoinRoomByID(
 
 	// Force a federated join if we're dealing with a pending invite
 	// and we aren't in the room.
-	isInvitePending, inviteSender, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID)
+	isInvitePending, inviteSender, _, inviteEvent, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID)
 	if err == nil && !serverInRoom && isInvitePending {
 		_, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender)
 		if ierr != nil {
@@ -248,6 +250,17 @@ func (r *Joiner) performJoinRoomByID(
 		if inviterDomain != r.Cfg.Matrix.ServerName {
 			req.ServerNames = append(req.ServerNames, inviterDomain)
 			forceFederatedJoin = true
+			memberEvent := gjson.Parse(string(inviteEvent.JSON()))
+			// only set unsigned if we've got a content.membership, which we _should_
+			if memberEvent.Get("content.membership").Exists() {
+				req.Unsigned = map[string]interface{}{
+					"prev_sender": memberEvent.Get("sender").Str,
+					"prev_content": map[string]interface{}{
+						"is_direct":  memberEvent.Get("content.is_direct").Bool(),
+						"membership": memberEvent.Get("content.membership").Str,
+					},
+				}
+			}
 		}
 	}
 
@@ -348,6 +361,7 @@ func (r *Joiner) performFederatedJoinRoomByID(
 		UserID:      req.UserID,        // the user ID joining the room
 		ServerNames: req.ServerNames,   // the server to try joining with
 		Content:     req.Content,       // the membership event content
+		Unsigned:    req.Unsigned,      // the unsigned event content, if any
 	}
 	fedRes := fsAPI.PerformJoinResponse{}
 	r.FSAPI.PerformJoin(ctx, &fedReq, &fedRes)
diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index ada3aab06..85b659814 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -79,7 +79,7 @@ func (r *Leaver) performLeaveRoomByID(
 ) ([]api.OutputEvent, error) {
 	// If there's an invite outstanding for the room then respond to
 	// that.
-	isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID)
+	isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID)
 	if err == nil && isInvitePending {
 		_, senderDomain, serr := gomatrixserverlib.SplitID('@', senderUser)
 		if serr != nil {
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index ee8e1cfe7..7a424a334 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -872,7 +872,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
 	// but we don't specify an authorised via user, since the event auth
 	// will allow the join anyway.
 	var pending bool
-	if pending, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil {
+	if pending, _, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil {
 		return fmt.Errorf("helpers.IsInvitePending: %w", err)
 	} else if pending {
 		res.Allowed = true
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 11e175f55..ee0624b21 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -17,10 +17,11 @@ package storage
 import (
 	"context"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
@@ -104,7 +105,7 @@ type Database interface {
 	// Look up the active invites targeting a user in a room and return the
 	// numeric state key IDs for the user IDs who sent them along with the event IDs for the invites.
 	// Returns an error if there was a problem talking to the database.
-	GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error)
+	GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, inviteEventJSON []byte, err error)
 	// Save a given room alias with the room ID it refers to.
 	// Returns an error if there was a problem talking to the database.
 	SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error
diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go
index 4cddfe2e9..009fd1ac1 100644
--- a/roomserver/storage/postgres/invite_table.go
+++ b/roomserver/storage/postgres/invite_table.go
@@ -61,7 +61,7 @@ const insertInviteEventSQL = "" +
 	" ON CONFLICT DO NOTHING"
 
 const selectInviteActiveForUserInRoomSQL = "" +
-	"SELECT invite_event_id, sender_nid FROM roomserver_invites" +
+	"SELECT invite_event_id, sender_nid, invite_event_json FROM roomserver_invites" +
 	" WHERE target_nid = $1 AND room_nid = $2" +
 	" AND NOT retired"
 
@@ -141,25 +141,26 @@ func (s *inviteStatements) UpdateInviteRetired(
 func (s *inviteStatements) SelectInviteActiveForUserInRoom(
 	ctx context.Context, txn *sql.Tx,
 	targetUserNID types.EventStateKeyNID, roomNID types.RoomNID,
-) ([]types.EventStateKeyNID, []string, error) {
+) ([]types.EventStateKeyNID, []string, []byte, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectInviteActiveForUserInRoomStmt)
 	rows, err := stmt.QueryContext(
 		ctx, targetUserNID, roomNID,
 	)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed")
 	var result []types.EventStateKeyNID
 	var eventIDs []string
 	var inviteEventID string
 	var senderUserNID int64
+	var eventJSON []byte
 	for rows.Next() {
-		if err := rows.Scan(&inviteEventID, &senderUserNID); err != nil {
-			return nil, nil, err
+		if err := rows.Scan(&inviteEventID, &senderUserNID, &eventJSON); err != nil {
+			return nil, nil, nil, err
 		}
 		result = append(result, types.EventStateKeyNID(senderUserNID))
 		eventIDs = append(eventIDs, inviteEventID)
 	}
-	return result, eventIDs, rows.Err()
+	return result, eventIDs, eventJSON, rows.Err()
 }
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index d83a1ff74..e401f17dc 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -7,13 +7,14 @@ import (
 	"fmt"
 	"sort"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/tidwall/gjson"
+
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/tidwall/gjson"
 )
 
 // Ideally, when we have both events we should redact the event JSON and forget about the redaction, but we currently
@@ -445,7 +446,7 @@ func (d *Database) GetInvitesForUser(
 	ctx context.Context,
 	roomNID types.RoomNID,
 	targetUserNID types.EventStateKeyNID,
-) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error) {
+) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, inviteEventJSON []byte, err error) {
 	return d.InvitesTable.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
 }
 
diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go
index e051d63af..ca6e7c511 100644
--- a/roomserver/storage/sqlite3/invite_table.go
+++ b/roomserver/storage/sqlite3/invite_table.go
@@ -44,7 +44,7 @@ const insertInviteEventSQL = "" +
 	" ON CONFLICT DO NOTHING"
 
 const selectInviteActiveForUserInRoomSQL = "" +
-	"SELECT invite_event_id, sender_nid FROM roomserver_invites" +
+	"SELECT invite_event_id, sender_nid, invite_event_json FROM roomserver_invites" +
 	" WHERE target_nid = $1 AND room_nid = $2" +
 	" AND NOT retired"
 
@@ -136,25 +136,26 @@ func (s *inviteStatements) UpdateInviteRetired(
 func (s *inviteStatements) SelectInviteActiveForUserInRoom(
 	ctx context.Context, txn *sql.Tx,
 	targetUserNID types.EventStateKeyNID, roomNID types.RoomNID,
-) ([]types.EventStateKeyNID, []string, error) {
+) ([]types.EventStateKeyNID, []string, []byte, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectInviteActiveForUserInRoomStmt)
 	rows, err := stmt.QueryContext(
 		ctx, targetUserNID, roomNID,
 	)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed")
 	var result []types.EventStateKeyNID
 	var eventIDs []string
 	var eventID string
 	var senderUserNID int64
+	var eventJSON []byte
 	for rows.Next() {
-		if err := rows.Scan(&eventID, &senderUserNID); err != nil {
-			return nil, nil, err
+		if err := rows.Scan(&eventID, &senderUserNID, &eventJSON); err != nil {
+			return nil, nil, nil, err
 		}
 		result = append(result, types.EventStateKeyNID(senderUserNID))
 		eventIDs = append(eventIDs, eventID)
 	}
-	return result, eventIDs, nil
+	return result, eventIDs, eventJSON, nil
 }
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index d7bcc95ab..8be47855f 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -116,7 +116,7 @@ type Invites interface {
 	InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte) (bool, error)
 	UpdateInviteRetired(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) ([]string, error)
 	// SelectInviteActiveForUserInRoom returns a list of sender state key NIDs and invite event IDs matching those nids.
-	SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, error)
+	SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, []byte, error)
 }
 
 type MembershipState int64
diff --git a/roomserver/storage/tables/invite_table_test.go b/roomserver/storage/tables/invite_table_test.go
index 8df3faa2d..e3eedbf14 100644
--- a/roomserver/storage/tables/invite_table_test.go
+++ b/roomserver/storage/tables/invite_table_test.go
@@ -4,6 +4,9 @@ import (
 	"context"
 	"testing"
 
+	"github.com/matrix-org/util"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
@@ -11,8 +14,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/util"
-	"github.com/stretchr/testify/assert"
 )
 
 func mustCreateInviteTable(t *testing.T, dbType test.DBType) (tables.Invites, func()) {
@@ -67,7 +68,7 @@ func TestInviteTable(t *testing.T) {
 		assert.NoError(t, err)
 		assert.True(t, newInvite)
 
-		stateKeyNIDs, eventIDs, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
+		stateKeyNIDs, eventIDs, _, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
 		assert.NoError(t, err)
 		assert.Equal(t, []string{eventID1, eventID2}, eventIDs)
 		assert.Equal(t, []types.EventStateKeyNID{2, 2}, stateKeyNIDs)
@@ -78,13 +79,13 @@ func TestInviteTable(t *testing.T) {
 		assert.Equal(t, []string{eventID1, eventID2}, retiredEventIDs)
 
 		// This should now be empty
-		stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
+		stateKeyNIDs, eventIDs, _, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
 		assert.NoError(t, err)
 		assert.Empty(t, eventIDs)
 		assert.Empty(t, stateKeyNIDs)
 
 		// Non-existent targetUserNID
-		stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID)
+		stateKeyNIDs, eventIDs, _, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID)
 		assert.NoError(t, err)
 		assert.Empty(t, stateKeyNIDs)
 		assert.Empty(t, eventIDs)

From f1b8df0f49a8a2f2c3c9c4d421b880bd2b008e08 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 7 Oct 2022 15:55:29 +0100
Subject: [PATCH 27/42] Version 0.10.2 (#2778)

Changelog and version bump.
---
 CHANGES.md          | 22 ++++++++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index dbe2ccf02..7552e41e5 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,27 @@
 # Changelog
 
+## Dendrite 0.10.2 (2022-10-07)
+
+### Features
+
+* Dendrite will now fail to start if there is an obvious problem with the configured `max_open_conns` when using PostgreSQL database backends, since this can lead to instability and performance issues
+  * More information on this is available [in the documentation](https://matrix-org.github.io/dendrite/installation/start/optimisation#postgresql-connection-limit)
+* Unnecessary/empty fields will no longer be sent in `/sync` responses
+* It is now possible to configure `old_private_keys` from previous Matrix installations on the same domain if only public key is known, to make it easier to expire old keys correctly
+  * You can configure either just the `private_key` path, or you can supply both the `public_key` and `key_id`
+
+### Fixes
+
+* The sync transaction behaviour has been modified further so that errors in one stream should not propagate to other streams unnecessarily
+* Rooms should now be classified as DM rooms correctly by passing through `is_direct` and unsigned hints
+* A bug which caused marking device lists as stale to consume lots of CPU has been fixed
+* Users accepting invites should no longer cause unnecessary federated joins if there are already other local users in the room
+* The sync API state range queries have been optimised by adding missing indexes
+* It should now be possible to configure non-English languages for full-text search in `search.language`
+* The roomserver will no longer attempt to perform federated requests to the local server when trying to fetch missing events
+* The `/keys/upload` endpoint will now always return the `one_time_keys_counts`, which may help with E2EE reliability
+* The sync API will now retrieve the latest stream position before processing each stream rather than at the beginning of the request, to hopefully reduce the number of round-trips to `/sync`
+
 ## Dendrite 0.10.1 (2022-09-30)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index d508517be..56b83f852 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 10
-	VersionPatch = 1
+	VersionPatch = 2
 	VersionTag   = "" // example: "rc1"
 )
 

From 980fa55846811eeff89f116c49b38b085143c64e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 10:39:29 +0100
Subject: [PATCH 28/42] Stronger passwordless account checks (fixes #2780)

---
 userapi/internal/api.go           | 2 ++
 userapi/storage/shared/storage.go | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 591faffd6..2f7795dfe 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -838,6 +838,8 @@ func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.Q
 		return nil
 	case bcrypt.ErrMismatchedHashAndPassword: // user exists, but password doesn't match
 		return nil
+	case bcrypt.ErrHashTooShort: // user exists, but probably a passwordless account
+		return nil
 	default:
 		res.Exists = true
 		res.Account = acc
diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go
index 3ff299f1b..09eeedc9f 100644
--- a/userapi/storage/shared/storage.go
+++ b/userapi/storage/shared/storage.go
@@ -75,6 +75,9 @@ func (d *Database) GetAccountByPassword(
 	if err != nil {
 		return nil, err
 	}
+	if hash == "" {
+		return nil, bcrypt.ErrHashTooShort
+	}
 	if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil {
 		return nil, err
 	}

From 04bab142901b9dc4479b2cf4e5504fbdd9dd3cf1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 10:45:15 +0100
Subject: [PATCH 29/42] Add regression test for
 980fa55846811eeff89f116c49b38b085143c64e

---
 userapi/userapi_test.go | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go
index 31a69793b..984fe8854 100644
--- a/userapi/userapi_test.go
+++ b/userapi/userapi_test.go
@@ -151,6 +151,33 @@ func TestQueryProfile(t *testing.T) {
 	})
 }
 
+// TestPasswordlessLoginFails ensures that a passwordless account cannot
+// be logged into using an arbitrary password (effectively a regression test
+// for https://github.com/matrix-org/dendrite/issues/2780).
+func TestPasswordlessLoginFails(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType)
+		defer close()
+		_, err := accountDB.CreateAccount(ctx, "auser", "", "", api.AccountTypeAppService)
+		if err != nil {
+			t.Fatalf("failed to make account: %s", err)
+		}
+
+		userReq := &api.QueryAccountByPasswordRequest{
+			Localpart:         "auser",
+			PlaintextPassword: "apassword",
+		}
+		userRes := &api.QueryAccountByPasswordResponse{}
+		if err := userAPI.QueryAccountByPassword(ctx, userReq, userRes); err != nil {
+			t.Fatal(err)
+		}
+		if userRes.Exists || userRes.Account != nil {
+			t.Fatal("QueryAccountByPassword should not return correctly for a passwordless account")
+		}
+	})
+}
+
 func TestLoginToken(t *testing.T) {
 	ctx := context.Background()
 

From b32b6d6e8eb5d0a6c22e9215e2fccf4bdd1db8c3 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 11:03:52 +0100
Subject: [PATCH 30/42] Update issue and pull request templates

---
 .github/ISSUE_TEMPLATE/BUG_REPORT.md | 22 ++++++++++++----------
 .github/PULL_REQUEST_TEMPLATE.md     |  4 ++--
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
index 206713e04..49c9a37ef 100644
--- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md
+++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
@@ -7,24 +7,27 @@ about: Create a report to help us improve
 <!--
 All bug reports must provide the following background information
 Text between <!-- and --​> marks will be invisible in the report.
+
+IF YOUR ISSUE IS CONSIDERED A SECURITY VULNERABILITY THEN PLEASE STOP AND
+DO NOT POST IT AS A GITHUB ISSUE! Please report the issue responsibly by
+disclosing in private by email to security@matrix.org instead.
 -->
 
 ### Background information
 <!-- Please include versions of all software when known e.g database versions, docker versions, client versions -->
-- **Dendrite version or git SHA**: 
-- **Monolith or Polylith?**: 
-- **SQLite3 or Postgres?**: 
-- **Running in Docker?**: 
+- **Dendrite version or git SHA**:
+- **Monolith or Polylith?**:
+- **SQLite3 or Postgres?**:
+- **Running in Docker?**:
 - **`go version`**:
 - **Client used (if applicable)**:
 
-
 ### Description
 
- - **What** is the problem:
- - **Who** is affected:
- - **How** is this bug manifesting:
- - **When** did this first appear:
+- **What** is the problem:
+- **Who** is affected:
+- **How** is this bug manifesting:
+- **When** did this first appear:
 
 <!--
 Examples of good descriptions:
@@ -38,7 +41,6 @@ Examples of good descriptions:
 - How: "Lots of logs about device change updates"
 - When: "After my server joined Matrix HQ"
 
-
 Examples of bad descriptions:
 - What: "Can't send messages"  - This is bad because it isn't specfic enough. Which endpoint isn't working and what is the response code? Does the message send but encryption fail?
 - Who: "Me" - Who are you? Running the server or a user on a Dendrite server?
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index e0b82e2aa..d6c54ead1 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,8 +1,8 @@
 ### Pull Request Checklist
 
-<!-- Please read docs/CONTRIBUTING.md before submitting your pull request -->
+<!-- Please read https://matrix-org.github.io/dendrite/development/contributing before submitting your pull request -->
 
 * [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests.
-* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)
+* [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately
 
 Signed-off-by: `Your Name <your@email.example.org>`

From 80a0ab6246aa095f428430c38b13861406dd5c78 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 11:09:40 +0100
Subject: [PATCH 31/42] Further tweak to the issue template

---
 .github/ISSUE_TEMPLATE/BUG_REPORT.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
index 49c9a37ef..f40c56609 100644
--- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md
+++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
@@ -8,9 +8,10 @@ about: Create a report to help us improve
 All bug reports must provide the following background information
 Text between <!-- and --​> marks will be invisible in the report.
 
-IF YOUR ISSUE IS CONSIDERED A SECURITY VULNERABILITY THEN PLEASE STOP AND
-DO NOT POST IT AS A GITHUB ISSUE! Please report the issue responsibly by
-disclosing in private by email to security@matrix.org instead.
+IF YOUR ISSUE IS CONSIDERED A SECURITY VULNERABILITY THEN PLEASE STOP
+AND DO NOT POST IT AS A GITHUB ISSUE! Please report the issue responsibly by
+disclosing in private by email to security@matrix.org instead. For more details, please
+see: https://www.matrix.org/security-disclosure-policy/
 -->
 
 ### Background information

From fb6cb2dbcbeb7cd7546ca4d126394720d215c310 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 11:14:16 +0100
Subject: [PATCH 32/42] Tweak `GetAccountByPassword` more

---
 clientapi/auth/password.go        | 6 ++++++
 userapi/storage/shared/storage.go | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go
index bcb4ca97b..890b18183 100644
--- a/clientapi/auth/password.go
+++ b/clientapi/auth/password.go
@@ -68,6 +68,12 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			JSON: jsonerror.BadJSON("A username must be supplied."),
 		}
 	}
+	if len(r.Password) == 0 {
+		return nil, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.BadJSON("A password must be supplied."),
+		}
+	}
 	localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName)
 	if err != nil {
 		return nil, &util.JSONResponse{
diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go
index 09eeedc9f..4e28f7b5a 100644
--- a/userapi/storage/shared/storage.go
+++ b/userapi/storage/shared/storage.go
@@ -75,7 +75,7 @@ func (d *Database) GetAccountByPassword(
 	if err != nil {
 		return nil, err
 	}
-	if hash == "" {
+	if len(hash) == 0 && len(plaintextPassword) > 0 {
 		return nil, bcrypt.ErrHashTooShort
 	}
 	if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil {

From 0f09e9d196d3375e38a490881e06668a82fb6c40 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 10 Oct 2022 12:19:16 +0200
Subject: [PATCH 33/42] Move /event to the SyncAPI (#2782)

This allows us to apply history visibility without having to recalculate
it in the roomserver.
Unblocks https://github.com/matrix-org/complement/pull/495, fix missing
part of https://github.com/matrix-org/dendrite/issues/617
---
 clientapi/routing/getevent.go          | 138 -------------------------
 clientapi/routing/routing.go           |   9 --
 docs/caddy/polylith/Caddyfile          |   2 +-
 docs/hiawatha/polylith-sample.conf     |   4 +-
 docs/nginx/polylith-sample.conf        |   4 +-
 syncapi/internal/history_visibility.go |   8 +-
 syncapi/routing/getevent.go            | 102 ++++++++++++++++++
 syncapi/routing/routing.go             |  10 ++
 8 files changed, 124 insertions(+), 153 deletions(-)
 delete mode 100644 clientapi/routing/getevent.go
 create mode 100644 syncapi/routing/getevent.go

diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go
deleted file mode 100644
index 7f5842800..000000000
--- a/clientapi/routing/getevent.go
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright 2019 Alex Chen
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package routing
-
-import (
-	"net/http"
-
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/config"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-type getEventRequest struct {
-	req            *http.Request
-	device         *userapi.Device
-	roomID         string
-	eventID        string
-	cfg            *config.ClientAPI
-	requestedEvent *gomatrixserverlib.Event
-}
-
-// GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}
-// https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid
-func GetEvent(
-	req *http.Request,
-	device *userapi.Device,
-	roomID string,
-	eventID string,
-	cfg *config.ClientAPI,
-	rsAPI api.ClientRoomserverAPI,
-) util.JSONResponse {
-	eventsReq := api.QueryEventsByIDRequest{
-		EventIDs: []string{eventID},
-	}
-	var eventsResp api.QueryEventsByIDResponse
-	err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if len(eventsResp.Events) == 0 {
-		// Event not found locally
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
-		}
-	}
-
-	requestedEvent := eventsResp.Events[0].Event
-
-	r := getEventRequest{
-		req:            req,
-		device:         device,
-		roomID:         roomID,
-		eventID:        eventID,
-		cfg:            cfg,
-		requestedEvent: requestedEvent,
-	}
-
-	stateReq := api.QueryStateAfterEventsRequest{
-		RoomID:       r.requestedEvent.RoomID(),
-		PrevEventIDs: r.requestedEvent.PrevEventIDs(),
-		StateToFetch: []gomatrixserverlib.StateKeyTuple{{
-			EventType: gomatrixserverlib.MRoomMember,
-			StateKey:  device.UserID,
-		}},
-	}
-	var stateResp api.QueryStateAfterEventsResponse
-	if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if !stateResp.RoomExists {
-		util.GetLogger(req.Context()).Errorf("Expected to find room for event %s but failed", r.requestedEvent.EventID())
-		return jsonerror.InternalServerError()
-	}
-
-	if !stateResp.PrevEventsExist {
-		// Missing some events locally; stateResp.StateEvents unavailable.
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
-		}
-	}
-
-	var appService *config.ApplicationService
-	if device.AppserviceID != "" {
-		for _, as := range cfg.Derived.ApplicationServices {
-			if as.ID == device.AppserviceID {
-				appService = &as
-				break
-			}
-		}
-	}
-
-	for _, stateEvent := range stateResp.StateEvents {
-		if appService != nil {
-			if !appService.IsInterestedInUserID(*stateEvent.StateKey()) {
-				continue
-			}
-		} else if !stateEvent.StateKeyEquals(device.UserID) {
-			continue
-		}
-		membership, err := stateEvent.Membership()
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("stateEvent.Membership failed")
-			return jsonerror.InternalServerError()
-		}
-		if membership == gomatrixserverlib.Join {
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: gomatrixserverlib.ToClientEvent(r.requestedEvent, gomatrixserverlib.FormatAll),
-			}
-		}
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusNotFound,
-		JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
-	}
-}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 7d1c434c4..f1fa66ca6 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -367,15 +367,6 @@ func Setup(
 				nil, cfg, rsAPI, transactionsCache)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
-	v3mux.Handle("/rooms/{roomID}/event/{eventID}",
-		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
-		}),
-	).Methods(http.MethodGet, http.MethodOptions)
 
 	v3mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile
index 244e50e7e..906097e4e 100644
--- a/docs/caddy/polylith/Caddyfile
+++ b/docs/caddy/polylith/Caddyfile
@@ -55,7 +55,7 @@ matrix.example.com {
 	# Change the end of each reverse_proxy line to the correct
 	# address for your various services.
 	@sync_api {
-		path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$
+		path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$
 	}
 	reverse_proxy @sync_api sync_api:8073
 
diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf
index 5ed0cb5ae..036140643 100644
--- a/docs/hiawatha/polylith-sample.conf
+++ b/docs/hiawatha/polylith-sample.conf
@@ -18,8 +18,10 @@ VirtualHost {
         # /_matrix/client/.*/user/{userId}/filter/{filterID}
         # /_matrix/client/.*/keys/changes
         # /_matrix/client/.*/rooms/{roomId}/messages
+        # /_matrix/client/.*/rooms/{roomId}/context/{eventID}
+        # /_matrix/client/.*/rooms/{roomId}/event/{eventID}
         # to sync_api
-        ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 600
+        ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ http://localhost:8073 600
         ReverseProxy = /_matrix/client http://localhost:8071 600
         ReverseProxy = /_matrix/federation http://localhost:8072 600
         ReverseProxy = /_matrix/key http://localhost:8072 600
diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf
index 274d75658..345d8a6b4 100644
--- a/docs/nginx/polylith-sample.conf
+++ b/docs/nginx/polylith-sample.conf
@@ -28,8 +28,10 @@ server {
     # /_matrix/client/.*/user/{userId}/filter/{filterID}
     # /_matrix/client/.*/keys/changes
     # /_matrix/client/.*/rooms/{roomId}/messages
+    # /_matrix/client/.*/rooms/{roomId}/context/{eventID}
+    # /_matrix/client/.*/rooms/{roomId}/event/{eventID}
     # to sync_api
-    location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$  {
+    location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$  {
         proxy_pass http://sync_api:8073;
     }
 
diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go
index bbfe19f4c..71d7ddd15 100644
--- a/syncapi/internal/history_visibility.go
+++ b/syncapi/internal/history_visibility.go
@@ -19,11 +19,13 @@ import (
 	"math"
 	"time"
 
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage"
 )
 
 func init() {
@@ -189,7 +191,7 @@ func visibilityForEvents(
 		UserID:   userID,
 	}, membershipResp)
 	if err != nil {
-		return result, err
+		logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'")
 	}
 
 	// Create a map from eventID -> eventVisibility
diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go
new file mode 100644
index 000000000..d2cdc1b5f
--- /dev/null
+++ b/syncapi/routing/getevent.go
@@ -0,0 +1,102 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/internal"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+// GetEvent implements
+//
+//	GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}
+//
+// https://spec.matrix.org/v1.4/client-server-api/#get_matrixclientv3roomsroomideventeventid
+func GetEvent(
+	req *http.Request,
+	device *userapi.Device,
+	roomID string,
+	eventID string,
+	cfg *config.SyncAPI,
+	syncDB storage.Database,
+	rsAPI api.SyncRoomserverAPI,
+) util.JSONResponse {
+	ctx := req.Context()
+	db, err := syncDB.NewDatabaseTransaction(ctx)
+	logger := util.GetLogger(ctx).WithFields(logrus.Fields{
+		"event_id": eventID,
+		"room_id":  roomID,
+	})
+	if err != nil {
+		logger.WithError(err).Error("GetEvent: syncDB.NewDatabaseTransaction failed")
+		return jsonerror.InternalServerError()
+	}
+
+	events, err := db.Events(ctx, []string{eventID})
+	if err != nil {
+		logger.WithError(err).Error("GetEvent: syncDB.Events failed")
+		return jsonerror.InternalServerError()
+	}
+
+	// The requested event does not exist in our database
+	if len(events) == 0 {
+		logger.Debugf("GetEvent: requested event doesn't exist locally")
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
+		}
+	}
+
+	// If the request is coming from an appservice, get the user from the request
+	userID := device.UserID
+	if asUserID := req.FormValue("user_id"); device.AppserviceID != "" && asUserID != "" {
+		userID = asUserID
+	}
+
+	// Apply history visibility to determine if the user is allowed to view the event
+	events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, userID, "event")
+	if err != nil {
+		logger.WithError(err).Error("GetEvent: internal.ApplyHistoryVisibilityFilter failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+
+	// We only ever expect there to be one event
+	if len(events) != 1 {
+		// 0 events -> not allowed to view event; > 1 events -> something that shouldn't happen
+		logger.WithField("event_count", len(events)).Debug("GetEvent: can't return the requested event")
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: gomatrixserverlib.HeaderedToClientEvent(events[0], gomatrixserverlib.FormatAll),
+	}
+}
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index 8f84a1341..069dee81f 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -60,6 +60,16 @@ func Setup(
 		return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, rsAPI, cfg, srp, lazyLoadCache)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
+	v3mux.Handle("/rooms/{roomID}/event/{eventID}",
+		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, syncDB, rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	v3mux.Handle("/user/{userId}/filter",
 		httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))

From dcc01162874d2634738080614f02cbbeb6bc598f Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 10 Oct 2022 15:38:00 +0200
Subject: [PATCH 34/42] SyTest List Maintenance

---
 sytest-blacklist | 1 -
 sytest-whitelist | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/sytest-blacklist b/sytest-blacklist
index 5b2e973a6..634c07cf3 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -47,7 +47,6 @@ Notifications can be viewed with GET /notifications
 
 # More flakey
 
-If remote user leaves room we no longer receive device updates
 Guest users can join guest_access rooms
 
 # This will fail in HTTP API mode, so blacklisted for now
diff --git a/sytest-whitelist b/sytest-whitelist
index 31940b884..9ba9df75d 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -742,3 +742,4 @@ User in private room doesn't appear in user directory
 User joining then leaving public room appears and dissappears from directory
 User in remote room doesn't appear in user directory after server left room
 User in shared private room does appear in user directory until leave
+Existing members see new member's presence
\ No newline at end of file

From 39581af3ba657032fbf66ba3719a1cb334c0519b Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 10 Oct 2022 15:49:56 +0200
Subject: [PATCH 35/42] CI update

---
 .github/workflows/dendrite.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index be3c7c173..f8019b3ea 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -342,7 +342,7 @@ jobs:
         # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
+          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
 
       - name: Run actions/checkout@v2 for dendrite
         uses: actions/checkout@v2

From b000db81ca889b350663493c27eb58654c75e295 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 10 Oct 2022 17:36:26 +0200
Subject: [PATCH 36/42] Send E2EE related errors to sentry (#2784)

Only sends errors if we're not retrying them in NATS.
Not sure if those should be scoped/tagged with something like "E2EE".
---
 federationapi/consumers/keychange.go    | 17 ++++++++++++++---
 federationapi/consumers/sendtodevice.go | 14 ++++++++++----
 federationapi/queue/queue.go            |  3 +++
 federationapi/routing/send.go           |  4 ++++
 4 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go
index f3314bc98..67dfdc1d3 100644
--- a/federationapi/consumers/keychange.go
+++ b/federationapi/consumers/keychange.go
@@ -18,6 +18,11 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/types"
@@ -26,9 +31,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	"github.com/sirupsen/logrus"
 )
 
 // KeyChangeConsumer consumes events that originate in key server.
@@ -78,6 +80,7 @@ func (t *KeyChangeConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) boo
 	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	var m api.DeviceMessage
 	if err := json.Unmarshal(msg.Data, &m); err != nil {
+		sentry.CaptureException(err)
 		logrus.WithError(err).Errorf("failed to read device message from key change topic")
 		return true
 	}
@@ -105,6 +108,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 	// only send key change events which originated from us
 	_, originServerName, err := gomatrixserverlib.SplitID('@', m.UserID)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("Failed to extract domain from key change event")
 		return true
 	}
@@ -118,6 +122,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 		WantMembership: "join",
 	}, &queryRes)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to calculate joined rooms for user")
 		return true
 	}
@@ -125,6 +130,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 	// send this key change to all servers who share rooms with this user.
 	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in")
 		return true
 	}
@@ -147,6 +153,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 		Keys:              m.KeyJSON,
 	}
 	if edu.Content, err = json.Marshal(event); err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to marshal EDU JSON")
 		return true
 	}
@@ -160,6 +167,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 	output := m.CrossSigningKeyUpdate
 	_, host, err := gomatrixserverlib.SplitID('@', output.UserID)
 	if err != nil {
+		sentry.CaptureException(err)
 		logrus.WithError(err).Errorf("fedsender key change consumer: user ID parse failure")
 		return true
 	}
@@ -176,12 +184,14 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 		WantMembership: "join",
 	}, &queryRes)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user")
 		return true
 	}
 	// send this key change to all servers who share rooms with this user.
 	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in")
 		return true
 	}
@@ -196,6 +206,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 		Origin: string(t.serverName),
 	}
 	if edu.Content, err = json.Marshal(output); err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to marshal output, dropping")
 		return true
 	}
diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go
index ffc1d8894..9aec22a3e 100644
--- a/federationapi/consumers/sendtodevice.go
+++ b/federationapi/consumers/sendtodevice.go
@@ -18,16 +18,18 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/nats-io/nats.go"
-	log "github.com/sirupsen/logrus"
 )
 
 // OutputSendToDeviceConsumer consumes events that originate in the clientapi.
@@ -76,6 +78,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
 	sender := msg.Header.Get("sender")
 	_, originServerName, err := gomatrixserverlib.SplitID('@', sender)
 	if err != nil {
+		sentry.CaptureException(err)
 		log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender")
 		return true
 	}
@@ -85,12 +88,14 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
 	// Extract the send-to-device event from msg.
 	var ote syncTypes.OutputSendToDeviceEvent
 	if err = json.Unmarshal(msg.Data, &ote); err != nil {
+		sentry.CaptureException(err)
 		log.WithError(err).Errorf("output log: message parse failed (expected send-to-device)")
 		return true
 	}
 
 	_, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID)
 	if err != nil {
+		sentry.CaptureException(err)
 		log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination")
 		return true
 	}
@@ -116,6 +121,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
 		},
 	}
 	if edu.Content, err = json.Marshal(tdm); err != nil {
+		sentry.CaptureException(err)
 		log.WithError(err).Error("failed to marshal EDU JSON")
 		return true
 	}
diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go
index 88664fcf9..8245aa5bd 100644
--- a/federationapi/queue/queue.go
+++ b/federationapi/queue/queue.go
@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
@@ -307,11 +308,13 @@ func (oqs *OutgoingQueues) SendEDU(
 
 	ephemeralJSON, err := json.Marshal(e)
 	if err != nil {
+		sentry.CaptureException(err)
 		return fmt.Errorf("json.Marshal: %w", err)
 	}
 
 	nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(ephemeralJSON))
 	if err != nil {
+		sentry.CaptureException(err)
 		return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
 	}
 
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 060af676d..b3bbaa394 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -22,6 +22,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
@@ -350,6 +351,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				for deviceID, message := range byUser {
 					// TODO: check that the user and the device actually exist here
 					if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil {
+						sentry.CaptureException(err)
 						util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
 							"sender":    directPayload.Sender,
 							"user_id":   userID,
@@ -360,6 +362,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			}
 		case gomatrixserverlib.MDeviceListUpdate:
 			if err := t.producer.SendDeviceListUpdate(ctx, e.Content, t.Origin); err != nil {
+				sentry.CaptureException(err)
 				util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate")
 			}
 		case gomatrixserverlib.MReceipt:
@@ -395,6 +398,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			}
 		case types.MSigningKeyUpdate:
 			if err := t.producer.SendSigningKeyUpdate(ctx, e.Content, t.Origin); err != nil {
+				sentry.CaptureException(err)
 				logrus.WithError(err).Errorf("Failed to process signing key update")
 			}
 		case gomatrixserverlib.MPresence:

From 6bf1912525724baa1a8bf5e0bf51524d7cee59ba Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 16:54:04 +0100
Subject: [PATCH 37/42] Fix joined hosts with `RewritesState` (#2785)

This ensures that the joined hosts in the federation API are correct
after the state is rewritten. This might fix some races around the time
of joining federated rooms.
---
 federationapi/storage/shared/storage.go | 32 ++++++++++++-------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/federationapi/storage/shared/storage.go b/federationapi/storage/shared/storage.go
index a00d782f1..9e40f311c 100644
--- a/federationapi/storage/shared/storage.go
+++ b/federationapi/storage/shared/storage.go
@@ -70,27 +70,27 @@ func (d *Database) UpdateRoom(
 ) (joinedHosts []types.JoinedHost, err error) {
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		if purgeRoomFirst {
-			// If the event is a create event then we'll delete all of the existing
-			// data for the room. The only reason that a create event would be replayed
-			// to us in this way is if we're about to receive the entire room state.
 			if err = d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
 				return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err)
 			}
-		}
-
-		joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID)
-		if err != nil {
-			return err
-		}
-
-		for _, add := range addHosts {
-			err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName)
-			if err != nil {
+			for _, add := range addHosts {
+				if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil {
+					return err
+				}
+				joinedHosts = append(joinedHosts, add)
+			}
+		} else {
+			if joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID); err != nil {
+				return err
+			}
+			for _, add := range addHosts {
+				if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil {
+					return err
+				}
+			}
+			if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil {
 				return err
 			}
-		}
-		if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil {
-			return err
 		}
 		return nil
 	})

From 9ed8ff6b938e0452d122c68b38cf4fedcd74c0bf Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 11 Oct 2022 10:48:36 +0100
Subject: [PATCH 38/42] Tweak federation `M_NOT_FOUND` errors

---
 federationapi/routing/events.go |  6 +++++-
 federationapi/routing/state.go  | 15 ++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go
index 23796edfa..6168912bd 100644
--- a/federationapi/routing/events.go
+++ b/federationapi/routing/events.go
@@ -20,6 +20,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -95,7 +96,10 @@ func fetchEvent(ctx context.Context, rsAPI api.FederationRoomserverAPI, eventID
 	}
 
 	if len(eventsResponse.Events) == 0 {
-		return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
+		return nil, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Event not found"),
+		}
 	}
 
 	return eventsResponse.Events[0].Event, nil
diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go
index 5377eb88f..1d08d0a82 100644
--- a/federationapi/routing/state.go
+++ b/federationapi/routing/state.go
@@ -135,23 +135,24 @@ func getState(
 		return nil, nil, &resErr
 	}
 
-	if !response.StateKnown {
+	switch {
+	case !response.RoomExists:
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Room not found"),
+		}
+	case !response.StateKnown:
 		return nil, nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound("State not known"),
 		}
-	}
-	if response.IsRejected {
+	case response.IsRejected:
 		return nil, nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound("Event not found"),
 		}
 	}
 
-	if !response.RoomExists {
-		return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
-	}
-
 	return response.StateEvents, response.AuthChainEvents, nil
 }
 

From 3920b9f9b6155db69822d0dbcd36acb1eaa51c34 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 11 Oct 2022 10:58:34 +0100
Subject: [PATCH 39/42] Tweak `GetStateDeltas` behaviour (#2788)

Improves the control flow of `GetStateDeltas` for clarity and possibly
also fixes a bug where duplicate state delta entries could be inserted
with different memberships instead of being correctly overridden by
`join`.
---
 syncapi/storage/shared/storage_sync.go | 44 ++++++++++++++++++--------
 1 file changed, 30 insertions(+), 14 deletions(-)

diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go
index 0e19d97d2..d5b5b3121 100644
--- a/syncapi/storage/shared/storage_sync.go
+++ b/syncapi/storage/shared/storage_sync.go
@@ -360,34 +360,50 @@ func (d *DatabaseTransaction) GetStateDeltas(
 	newlyJoinedRooms := make(map[string]bool, len(state))
 	for roomID, stateStreamEvents := range state {
 		for _, ev := range stateStreamEvents {
-			if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" {
-				if membership == gomatrixserverlib.Join && prevMembership != membership {
-					// send full room state down instead of a delta
+			// Look for our membership in the state events and skip over any
+			// membership events that are not related to us.
+			membership, prevMembership := getMembershipFromEvent(ev.Event, userID)
+			if membership == "" {
+				continue
+			}
+
+			if membership == gomatrixserverlib.Join {
+				// If our membership is now join but the previous membership wasn't
+				// then this is a "join transition", so we'll insert this room.
+				if prevMembership != membership {
+					// Get the full room state, as we'll send that down for a newly
+					// joined room instead of a delta.
 					var s []types.StreamEvent
-					s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter)
-					if err != nil {
+					if s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter); err != nil {
 						if err == sql.ErrNoRows {
 							continue
 						}
 						return nil, nil, err
 					}
+
+					// Add the information for this room into the state so that
+					// it will get added with all of the rest of the joined rooms.
 					state[roomID] = s
 					newlyJoinedRooms[roomID] = true
-					continue // we'll add this room in when we do joined rooms
 				}
 
-				deltas = append(deltas, types.StateDelta{
-					Membership:    membership,
-					MembershipPos: ev.StreamPosition,
-					StateEvents:   d.StreamEventsToEvents(device, stateStreamEvents),
-					RoomID:        roomID,
-				})
-				break
+				// We won't add joined rooms into the delta at this point as they
+				// are added later on.
+				continue
 			}
+
+			deltas = append(deltas, types.StateDelta{
+				Membership:    membership,
+				MembershipPos: ev.StreamPosition,
+				StateEvents:   d.StreamEventsToEvents(device, stateStreamEvents),
+				RoomID:        roomID,
+			})
+			break
 		}
 	}
 
-	// Add in currently joined rooms
+	// Finally, add in currently joined rooms, including those from the
+	// join transitions above.
 	for _, joinedRoomID := range joinedRoomIDs {
 		deltas = append(deltas, types.StateDelta{
 			Membership:  gomatrixserverlib.Join,

From 0a9aebdf011921680e5b0646bf50d7900423aa69 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 11 Oct 2022 12:27:21 +0100
Subject: [PATCH 40/42] Private read receipts (#2789)

Implement behaviours for `m.read.private` receipts.
---
 clientapi/routing/account_data.go   | 44 ++++++++++++++---------------
 clientapi/routing/receipt.go        | 36 ++++++++++++++++++-----
 clientapi/routing/routing.go        |  2 +-
 federationapi/consumers/receipts.go |  8 ++++++
 internal/eventutil/types.go         |  5 ++--
 syncapi/streams/stream_receipt.go   |  4 +++
 6 files changed, 66 insertions(+), 33 deletions(-)

diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go
index b28f0bb1f..4742b1240 100644
--- a/clientapi/routing/account_data.go
+++ b/clientapi/routing/account_data.go
@@ -154,33 +154,31 @@ func SaveReadMarker(
 		return *resErr
 	}
 
-	if r.FullyRead == "" {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"),
+	if r.FullyRead != "" {
+		data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead})
+		if err != nil {
+			return jsonerror.InternalServerError()
+		}
+
+		dataReq := api.InputAccountDataRequest{
+			UserID:      device.UserID,
+			DataType:    "m.fully_read",
+			RoomID:      roomID,
+			AccountData: data,
+		}
+		dataRes := api.InputAccountDataResponse{}
+		if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
+			return util.ErrorResponse(err)
 		}
 	}
 
-	data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead})
-	if err != nil {
-		return jsonerror.InternalServerError()
-	}
-
-	dataReq := api.InputAccountDataRequest{
-		UserID:      device.UserID,
-		DataType:    "m.fully_read",
-		RoomID:      roomID,
-		AccountData: data,
-	}
-	dataRes := api.InputAccountDataResponse{}
-	if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
-		return util.ErrorResponse(err)
-	}
-
-	// Handle the read receipt that may be included in the read marker
+	// Handle the read receipts that may be included in the read marker.
 	if r.Read != "" {
-		return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read)
+		return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read", r.Read)
+	}
+	if r.ReadPrivate != "" {
+		return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read.private", r.ReadPrivate)
 	}
 
 	return util.JSONResponse{
diff --git a/clientapi/routing/receipt.go b/clientapi/routing/receipt.go
index 0f9b1b4ff..99217a780 100644
--- a/clientapi/routing/receipt.go
+++ b/clientapi/routing/receipt.go
@@ -15,19 +15,22 @@
 package routing
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/matrix-org/dendrite/userapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )
 
-func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
+func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
 	timestamp := gomatrixserverlib.AsTimestamp(time.Now())
 	logrus.WithFields(logrus.Fields{
 		"roomID":      roomID,
@@ -37,13 +40,32 @@ func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, devi
 		"timestamp":   timestamp,
 	}).Debug("Setting receipt")
 
-	// currently only m.read is accepted
-	if receiptType != "m.read" {
-		return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType))
-	}
+	switch receiptType {
+	case "m.read", "m.read.private":
+		if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil {
+			return util.ErrorResponse(err)
+		}
 
-	if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil {
-		return util.ErrorResponse(err)
+	case "m.fully_read":
+		data, err := json.Marshal(fullyReadEvent{EventID: eventID})
+		if err != nil {
+			return jsonerror.InternalServerError()
+		}
+
+		dataReq := api.InputAccountDataRequest{
+			UserID:      device.UserID,
+			DataType:    "m.fully_read",
+			RoomID:      roomID,
+			AccountData: data,
+		}
+		dataRes := api.InputAccountDataResponse{}
+		if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
+			return util.ErrorResponse(err)
+		}
+
+	default:
+		return util.MessageResponse(400, fmt.Sprintf("Receipt type '%s' not known", receiptType))
 	}
 
 	return util.JSONResponse{
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index f1fa66ca6..e72880ec5 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -1343,7 +1343,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 
-			return SetReceipt(req, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"])
+			return SetReceipt(req, userAPI, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"])
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/presence/{userId}/status",
diff --git a/federationapi/consumers/receipts.go b/federationapi/consumers/receipts.go
index 366cb264e..75827cb68 100644
--- a/federationapi/consumers/receipts.go
+++ b/federationapi/consumers/receipts.go
@@ -81,6 +81,14 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg)
 		Type:    msg.Header.Get("type"),
 	}
 
+	switch receipt.Type {
+	case "m.read":
+		// These are allowed to be sent over federation
+	case "m.read.private", "m.fully_read":
+		// These must not be sent over federation
+		return true
+	}
+
 	// only send receipt events which originated from us
 	_, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID)
 	if err != nil {
diff --git a/internal/eventutil/types.go b/internal/eventutil/types.go
index afc62d8c2..18175d6a0 100644
--- a/internal/eventutil/types.go
+++ b/internal/eventutil/types.go
@@ -35,8 +35,9 @@ type AccountData struct {
 }
 
 type ReadMarkerJSON struct {
-	FullyRead string `json:"m.fully_read"`
-	Read      string `json:"m.read"`
+	FullyRead   string `json:"m.fully_read"`
+	Read        string `json:"m.read"`
+	ReadPrivate string `json:"m.read.private"`
 }
 
 // NotificationData contains statistics about notifications, sent from
diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index bba911022..977815078 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -67,6 +67,10 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 		if _, ok := req.IgnoredUsers.List[receipt.UserID]; ok {
 			continue
 		}
+		// Don't send private read receipts to other users
+		if receipt.Type == "m.read.private" && req.Device.UserID != receipt.UserID {
+			continue
+		}
 		receiptsByRoom[receipt.RoomID] = append(receiptsByRoom[receipt.RoomID], receipt)
 	}
 

From 3c1474f68f2ac5e564d2bd4bd15f01d2a73f2845 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 11 Oct 2022 16:04:02 +0200
Subject: [PATCH 41/42] Fix `/get_missing_events` for rooms with
 `joined`/`invited` history_visibility (#2787)

Sytest was using a wrong `history_visibility` for `invited`
(https://github.com/matrix-org/sytest/pull/1303), so `invited` was
passing for the wrong reason (-> defaulted to `shared`, as `invite`
wasn't understood).
This change now handles missing events like Synapse, if a server isn't
allowed to see the event, it gets a redacted version of it, making the
`get_missing_events` tests pass.
---
 are-we-synapse-yet.list                       |  2 +-
 roomserver/auth/auth.go                       | 30 +++++++------------
 roomserver/internal/helpers/helpers.go        | 15 +++++-----
 .../internal/perform/perform_backfill.go      |  5 +++-
 roomserver/internal/query/query.go            |  6 ++--
 .../storage/postgres/state_snapshot_table.go  |  4 ++-
 sytest-whitelist                              |  5 ++--
 7 files changed, 33 insertions(+), 34 deletions(-)

diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list
index c776a7400..81c0f8049 100644
--- a/are-we-synapse-yet.list
+++ b/are-we-synapse-yet.list
@@ -643,7 +643,7 @@ fed Inbound federation redacts events from erased users
 fme Outbound federation can request missing events 
 fme Inbound federation can return missing events for world_readable visibility 
 fme Inbound federation can return missing events for shared visibility 
-fme Inbound federation can return missing events for invite visibility 
+fme Inbound federation can return missing events for invited visibility
 fme Inbound federation can return missing events for joined visibility 
 fme outliers whose auth_events are in a different room are correctly rejected 
 fbk Outbound federation can backfill events 
diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go
index aa1d5bc25..31a856e8e 100644
--- a/roomserver/auth/auth.go
+++ b/roomserver/auth/auth.go
@@ -13,8 +13,6 @@
 package auth
 
 import (
-	"encoding/json"
-
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -30,7 +28,7 @@ func IsServerAllowed(
 	historyVisibility := HistoryVisibilityForRoom(authEvents)
 
 	// 1. If the history_visibility was set to world_readable, allow.
-	if historyVisibility == "world_readable" {
+	if historyVisibility == gomatrixserverlib.HistoryVisibilityWorldReadable {
 		return true
 	}
 	// 2. If the user's membership was join, allow.
@@ -39,12 +37,12 @@ func IsServerAllowed(
 		return true
 	}
 	// 3. If history_visibility was set to shared, and the user joined the room at any point after the event was sent, allow.
-	if historyVisibility == "shared" && serverCurrentlyInRoom {
+	if historyVisibility == gomatrixserverlib.HistoryVisibilityShared && serverCurrentlyInRoom {
 		return true
 	}
 	// 4. If the user's membership was invite, and the history_visibility was set to invited, allow.
 	invitedUserExists := IsAnyUserOnServerWithMembership(serverName, authEvents, gomatrixserverlib.Invite)
-	if invitedUserExists && historyVisibility == "invited" {
+	if invitedUserExists && historyVisibility == gomatrixserverlib.HistoryVisibilityInvited {
 		return true
 	}
 
@@ -52,27 +50,16 @@ func IsServerAllowed(
 	return false
 }
 
-func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) string {
+func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) gomatrixserverlib.HistoryVisibility {
 	// https://matrix.org/docs/spec/client_server/r0.6.0#id87
 	// By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared.
-	visibility := "shared"
-	knownStates := []string{"invited", "joined", "shared", "world_readable"}
+	visibility := gomatrixserverlib.HistoryVisibilityShared
 	for _, ev := range authEvents {
 		if ev.Type() != gomatrixserverlib.MRoomHistoryVisibility {
 			continue
 		}
-		// TODO: This should be HistoryVisibilityContent to match things like 'MemberContent'. Do this when moving to GMSL
-		content := struct {
-			HistoryVisibility string `json:"history_visibility"`
-		}{}
-		if err := json.Unmarshal(ev.Content(), &content); err != nil {
-			break // value is not understood
-		}
-		for _, s := range knownStates {
-			if s == content.HistoryVisibility {
-				visibility = s
-				break
-			}
+		if vis, err := ev.HistoryVisibility(); err == nil {
+			visibility = vis
 		}
 	}
 	return visibility
@@ -80,6 +67,9 @@ func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) string {
 
 func IsAnyUserOnServerWithMembership(serverName gomatrixserverlib.ServerName, authEvents []*gomatrixserverlib.Event, wantMembership string) bool {
 	for _, ev := range authEvents {
+		if ev.Type() != gomatrixserverlib.MRoomMember {
+			continue
+		}
 		membership, err := ev.Membership()
 		if err != nil || membership != wantMembership {
 			continue
diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index 3b83a0a6d..a6de8ac84 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -324,7 +324,7 @@ func slowGetHistoryVisibilityState(
 func ScanEventTree(
 	ctx context.Context, db storage.Database, info *types.RoomInfo, front []string, visited map[string]bool, limit int,
 	serverName gomatrixserverlib.ServerName,
-) ([]types.EventNID, error) {
+) ([]types.EventNID, map[string]struct{}, error) {
 	var resultNIDs []types.EventNID
 	var err error
 	var allowed bool
@@ -345,6 +345,7 @@ func ScanEventTree(
 
 	var checkedServerInRoom bool
 	var isServerInRoom bool
+	redactEventIDs := make(map[string]struct{})
 
 	// Loop through the event IDs to retrieve the requested events and go
 	// through the whole tree (up to the provided limit) using the events'
@@ -358,7 +359,7 @@ BFSLoop:
 		// Retrieve the events to process from the database.
 		events, err = db.EventsFromIDs(ctx, front)
 		if err != nil {
-			return resultNIDs, err
+			return resultNIDs, redactEventIDs, err
 		}
 
 		if !checkedServerInRoom && len(events) > 0 {
@@ -395,16 +396,16 @@ BFSLoop:
 						)
 						// drop the error, as we will often error at the DB level if we don't have the prev_event itself. Let's
 						// just return what we have.
-						return resultNIDs, nil
+						return resultNIDs, redactEventIDs, nil
 					}
 
 					// If the event hasn't been seen before and the HS
 					// requesting to retrieve it is allowed to do so, add it to
 					// the list of events to retrieve.
-					if allowed {
-						next = append(next, pre)
-					} else {
+					next = append(next, pre)
+					if !allowed {
 						util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).Info("Not allowed to see event")
+						redactEventIDs[pre] = struct{}{}
 					}
 				}
 			}
@@ -413,7 +414,7 @@ BFSLoop:
 		front = next
 	}
 
-	return resultNIDs, err
+	return resultNIDs, redactEventIDs, err
 }
 
 func QueryLatestEventsAndState(
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index 69a075733..57e121ea2 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -78,7 +78,7 @@ func (r *Backfiller) PerformBackfill(
 	}
 
 	// Scan the event tree for events to send back.
-	resultNIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName)
+	resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName)
 	if err != nil {
 		return err
 	}
@@ -95,6 +95,9 @@ func (r *Backfiller) PerformBackfill(
 	}
 
 	for _, event := range loadedEvents {
+		if _, ok := redactEventIDs[event.EventID()]; ok {
+			event.Redact()
+		}
 		response.Events = append(response.Events, event.Headered(info.RoomVersion))
 	}
 
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 7a424a334..f41132403 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -453,7 +453,7 @@ func (r *Queryer) QueryMissingEvents(
 		return fmt.Errorf("missing RoomInfo for room %s", events[0].RoomID())
 	}
 
-	resultNIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName)
+	resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName)
 	if err != nil {
 		return err
 	}
@@ -470,7 +470,9 @@ func (r *Queryer) QueryMissingEvents(
 			if verr != nil {
 				return verr
 			}
-
+			if _, ok := redactEventIDs[event.EventID()]; ok {
+				event.Redact()
+			}
 			response.Events = append(response.Events, event.Headered(roomVersion))
 		}
 	}
diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go
index 99c76befe..a00c026f4 100644
--- a/roomserver/storage/postgres/state_snapshot_table.go
+++ b/roomserver/storage/postgres/state_snapshot_table.go
@@ -21,10 +21,11 @@ import (
 	"fmt"
 
 	"github.com/lib/pq"
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/util"
 )
 
 const stateSnapshotSchema = `
@@ -91,6 +92,7 @@ const bulkSelectStateForHistoryVisibilitySQL = `
 	      WHERE state_snapshot_nid = $1
 	    )
 	  )
+	  ORDER BY depth ASC
 	) AS roomserver_events
 	INNER JOIN roomserver_event_state_keys
 	  ON roomserver_events.event_state_key_nid = roomserver_event_state_keys.event_state_key_nid
diff --git a/sytest-whitelist b/sytest-whitelist
index 9ba9df75d..a3218ed70 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -306,7 +306,7 @@ Alternative server names do not cause a routing loop
 Events whose auth_events are in the wrong room do not mess up the room state
 Inbound federation can return events
 Inbound federation can return missing events for world_readable visibility
-Inbound federation can return missing events for invite visibility
+Inbound federation can return missing events for invited visibility
 Inbound federation can get public room list
 PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message
 PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message
@@ -742,4 +742,5 @@ User in private room doesn't appear in user directory
 User joining then leaving public room appears and dissappears from directory
 User in remote room doesn't appear in user directory after server left room
 User in shared private room does appear in user directory until leave
-Existing members see new member's presence
\ No newline at end of file
+Existing members see new member's presence
+Inbound federation can return missing events for joined visibility
\ No newline at end of file

From 23a3e04579172de89266e9554428b71172c58495 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 13 Oct 2022 14:50:52 +0100
Subject: [PATCH 42/42] Event relations (#2790)

This adds support for tracking `m.relates_to`, as well as adding support
for the various `/room/{roomID}/relations/...` endpoints to the CS API.
---
 docs/caddy/polylith/Caddyfile               | 107 ++++++-----
 docs/hiawatha/polylith-sample.conf          |   5 +-
 docs/nginx/polylith-sample.conf             |   5 +-
 go.mod                                      |   2 +-
 go.sum                                      |   4 +-
 syncapi/consumers/roomserver.go             |  27 +++
 syncapi/routing/relations.go                | 124 +++++++++++++
 syncapi/routing/routing.go                  |  43 +++++
 syncapi/storage/interface.go                |   4 +
 syncapi/storage/postgres/relations_table.go | 158 +++++++++++++++++
 syncapi/storage/postgres/syncserver.go      |   5 +
 syncapi/storage/shared/storage_consumer.go  |  35 +++-
 syncapi/storage/shared/storage_sync.go      |  81 +++++++++
 syncapi/storage/sqlite3/relations_table.go  | 163 +++++++++++++++++
 syncapi/storage/sqlite3/stream_id_table.go  |   8 +
 syncapi/storage/sqlite3/syncserver.go       |   5 +
 syncapi/storage/tables/interface.go         |  19 ++
 syncapi/storage/tables/relations_test.go    | 186 ++++++++++++++++++++
 syncapi/types/types.go                      |  13 ++
 19 files changed, 943 insertions(+), 51 deletions(-)
 create mode 100644 syncapi/routing/relations.go
 create mode 100644 syncapi/storage/postgres/relations_table.go
 create mode 100644 syncapi/storage/sqlite3/relations_table.go
 create mode 100644 syncapi/storage/tables/relations_test.go

diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile
index 906097e4e..8aeb9317f 100644
--- a/docs/caddy/polylith/Caddyfile
+++ b/docs/caddy/polylith/Caddyfile
@@ -1,66 +1,85 @@
-# Sample Caddyfile for using Caddy in front of Dendrite.
-#
-# Customize email address and domain names.
-# Optional settings commented out.
-#
-# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST.
-# Documentation: https://caddyserver.com/docs/
-#
-# Bonus tip: If your IP address changes, use Caddy's
-# dynamic DNS plugin to update your DNS records to
-# point to your new IP automatically:
-# https://github.com/mholt/caddy-dynamicdns
+# Sample Caddyfile for using Caddy in front of Dendrite
+
 #
 
+# Customize email address and domain names
+
+# Optional settings commented out
+
+#
+
+# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST
+
+# Documentation: <https://caddyserver.com/docs/>
+
+#
+
+# Bonus tip: If your IP address changes, use Caddy's
+
+# dynamic DNS plugin to update your DNS records to
+
+# point to your new IP automatically
+
+# <https://github.com/mholt/caddy-dynamicdns>
+
+#
 
 # Global options block
+
 {
-	# In case there is a problem with your certificates.
-	# email example@example.com
+ # In case there is a problem with your certificates.
+ # email example@example.com
 
-	# Turn off the admin endpoint if you don't need graceful config
-	# changes and/or are running untrusted code on your machine.
-	# admin off
+ # Turn off the admin endpoint if you don't need graceful config
+ # changes and/or are running untrusted code on your machine.
+ # admin off
 
-	# Enable this if your clients don't send ServerName in TLS handshakes.
-	# default_sni example.com
+ # Enable this if your clients don't send ServerName in TLS handshakes.
+ # default_sni example.com
 
-	# Enable debug mode for verbose logging.
-	# debug
+ # Enable debug mode for verbose logging.
+ # debug
 
-	# Use Let's Encrypt's staging endpoint for testing.
-	# acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
+ # Use Let's Encrypt's staging endpoint for testing.
+ # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
 
-	# If you're port-forwarding HTTP/HTTPS ports from 80/443 to something
-	# else, enable these and put the alternate port numbers here.
-	# http_port  8080
-	# https_port 8443
+ # If you're port-forwarding HTTP/HTTPS ports from 80/443 to something
+ # else, enable these and put the alternate port numbers here.
+ # http_port  8080
+ # https_port 8443
 }
 
 # The server name of your matrix homeserver. This example shows
-# "well-known delegation" from the registered domain to a subdomain,
+
+# "well-known delegation" from the registered domain to a subdomain
+
 # which is only needed if your server_name doesn't match your Matrix
+
 # homeserver URL (i.e. you can show users a vanity domain that looks
+
 # nice and is easy to remember but still have your Matrix server on
-# its own subdomain or hosted service).
+
+# its own subdomain or hosted service)
+
 example.com {
-	header /.well-known/matrix/* Content-Type application/json
-	header /.well-known/matrix/* Access-Control-Allow-Origin *
-	respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}`
-	respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}`
+ header /.well-known/matrix/*Content-Type application/json
+ header /.well-known/matrix/* Access-Control-Allow-Origin *
+ respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}`
+ respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}`
 }
 
-# The actual domain name whereby your Matrix server is accessed.
+# The actual domain name whereby your Matrix server is accessed
+
 matrix.example.com {
-	# Change the end of each reverse_proxy line to the correct
-	# address for your various services.
-	@sync_api {
-		path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$
-	}
-	reverse_proxy @sync_api sync_api:8073
+ # Change the end of each reverse_proxy line to the correct
+ # address for your various services.
+ @sync_api {
+  path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$
+ }
+ reverse_proxy @sync_api sync_api:8073
 
-	reverse_proxy /_matrix/client* client_api:8071
-	reverse_proxy /_matrix/federation* federation_api:8071
-	reverse_proxy /_matrix/key* federation_api:8071
-	reverse_proxy /_matrix/media* media_api:8071
+ reverse_proxy /_matrix/client* client_api:8071
+ reverse_proxy /_matrix/federation* federation_api:8071
+ reverse_proxy /_matrix/key* federation_api:8071
+ reverse_proxy /_matrix/media* media_api:8071
 }
diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf
index 036140643..0093fdcf2 100644
--- a/docs/hiawatha/polylith-sample.conf
+++ b/docs/hiawatha/polylith-sample.conf
@@ -20,8 +20,11 @@ VirtualHost {
         # /_matrix/client/.*/rooms/{roomId}/messages
         # /_matrix/client/.*/rooms/{roomId}/context/{eventID}
         # /_matrix/client/.*/rooms/{roomId}/event/{eventID}
+        # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}
+        # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}
+        # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType}
         # to sync_api
-        ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ http://localhost:8073 600
+        ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ http://localhost:8073 600
         ReverseProxy = /_matrix/client http://localhost:8071 600
         ReverseProxy = /_matrix/federation http://localhost:8072 600
         ReverseProxy = /_matrix/key http://localhost:8072 600
diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf
index 345d8a6b4..6e81eb5f2 100644
--- a/docs/nginx/polylith-sample.conf
+++ b/docs/nginx/polylith-sample.conf
@@ -30,8 +30,11 @@ server {
     # /_matrix/client/.*/rooms/{roomId}/messages
     # /_matrix/client/.*/rooms/{roomId}/context/{eventID}
     # /_matrix/client/.*/rooms/{roomId}/event/{eventID}
+    # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}
+    # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}
+    # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType}
     # to sync_api
-    location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$  {
+    location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$  {
         proxy_pass http://sync_api:8073;
     }
 
diff --git a/go.mod b/go.mod
index c82f76d41..eefad89e6 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64
 	github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
diff --git a/go.sum b/go.sum
index a99599cb1..0d08ac692 100644
--- a/go.sum
+++ b/go.sum
@@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621 h1:a8IaoSPDxevkgXnOUrtIW9AqVNvXBJAG0gtnX687S7g=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64 h1:QJmfAPC3P0ZHJzYD/QtbNc5EztKlK1ipRWP5SO/m4jw=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk=
 github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index c7a11dbb4..cfbb05327 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -148,6 +148,16 @@ func (s *OutputRoomEventConsumer) onRedactEvent(
 		log.WithError(err).Error("RedactEvent error'd")
 		return err
 	}
+
+	if err = s.db.RedactRelations(ctx, msg.RedactedBecause.RoomID(), msg.RedactedEventID); err != nil {
+		log.WithFields(log.Fields{
+			"room_id":           msg.RedactedBecause.RoomID(),
+			"event_id":          msg.RedactedBecause.EventID(),
+			"redacted_event_id": msg.RedactedEventID,
+		}).WithError(err).Warn("Failed to redact relations")
+		return err
+	}
+
 	// fake a room event so we notify clients about the redaction, as if it were
 	// a normal event.
 	return s.onNewRoomEvent(ctx, api.OutputNewRoomEvent{
@@ -271,6 +281,14 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 		return err
 	}
 
+	if err = s.db.UpdateRelations(ctx, ev); err != nil {
+		log.WithFields(log.Fields{
+			"event_id": ev.EventID(),
+			"type":     ev.Type(),
+		}).WithError(err).Warn("Failed to update relations")
+		return err
+	}
+
 	s.pduStream.Advance(pduPos)
 	s.notifier.OnNewEvent(ev, ev.RoomID(), nil, types.StreamingToken{PDUPosition: pduPos})
 
@@ -315,6 +333,15 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent(
 		}).WithError(err).Warn("failed to index fulltext element")
 	}
 
+	if err = s.db.UpdateRelations(ctx, ev); err != nil {
+		log.WithFields(log.Fields{
+			"room_id":  ev.RoomID(),
+			"event_id": ev.EventID(),
+			"type":     ev.Type(),
+		}).WithError(err).Warn("Failed to update relations")
+		return err
+	}
+
 	if pduPos, err = s.notifyJoinedPeeks(ctx, ev, pduPos); err != nil {
 		log.WithError(err).Errorf("Failed to notifyJoinedPeeks for PDU pos %d", pduPos)
 		return err
diff --git a/syncapi/routing/relations.go b/syncapi/routing/relations.go
new file mode 100644
index 000000000..fee61b0df
--- /dev/null
+++ b/syncapi/routing/relations.go
@@ -0,0 +1,124 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/internal"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+type RelationsResponse struct {
+	Chunk     []gomatrixserverlib.ClientEvent `json:"chunk"`
+	NextBatch string                          `json:"next_batch,omitempty"`
+	PrevBatch string                          `json:"prev_batch,omitempty"`
+}
+
+// nolint:gocyclo
+func Relations(
+	req *http.Request, device *userapi.Device,
+	syncDB storage.Database,
+	rsAPI api.SyncRoomserverAPI,
+	roomID, eventID, relType, eventType string,
+) util.JSONResponse {
+	var err error
+	var from, to types.StreamPosition
+	var limit int
+	dir := req.URL.Query().Get("dir")
+	if f := req.URL.Query().Get("from"); f != "" {
+		if from, err = types.NewStreamPositionFromString(f); err != nil {
+			return util.ErrorResponse(err)
+		}
+	}
+	if t := req.URL.Query().Get("to"); t != "" {
+		if to, err = types.NewStreamPositionFromString(t); err != nil {
+			return util.ErrorResponse(err)
+		}
+	}
+	if l := req.URL.Query().Get("limit"); l != "" {
+		if limit, err = strconv.Atoi(l); err != nil {
+			return util.ErrorResponse(err)
+		}
+	}
+	if limit == 0 || limit > 50 {
+		limit = 50
+	}
+	if dir == "" {
+		dir = "b"
+	}
+	if dir != "b" && dir != "f" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Bad or missing dir query parameter (should be either 'b' or 'f')"),
+		}
+	}
+
+	snapshot, err := syncDB.NewDatabaseSnapshot(req.Context())
+	if err != nil {
+		logrus.WithError(err).Error("Failed to get snapshot for relations")
+		return jsonerror.InternalServerError()
+	}
+	var succeeded bool
+	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
+
+	res := &RelationsResponse{
+		Chunk: []gomatrixserverlib.ClientEvent{},
+	}
+	var events []types.StreamEvent
+	events, res.PrevBatch, res.NextBatch, err = snapshot.RelationsFor(
+		req.Context(), roomID, eventID, relType, eventType, from, to, dir == "b", limit,
+	)
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+
+	headeredEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(events))
+	for _, event := range events {
+		headeredEvents = append(headeredEvents, event.HeaderedEvent)
+	}
+
+	// Apply history visibility to the result events.
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(req.Context(), snapshot, rsAPI, headeredEvents, nil, device.UserID, "relations")
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+
+	// Convert the events into client events, and optionally filter based on the event
+	// type if it was specified.
+	res.Chunk = make([]gomatrixserverlib.ClientEvent, 0, len(filteredEvents))
+	for _, event := range filteredEvents {
+		res.Chunk = append(
+			res.Chunk,
+			gomatrixserverlib.ToClientEvent(event.Event, gomatrixserverlib.FormatAll),
+		)
+	}
+
+	succeeded = true
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index 069dee81f..71fa93c1e 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -45,6 +45,7 @@ func Setup(
 	lazyLoadCache caching.LazyLoadCache,
 	fts *fulltext.Search,
 ) {
+	v1unstablemux := csMux.PathPrefix("/{apiversion:(?:v1|unstable)}/").Subrouter()
 	v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
 
 	// TODO: Add AS support for all handlers below.
@@ -110,6 +111,48 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	v1unstablemux.Handle("/rooms/{roomId}/relations/{eventId}",
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+
+			return Relations(
+				req, device, syncDB, rsAPI,
+				vars["roomId"], vars["eventId"], "", "",
+			)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v1unstablemux.Handle("/rooms/{roomId}/relations/{eventId}/{relType}",
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+
+			return Relations(
+				req, device, syncDB, rsAPI,
+				vars["roomId"], vars["eventId"], vars["relType"], "",
+			)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v1unstablemux.Handle("/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}",
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+
+			return Relations(
+				req, device, syncDB, rsAPI,
+				vars["roomId"], vars["eventId"], vars["relType"], vars["eventType"],
+			)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	v3mux.Handle("/search",
 		httputil.MakeAuthAPI("search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if !cfg.Fulltext.Enabled {
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 4a03aca74..02d45f801 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -38,6 +38,7 @@ type DatabaseTransaction interface {
 	MaxStreamPositionForSendToDeviceMessages(ctx context.Context) (types.StreamPosition, error)
 	MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error)
 	MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error)
+	MaxStreamPositionForRelations(ctx context.Context) (types.StreamPosition, error)
 
 	CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error)
 	GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error)
@@ -107,6 +108,7 @@ type DatabaseTransaction interface {
 	GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error)
 	GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
 	PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
+	RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error)
 }
 
 type Database interface {
@@ -174,6 +176,8 @@ type Database interface {
 	StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error)
 	UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error
 	ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error)
+	UpdateRelations(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error
+	RedactRelations(ctx context.Context, roomID, redactedEventID string) error
 }
 
 type Presence interface {
diff --git a/syncapi/storage/postgres/relations_table.go b/syncapi/storage/postgres/relations_table.go
new file mode 100644
index 000000000..5a76e9c33
--- /dev/null
+++ b/syncapi/storage/postgres/relations_table.go
@@ -0,0 +1,158 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
+	"github.com/matrix-org/dendrite/syncapi/types"
+)
+
+const relationsSchema = `
+CREATE SEQUENCE IF NOT EXISTS syncapi_relation_id;
+
+CREATE TABLE IF NOT EXISTS syncapi_relations (
+	id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_relation_id'),
+	room_id TEXT NOT NULL,
+	event_id TEXT NOT NULL,
+	child_event_id TEXT NOT NULL,
+	child_event_type TEXT NOT NULL,
+	rel_type TEXT NOT NULL,
+	CONSTRAINT syncapi_relations_unique UNIQUE (room_id, event_id, child_event_id, rel_type)
+);
+`
+
+const insertRelationSQL = "" +
+	"INSERT INTO syncapi_relations (" +
+	"  room_id, event_id, child_event_id, child_event_type, rel_type" +
+	") VALUES ($1, $2, $3, $4, $5) " +
+	" ON CONFLICT DO NOTHING"
+
+const deleteRelationSQL = "" +
+	"DELETE FROM syncapi_relations WHERE room_id = $1 AND child_event_id = $2"
+
+const selectRelationsInRangeAscSQL = "" +
+	"SELECT id, child_event_id, rel_type FROM syncapi_relations" +
+	" WHERE room_id = $1 AND event_id = $2" +
+	" AND ( $3 = '' OR rel_type = $3 )" +
+	" AND ( $4 = '' OR child_event_type = $4 )" +
+	" AND id > $5 AND id <= $6" +
+	" ORDER BY id ASC LIMIT $7"
+
+const selectRelationsInRangeDescSQL = "" +
+	"SELECT id, child_event_id, rel_type FROM syncapi_relations" +
+	" WHERE room_id = $1 AND event_id = $2" +
+	" AND ( $3 = '' OR rel_type = $3 )" +
+	" AND ( $4 = '' OR child_event_type = $4 )" +
+	" AND id >= $5 AND id < $6" +
+	" ORDER BY id DESC LIMIT $7"
+
+const selectMaxRelationIDSQL = "" +
+	"SELECT COALESCE(MAX(id), 0) FROM syncapi_relations"
+
+type relationsStatements struct {
+	insertRelationStmt             *sql.Stmt
+	selectRelationsInRangeAscStmt  *sql.Stmt
+	selectRelationsInRangeDescStmt *sql.Stmt
+	deleteRelationStmt             *sql.Stmt
+	selectMaxRelationIDStmt        *sql.Stmt
+}
+
+func NewPostgresRelationsTable(db *sql.DB) (tables.Relations, error) {
+	s := &relationsStatements{}
+	_, err := db.Exec(relationsSchema)
+	if err != nil {
+		return nil, err
+	}
+	return s, sqlutil.StatementList{
+		{&s.insertRelationStmt, insertRelationSQL},
+		{&s.selectRelationsInRangeAscStmt, selectRelationsInRangeAscSQL},
+		{&s.selectRelationsInRangeDescStmt, selectRelationsInRangeDescSQL},
+		{&s.deleteRelationStmt, deleteRelationSQL},
+		{&s.selectMaxRelationIDStmt, selectMaxRelationIDSQL},
+	}.Prepare(db)
+}
+
+func (s *relationsStatements) InsertRelation(
+	ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string,
+) (err error) {
+	_, err = sqlutil.TxStmt(txn, s.insertRelationStmt).ExecContext(
+		ctx, roomID, eventID, childEventID, childEventType, relType,
+	)
+	return
+}
+
+func (s *relationsStatements) DeleteRelation(
+	ctx context.Context, txn *sql.Tx, roomID, childEventID string,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteRelationStmt)
+	_, err := stmt.ExecContext(
+		ctx, roomID, childEventID,
+	)
+	return err
+}
+
+// SelectRelationsInRange returns a map rel_type -> []child_event_id
+func (s *relationsStatements) SelectRelationsInRange(
+	ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string,
+	r types.Range, limit int,
+) (map[string][]types.RelationEntry, types.StreamPosition, error) {
+	var lastPos types.StreamPosition
+	var stmt *sql.Stmt
+	if r.Backwards {
+		stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt)
+	} else {
+		stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt)
+	}
+	rows, err := stmt.QueryContext(ctx, roomID, eventID, relType, eventType, r.Low(), r.High(), limit)
+	if err != nil {
+		return nil, lastPos, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectRelationsInRange: rows.close() failed")
+	result := map[string][]types.RelationEntry{}
+	var (
+		id           types.StreamPosition
+		childEventID string
+		relationType string
+	)
+	for rows.Next() {
+		if err = rows.Scan(&id, &childEventID, &relationType); err != nil {
+			return nil, lastPos, err
+		}
+		if id > lastPos {
+			lastPos = id
+		}
+		result[relationType] = append(result[relationType], types.RelationEntry{
+			Position: id,
+			EventID:  childEventID,
+		})
+	}
+	if lastPos == 0 {
+		lastPos = r.To
+	}
+	return result, lastPos, rows.Err()
+}
+
+func (s *relationsStatements) SelectMaxRelationID(
+	ctx context.Context, txn *sql.Tx,
+) (id int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.selectMaxRelationIDStmt)
+	err = stmt.QueryRowContext(ctx).Scan(&id)
+	return
+}
diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go
index 979ff6647..850d24a07 100644
--- a/syncapi/storage/postgres/syncserver.go
+++ b/syncapi/storage/postgres/syncserver.go
@@ -98,6 +98,10 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	if err != nil {
 		return nil, err
 	}
+	relations, err := NewPostgresRelationsTable(d.db)
+	if err != nil {
+		return nil, err
+	}
 
 	// apply migrations which need multiple tables
 	m := sqlutil.NewMigrator(d.db)
@@ -129,6 +133,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 		NotificationData:    notificationData,
 		Ignores:             ignores,
 		Presence:            presence,
+		Relations:           relations,
 	}
 	return &d, nil
 }
diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go
index 937ced3a2..bf12203db 100644
--- a/syncapi/storage/shared/storage_consumer.go
+++ b/syncapi/storage/shared/storage_consumer.go
@@ -53,6 +53,7 @@ type Database struct {
 	NotificationData    tables.NotificationData
 	Ignores             tables.Ignores
 	Presence            tables.Presence
+	Relations           tables.Relations
 }
 
 func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) {
@@ -579,10 +580,40 @@ func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID s
 	return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos)
 }
 
-func (s *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) {
-	return s.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{
+func (d *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) {
+	return d.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{
 		gomatrixserverlib.MRoomName,
 		gomatrixserverlib.MRoomTopic,
 		"m.room.message",
 	})
 }
+
+func (d *Database) UpdateRelations(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
+	var content gomatrixserverlib.RelationContent
+	if err := json.Unmarshal(event.Content(), &content); err != nil {
+		return fmt.Errorf("json.Unmarshal: %w", err)
+	}
+	switch {
+	case content.Relations == nil:
+		return nil
+	case content.Relations.EventID == "":
+		return nil
+	case content.Relations.RelationType == "":
+		return nil
+	case event.Type() == gomatrixserverlib.MRoomRedaction:
+		return nil
+	default:
+		return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+			return d.Relations.InsertRelation(
+				ctx, txn, event.RoomID(), content.Relations.EventID,
+				event.EventID(), event.Type(), content.Relations.RelationType,
+			)
+		})
+	}
+}
+
+func (d *Database) RedactRelations(ctx context.Context, roomID, redactedEventID string) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.Relations.DeleteRelation(ctx, txn, roomID, redactedEventID)
+	})
+}
diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go
index d5b5b3121..cb61c1c26 100644
--- a/syncapi/storage/shared/storage_sync.go
+++ b/syncapi/storage/shared/storage_sync.go
@@ -589,3 +589,84 @@ func (d *DatabaseTransaction) PresenceAfter(ctx context.Context, after types.Str
 func (d *DatabaseTransaction) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) {
 	return d.Presence.GetMaxPresenceID(ctx, d.txn)
 }
+
+func (d *DatabaseTransaction) MaxStreamPositionForRelations(ctx context.Context) (types.StreamPosition, error) {
+	id, err := d.Relations.SelectMaxRelationID(ctx, d.txn)
+	return types.StreamPosition(id), err
+}
+
+func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (
+	events []types.StreamEvent, prevBatch, nextBatch string, err error,
+) {
+	r := types.Range{
+		From:      from,
+		To:        to,
+		Backwards: backwards,
+	}
+
+	if r.Backwards && r.From == 0 {
+		// If we're working backwards (dir=b) and there's no ?from= specified then
+		// we will automatically want to work backwards from the current position,
+		// so find out what that is.
+		if r.From, err = d.MaxStreamPositionForRelations(ctx); err != nil {
+			return nil, "", "", fmt.Errorf("d.MaxStreamPositionForRelations: %w", err)
+		}
+		// The result normally isn't inclusive of the event *at* the ?from=
+		// position, so add 1 here so that we include the most recent relation.
+		r.From++
+	} else if !r.Backwards && r.To == 0 {
+		// If we're working forwards (dir=f) and there's no ?to= specified then
+		// we will automatically want to work forwards towards the current position,
+		// so find out what that is.
+		if r.To, err = d.MaxStreamPositionForRelations(ctx); err != nil {
+			return nil, "", "", fmt.Errorf("d.MaxStreamPositionForRelations: %w", err)
+		}
+	}
+
+	// First look up any relations from the database. We add one to the limit here
+	// so that we can tell if we're overflowing, as we will only set the "next_batch"
+	// in the response if we are.
+	relations, _, err := d.Relations.SelectRelationsInRange(ctx, d.txn, roomID, eventID, relType, eventType, r, limit+1)
+	if err != nil {
+		return nil, "", "", fmt.Errorf("d.Relations.SelectRelationsInRange: %w", err)
+	}
+
+	// If we specified a relation type then just get those results, otherwise collate
+	// them from all of the returned relation types.
+	entries := []types.RelationEntry{}
+	if relType != "" {
+		entries = relations[relType]
+	} else {
+		for _, e := range relations {
+			entries = append(entries, e...)
+		}
+	}
+
+	// If there were no entries returned, there were no relations, so stop at this point.
+	if len(entries) == 0 {
+		return nil, "", "", nil
+	}
+
+	// Otherwise, let's try and work out what sensible prev_batch and next_batch values
+	// could be. We've requested an extra event by adding one to the limit already so
+	// that we can determine whether or not to provide a "next_batch", so trim off that
+	// event off the end if needs be.
+	if len(entries) > limit {
+		entries = entries[:len(entries)-1]
+		nextBatch = fmt.Sprintf("%d", entries[len(entries)-1].Position)
+	}
+	// TODO: set prevBatch? doesn't seem to affect the tests...
+
+	// Extract all of the event IDs from the relation entries so that we can pull the
+	// events out of the database. Then go and fetch the events.
+	eventIDs := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		eventIDs = append(eventIDs, entry.EventID)
+	}
+	events, err = d.OutputEvents.SelectEvents(ctx, d.txn, eventIDs, nil, true)
+	if err != nil {
+		return nil, "", "", fmt.Errorf("d.OutputEvents.SelectEvents: %w", err)
+	}
+
+	return events, prevBatch, nextBatch, nil
+}
diff --git a/syncapi/storage/sqlite3/relations_table.go b/syncapi/storage/sqlite3/relations_table.go
new file mode 100644
index 000000000..7cbb5408f
--- /dev/null
+++ b/syncapi/storage/sqlite3/relations_table.go
@@ -0,0 +1,163 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
+	"github.com/matrix-org/dendrite/syncapi/types"
+)
+
+const relationsSchema = `
+CREATE TABLE IF NOT EXISTS syncapi_relations (
+	id BIGINT PRIMARY KEY,
+	room_id TEXT NOT NULL,
+	event_id TEXT NOT NULL,
+	child_event_id TEXT NOT NULL,
+	child_event_type TEXT NOT NULL,
+	rel_type TEXT NOT NULL,
+	UNIQUE (room_id, event_id, child_event_id, rel_type)
+);
+`
+
+const insertRelationSQL = "" +
+	"INSERT INTO syncapi_relations (" +
+	"  id, room_id, event_id, child_event_id, child_event_type, rel_type" +
+	") VALUES ($1, $2, $3, $4, $5, $6) " +
+	" ON CONFLICT DO NOTHING"
+
+const deleteRelationSQL = "" +
+	"DELETE FROM syncapi_relations WHERE room_id = $1 AND child_event_id = $2"
+
+const selectRelationsInRangeAscSQL = "" +
+	"SELECT id, child_event_id, rel_type FROM syncapi_relations" +
+	" WHERE room_id = $1 AND event_id = $2" +
+	" AND ( $3 = '' OR rel_type = $3 )" +
+	" AND ( $4 = '' OR child_event_type = $4 )" +
+	" AND id > $5 AND id <= $6" +
+	" ORDER BY id ASC LIMIT $7"
+
+const selectRelationsInRangeDescSQL = "" +
+	"SELECT id, child_event_id, rel_type FROM syncapi_relations" +
+	" WHERE room_id = $1 AND event_id = $2" +
+	" AND ( $3 = '' OR rel_type = $3 )" +
+	" AND ( $4 = '' OR child_event_type = $4 )" +
+	" AND id >= $5 AND id < $6" +
+	" ORDER BY id DESC LIMIT $7"
+
+const selectMaxRelationIDSQL = "" +
+	"SELECT COALESCE(MAX(id), 0) FROM syncapi_relations"
+
+type relationsStatements struct {
+	streamIDStatements             *StreamIDStatements
+	insertRelationStmt             *sql.Stmt
+	selectRelationsInRangeAscStmt  *sql.Stmt
+	selectRelationsInRangeDescStmt *sql.Stmt
+	deleteRelationStmt             *sql.Stmt
+	selectMaxRelationIDStmt        *sql.Stmt
+}
+
+func NewSqliteRelationsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Relations, error) {
+	s := &relationsStatements{
+		streamIDStatements: streamID,
+	}
+	_, err := db.Exec(relationsSchema)
+	if err != nil {
+		return nil, err
+	}
+	return s, sqlutil.StatementList{
+		{&s.insertRelationStmt, insertRelationSQL},
+		{&s.selectRelationsInRangeAscStmt, selectRelationsInRangeAscSQL},
+		{&s.selectRelationsInRangeDescStmt, selectRelationsInRangeDescSQL},
+		{&s.deleteRelationStmt, deleteRelationSQL},
+		{&s.selectMaxRelationIDStmt, selectMaxRelationIDSQL},
+	}.Prepare(db)
+}
+
+func (s *relationsStatements) InsertRelation(
+	ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string,
+) (err error) {
+	var streamPos types.StreamPosition
+	if streamPos, err = s.streamIDStatements.nextRelationID(ctx, txn); err != nil {
+		return
+	}
+	_, err = sqlutil.TxStmt(txn, s.insertRelationStmt).ExecContext(
+		ctx, streamPos, roomID, eventID, childEventID, childEventType, relType,
+	)
+	return
+}
+
+func (s *relationsStatements) DeleteRelation(
+	ctx context.Context, txn *sql.Tx, roomID, childEventID string,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteRelationStmt)
+	_, err := stmt.ExecContext(
+		ctx, roomID, childEventID,
+	)
+	return err
+}
+
+// SelectRelationsInRange returns a map rel_type -> []child_event_id
+func (s *relationsStatements) SelectRelationsInRange(
+	ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string,
+	r types.Range, limit int,
+) (map[string][]types.RelationEntry, types.StreamPosition, error) {
+	var lastPos types.StreamPosition
+	var stmt *sql.Stmt
+	if r.Backwards {
+		stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt)
+	} else {
+		stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt)
+	}
+	rows, err := stmt.QueryContext(ctx, roomID, eventID, relType, eventType, r.Low(), r.High(), limit)
+	if err != nil {
+		return nil, lastPos, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectRelationsInRange: rows.close() failed")
+	result := map[string][]types.RelationEntry{}
+	var (
+		id           types.StreamPosition
+		childEventID string
+		relationType string
+	)
+	for rows.Next() {
+		if err = rows.Scan(&id, &childEventID, &relationType); err != nil {
+			return nil, lastPos, err
+		}
+		if id > lastPos {
+			lastPos = id
+		}
+		result[relationType] = append(result[relationType], types.RelationEntry{
+			Position: id,
+			EventID:  childEventID,
+		})
+	}
+	if lastPos == 0 {
+		lastPos = r.To
+	}
+	return result, lastPos, rows.Err()
+}
+
+func (s *relationsStatements) SelectMaxRelationID(
+	ctx context.Context, txn *sql.Tx,
+) (id int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.selectMaxRelationIDStmt)
+	err = stmt.QueryRowContext(ctx).Scan(&id)
+	return
+}
diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go
index 1160a437e..a4bba508e 100644
--- a/syncapi/storage/sqlite3/stream_id_table.go
+++ b/syncapi/storage/sqlite3/stream_id_table.go
@@ -28,6 +28,8 @@ INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("presence", 0)
   ON CONFLICT DO NOTHING;
 INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("notification", 0)
   ON CONFLICT DO NOTHING;
+INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("relation", 0)
+  ON CONFLICT DO NOTHING;
 `
 
 const increaseStreamIDStmt = "" +
@@ -86,3 +88,9 @@ func (s *StreamIDStatements) nextNotificationID(ctx context.Context, txn *sql.Tx
 	err = increaseStmt.QueryRowContext(ctx, "notification").Scan(&pos)
 	return
 }
+
+func (s *StreamIDStatements) nextRelationID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) {
+	increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt)
+	err = increaseStmt.QueryRowContext(ctx, "relation").Scan(&pos)
+	return
+}
diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go
index 0879030a6..510546909 100644
--- a/syncapi/storage/sqlite3/syncserver.go
+++ b/syncapi/storage/sqlite3/syncserver.go
@@ -123,6 +123,10 @@ func (d *SyncServerDatasource) prepare(ctx context.Context) (err error) {
 	if err != nil {
 		return err
 	}
+	relations, err := NewSqliteRelationsTable(d.db, &d.streamID)
+	if err != nil {
+		return err
+	}
 
 	// apply migrations which need multiple tables
 	m := sqlutil.NewMigrator(d.db)
@@ -153,6 +157,7 @@ func (d *SyncServerDatasource) prepare(ctx context.Context) (err error) {
 		NotificationData:    notificationData,
 		Ignores:             ignores,
 		Presence:            presence,
+		Relations:           relations,
 	}
 	return nil
 }
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 2fdc3cfbb..e48c050dd 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -206,3 +206,22 @@ type Presence interface {
 	GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error)
 	GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error)
 }
+
+type Relations interface {
+	// Inserts a relation which refers from the child event ID to the event ID in the given room.
+	// If the relation already exists then this function will do nothing and return no error.
+	InsertRelation(ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string) (err error)
+	// Deletes a relation which already exists as the result of an event redaction. If the relation
+	// does not exist then this function will do nothing and return no error.
+	DeleteRelation(ctx context.Context, txn *sql.Tx, roomID, childEventID string) error
+	// SelectRelationsInRange will return relations grouped by relation type within the given range.
+	// The map is relType -> []entry. If a relType parameter is specified then the results will only
+	// contain relations of that type, otherwise if "" is specified then all relations in the range
+	// will be returned, inclusive of the "to" position but excluding the "from" position. The stream
+	// position returned is the maximum position of the returned results.
+	SelectRelationsInRange(ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, r types.Range, limit int) (map[string][]types.RelationEntry, types.StreamPosition, error)
+	// SelectMaxRelationID returns the maximum ID of all relations, used to determine what the boundaries
+	// should be if there are no boundaries supplied (i.e. we want to work backwards but don't have a
+	// "from" or want to work forwards and don't have a "to").
+	SelectMaxRelationID(ctx context.Context, txn *sql.Tx) (id int64, err error)
+}
diff --git a/syncapi/storage/tables/relations_test.go b/syncapi/storage/tables/relations_test.go
new file mode 100644
index 000000000..46270e36d
--- /dev/null
+++ b/syncapi/storage/tables/relations_test.go
@@ -0,0 +1,186 @@
+package tables_test
+
+import (
+	"context"
+	"database/sql"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/storage/postgres"
+	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/dendrite/test"
+)
+
+func newRelationsTable(t *testing.T, dbType test.DBType) (tables.Relations, *sql.DB, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	if err != nil {
+		t.Fatalf("failed to open db: %s", err)
+	}
+
+	var tab tables.Relations
+	switch dbType {
+	case test.DBTypePostgres:
+		tab, err = postgres.NewPostgresRelationsTable(db)
+	case test.DBTypeSQLite:
+		var stream sqlite3.StreamIDStatements
+		if err = stream.Prepare(db); err != nil {
+			t.Fatalf("failed to prepare stream stmts: %s", err)
+		}
+		tab, err = sqlite3.NewSqliteRelationsTable(db, &stream)
+	}
+	if err != nil {
+		t.Fatalf("failed to make new table: %s", err)
+	}
+	return tab, db, close
+}
+
+func compareRelationsToExpected(t *testing.T, tab tables.Relations, r types.Range, expected []types.RelationEntry) {
+	ctx := context.Background()
+	relations, _, err := tab.SelectRelationsInRange(ctx, nil, roomID, "a", "", "", r, 50)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(relations[relType]) != len(expected) {
+		t.Fatalf("incorrect number of values returned for range %v (got %d, want %d)", r, len(relations[relType]), len(expected))
+	}
+	for i := 0; i < len(relations[relType]); i++ {
+		got := relations[relType][i]
+		want := expected[i]
+		if got != want {
+			t.Fatalf("range %v position %d should have been %q but got %q", r, i, got, want)
+		}
+	}
+}
+
+const roomID = "!roomid:server"
+const childType = "m.room.something"
+const relType = "m.reaction"
+
+func TestRelationsTable(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, _, close := newRelationsTable(t, dbType)
+		defer close()
+
+		// Insert some relations
+		for _, child := range []string{"b", "c", "d"} {
+			if err := tab.InsertRelation(ctx, nil, roomID, "a", child, childType, relType); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// Check the max position, we've inserted three things so it
+		// should be 3
+		if max, err := tab.SelectMaxRelationID(ctx, nil); err != nil {
+			t.Fatal(err)
+		} else if max != 3 {
+			t.Fatalf("max position should have been 3 but got %d", max)
+		}
+
+		// Query some ranges for "a"
+		for r, expected := range map[types.Range][]types.RelationEntry{
+			{From: 0, To: 10, Backwards: false}: {
+				{Position: 1, EventID: "b"},
+				{Position: 2, EventID: "c"},
+				{Position: 3, EventID: "d"},
+			},
+			{From: 1, To: 2, Backwards: false}: {
+				{Position: 2, EventID: "c"},
+			},
+			{From: 1, To: 3, Backwards: false}: {
+				{Position: 2, EventID: "c"},
+				{Position: 3, EventID: "d"},
+			},
+			{From: 10, To: 0, Backwards: true}: {
+				{Position: 3, EventID: "d"},
+				{Position: 2, EventID: "c"},
+				{Position: 1, EventID: "b"},
+			},
+			{From: 3, To: 1, Backwards: true}: {
+				{Position: 2, EventID: "c"},
+				{Position: 1, EventID: "b"},
+			},
+		} {
+			compareRelationsToExpected(t, tab, r, expected)
+		}
+
+		// Now delete one of the relations
+		if err := tab.DeleteRelation(ctx, nil, roomID, "c"); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query some more ranges for "a"
+		for r, expected := range map[types.Range][]types.RelationEntry{
+			{From: 0, To: 10, Backwards: false}: {
+				{Position: 1, EventID: "b"},
+				{Position: 3, EventID: "d"},
+			},
+			{From: 1, To: 2, Backwards: false}: {},
+			{From: 1, To: 3, Backwards: false}: {
+				{Position: 3, EventID: "d"},
+			},
+			{From: 10, To: 0, Backwards: true}: {
+				{Position: 3, EventID: "d"},
+				{Position: 1, EventID: "b"},
+			},
+			{From: 3, To: 1, Backwards: true}: {
+				{Position: 1, EventID: "b"},
+			},
+		} {
+			compareRelationsToExpected(t, tab, r, expected)
+		}
+
+		// Insert some new relations
+		for _, child := range []string{"e", "f", "g", "h"} {
+			if err := tab.InsertRelation(ctx, nil, roomID, "a", child, childType, relType); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// Check the max position, we've inserted four things so it
+		// should now be 7
+		if max, err := tab.SelectMaxRelationID(ctx, nil); err != nil {
+			t.Fatal(err)
+		} else if max != 7 {
+			t.Fatalf("max position should have been 3 but got %d", max)
+		}
+
+		// Query last set of ranges for "a"
+		for r, expected := range map[types.Range][]types.RelationEntry{
+			{From: 0, To: 10, Backwards: false}: {
+				{Position: 1, EventID: "b"},
+				{Position: 3, EventID: "d"},
+				{Position: 4, EventID: "e"},
+				{Position: 5, EventID: "f"},
+				{Position: 6, EventID: "g"},
+				{Position: 7, EventID: "h"},
+			},
+			{From: 1, To: 2, Backwards: false}: {},
+			{From: 1, To: 3, Backwards: false}: {
+				{Position: 3, EventID: "d"},
+			},
+			{From: 10, To: 0, Backwards: true}: {
+				{Position: 7, EventID: "h"},
+				{Position: 6, EventID: "g"},
+				{Position: 5, EventID: "f"},
+				{Position: 4, EventID: "e"},
+				{Position: 3, EventID: "d"},
+				{Position: 1, EventID: "b"},
+			},
+			{From: 6, To: 3, Backwards: true}: {
+				{Position: 5, EventID: "f"},
+				{Position: 4, EventID: "e"},
+				{Position: 3, EventID: "d"},
+			},
+		} {
+			compareRelationsToExpected(t, tab, r, expected)
+		}
+	})
+}
diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index b6d340f93..60a74a285 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -47,6 +47,14 @@ type StateDelta struct {
 // StreamPosition represents the offset in the sync stream a client is at.
 type StreamPosition int64
 
+func NewStreamPositionFromString(s string) (StreamPosition, error) {
+	n, err := strconv.Atoi(s)
+	if err != nil {
+		return 0, err
+	}
+	return StreamPosition(n), nil
+}
+
 // StreamEvent is the same as gomatrixserverlib.Event but also has the PDU stream position for this event.
 type StreamEvent struct {
 	*gomatrixserverlib.HeaderedEvent
@@ -599,3 +607,8 @@ type OutputSendToDeviceEvent struct {
 type IgnoredUsers struct {
 	List map[string]interface{} `json:"ignored_users"`
 }
+
+type RelationEntry struct {
+	Position StreamPosition
+	EventID  string
+}