Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/eduserver

.github/workflows/codeql-analysis.yml

@@ -1,34 +0,0 @@
-name: "CodeQL"
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ["go"]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-
-      - run: git checkout HEAD^2
-        if: ${{ github.event_name == 'pull_request' }}
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v1
-        with:
-          languages: ${{ matrix.language }}
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1

.github/workflows/dendrite.yml

@@ -0,0 +1,346 @@
+name: Dendrite
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  release:
+    types: [published]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  wasm:
+    name: WASM build test
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-wasm-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-wasm
+
+      - name: Install Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: 14
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Reconfigure Git to use HTTPS auth for repo packages
+        run: >
+          git config --global url."https://github.com/".insteadOf
+          ssh://git@github.com/
+
+      - name: Install test dependencies
+        working-directory: ./test/wasm
+        run: npm ci
+
+      - name: Test
+        run: ./test-dendritejs.sh
+
+  # Run golangci-lint
+  lint:
+    timeout-minutes: 5
+    name: Linting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+
+  # run go test with different go versions
+  test:
+    timeout-minutes: 5
+    name: Unit tests (Go ${{ matrix.go }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.16", "1.17", "1.18"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go${{ matrix.go }}-test-
+      - run: go test ./...
+
+  # build Dendrite for linux with different architectures and go versions
+  build:
+    name: Build for Linux
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.16", "1.17", "1.18"]
+        goos: ["linux"]
+        goarch: ["amd64", "386"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Install dependencies x86
+        if: ${{ matrix.goarch == '386' }}
+        run: sudo apt update && sudo apt-get install -y gcc-multilib
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}-
+      - env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+          CGO_ENABLED: 1
+        run: go build -trimpath -v -o "bin/" ./cmd/...
+
+  # build for Windows 64-bit
+  build_windows:
+    name: Build for Windows
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go: ["1.16", "1.17", "1.18"]
+        goos: ["windows"]
+        goarch: ["amd64"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Go ${{ matrix.go }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}
+      - env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+          CGO_ENABLED: 1
+          CC: "/usr/bin/x86_64-w64-mingw32-gcc"
+        run: go build -trimpath -v -o "bin/" ./cmd/...
+
+  # Dummy step to gate other tests on without repeating the whole list
+  initial-tests-done:
+    name: Initial tests passed
+    needs: [lint, test, build, build_windows]
+    runs-on: ubuntu-latest
+    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
+    steps:
+      - name: Check initial tests passed
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}
+
+  # run database upgrade tests
+  upgrade_test:
+    name: Upgrade tests
+    timeout-minutes: 20
+    needs: initial-tests-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.16"
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade
+      - name: Build upgrade-tests
+        run: go build ./cmd/dendrite-upgrade-tests
+      - name: Test upgrade
+        run: ./dendrite-upgrade-tests --head .
+
+  # run Sytest in different variations
+  sytest:
+    timeout-minutes: 20
+    needs: initial-tests-done
+    name: "Sytest (${{ matrix.label }})"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: SQLite
+
+          - label: SQLite, full HTTP APIs
+            api: full-http
+
+          - label: PostgreSQL
+            postgres: postgres
+
+          - label: PostgreSQL, full HTTP APIs
+            postgres: postgres
+            api: full-http
+    container:
+      image: matrixdotorg/sytest-dendrite:latest
+      volumes:
+        - ${{ github.workspace }}:/src
+      env:
+        POSTGRES: ${{ matrix.postgres && 1}}
+        API: ${{ matrix.api && 1 }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run Sytest
+        run: /bootstrap.sh dendrite
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+
+      - name: Upload Sytest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+  # run Complement
+  complement:
+    name: "Complement (${{ matrix.label }})"
+    timeout-minutes: 20
+    needs: initial-tests-done
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: SQLite
+
+          - label: SQLite, full HTTP APIs
+            api: full-http
+
+          - label: PostgreSQL
+            postgres: Postgres
+
+          - label: PostgreSQL, full HTTP APIs
+            postgres: Postgres
+            api: full-http
+    steps:
+      # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
+      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+      - name: "Set Go Version"
+        run: |
+          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+          echo "~/go/bin" >> $GITHUB_PATH
+
+      - name: "Install Complement Dependencies"
+        # We don't need to install Go because it is included on the Ubuntu 20.04 image:
+        # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
+        run: |
+          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
+          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
+
+      - name: Run actions/checkout@v2 for dendrite
+        uses: actions/checkout@v2
+        with:
+          path: dendrite
+
+      # Attempt to check out the same branch of Complement as the PR. If it
+      # doesn't exist, fallback to main.
+      - name: Checkout complement
+        shell: bash
+        run: |
+          mkdir -p complement
+          # Attempt to use the version of complement which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default complement branch ("master").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
+
+            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+          done
+
+      # Build initial Dendrite image
+      - run: docker build -t complement-dendrite -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
+        working-directory: dendrite
+
+      # Run Complement
+      - run: |
+          set -o pipefail &&
+          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+        shell: bash
+        name: Run Complement Tests
+        env:
+          COMPLEMENT_BASE_IMAGE: complement-dendrite:latest
+          API: ${{ matrix.api && 1 }}
+        working-directory: complement
+
+  integration-tests-done:
+    name: Integration tests passed
+    needs: [initial-tests-done, upgrade_test, sytest, complement]
+    runs-on: ubuntu-latest
+    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
+    steps:
+      - name: Check integration tests passed
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}
+
+  update-docker-images:
+    name: Update Docker images
+    permissions:
+      packages: write
+      contents: read
+    if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
+    needs: [integration-tests-done]
+    uses: matrix-org/dendrite/.github/workflows/docker.yml@main
+    secrets:
+      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}

.github/workflows/docker-hub.yml

@@ -1,71 +0,0 @@
-# Based on https://github.com/docker/build-push-action
-
-name: "Docker Hub"
-
-on:
-  release:
-    types: [published]
-
-env:
-  DOCKER_NAMESPACE: matrixdotorg
-  DOCKER_HUB_USER: dendritegithub
-  PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
-
-jobs:
-  Monolith:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get release tag
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1 
-        with:
-          username: ${{ env.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-
-      - name: Build monolith image
-        id: docker_build_monolith
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./build/docker/Dockerfile.monolith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
-
-  Polylith:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get release tag
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1 
-        with:
-          username: ${{ env.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-
-      - name: Build polylith image
-        id: docker_build_polylith
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./build/docker/Dockerfile.polylith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}

.github/workflows/docker.yml

@@ -0,0 +1,139 @@
+# Based on https://github.com/docker/build-push-action
+
+name: "Docker"
+
+on:
+  release: # A GitHub release was published
+    types: [published]
+  workflow_dispatch: # A build was manually requested
+  workflow_call: # Another pipeline called us
+    secrets:
+      DOCKER_TOKEN:
+        required: true
+
+env:
+  DOCKER_NAMESPACE: matrixdotorg
+  DOCKER_HUB_USER: dendritegithub
+  GHCR_NAMESPACE: matrix-org
+  PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
+
+jobs:
+  monolith:
+    name: Monolith image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get release tag
+        if: github.event_name == 'release' # Only for GitHub releases
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to GitHub Containers
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build main monolith image
+        if: github.ref_name == 'main'
+        id: docker_build_monolith
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: .
+          file: ./build/docker/Dockerfile.monolith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
+
+      - name: Build release monolith image
+        if: github.event_name == 'release' # Only for GitHub releases
+        id: docker_build_monolith_release
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: .
+          file: ./build/docker/Dockerfile.monolith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
+
+  polylith:
+    name: Polylith image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get release tag
+        if: github.event_name == 'release' # Only for GitHub releases
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to GitHub Containers
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build main polylith image
+        if: github.ref_name == 'main'
+        id: docker_build_polylith
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: .
+          file: ./build/docker/Dockerfile.polylith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
+
+      - name: Build release polylith image
+        if: github.event_name == 'release' # Only for GitHub releases
+        id: docker_build_polylith_release
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: .
+          file: ./build/docker/Dockerfile.polylith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}

.github/workflows/tests.yml

@@ -1,71 +0,0 @@
-name: Tests
-
-on:
-  push:
-    branches: ["main"]
-  pull_request:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  complement:
-    runs-on: ubuntu-latest
-    steps:
-      # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        # We don't need to install Go because it is included on the Ubuntu 20.04 image:
-        # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
-      - name: Run actions/checkout@v2 for dendrite
-        uses: actions/checkout@v2
-        with:
-          path: dendrite
-
-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to main.
-      - name: Checkout complement
-        shell: bash
-        run: |
-          mkdir -p complement
-          # Attempt to use the version of complement which best matches the current
-          # build. Depending on whether this is a PR or release, etc. we need to
-          # use different fallbacks.
-          #
-          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
-          #    for pull requests, otherwise GITHUB_REF).
-          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
-          #    (GITHUB_BASE_REF for pull requests).
-          # 3. Use the default complement branch ("master").
-          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
-            # Skip empty branch names and merge commits.
-            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
-              continue
-            fi
-
-            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
-          done
-
-      # Build initial Dendrite image
-      - run: docker build -t complement-dendrite -f build/scripts/Complement.Dockerfile .
-        working-directory: dendrite
-
-      # Run Complement
-      - run: |
-          set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
-        shell: bash
-        name: Run Complement Tests
-        env:
-          COMPLEMENT_BASE_IMAGE: complement-dendrite:latest
-        working-directory: complement

.github/workflows/wasm.yml

@@ -1,49 +0,0 @@
-name: WebAssembly
-
-on:
-    push:
-    pull_request:
-
-jobs:
-    test:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v2
-
-            - name: Install Go
-              uses: actions/setup-go@v2
-              with:
-                  go-version: 1.16.5
-
-            - uses: actions/cache@v2
-              with:
-                  path: |
-                      ~/.cache/go-build
-                      ~/go/pkg/mod
-                  key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-                  restore-keys: |
-                      ${{ runner.os }}-go-
-
-            - name: Install Node
-              uses: actions/setup-node@v2
-              with:
-                  node-version: 14
-
-            - uses: actions/cache@v2
-              with:
-                  path: ~/.npm
-                  key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-                  restore-keys: |
-                      ${{ runner.os }}-node-
-
-            - name: Reconfigure Git to use HTTPS auth for repo packages
-              run: >
-                  git config --global url."https://github.com/".insteadOf
-                  ssh://git@github.com/
-
-            - name: Install test dependencies
-              working-directory: ./test/wasm
-              run: npm ci
-
-            - name: Test
-              run: ./test-dendritejs.sh

CHANGES.md

@@ -1,5 +1,43 @@
 # Changelog
 
+## Dendrite 0.7.0 (2022-03-25)
+
+### Features
+
+* The roomserver input API will now queue all events into NATS, which provides better crash resilience
+* The roomserver input API now configures per-room consumers, which should use less memory
+* Canonical aliases can now be added and removed
+* MSC2946 Spaces Summary now works correctly, both locally and over federation
+* Healthcheck endpoints are now available at:
+  * `/_dendrite/monitor/up`, which will return 200 when Dendrite is ready to accept requests
+  * `/_dendrite/monitor/health`, which will return 200 if healthy and 503 if degraded for some reason
+* The `X-Matrix` federation authorisation header now includes a `destination` field, as per MSC3383
+* The `/sync` endpoint now uses less memory by only ranging state for rooms that the user has participated in
+* The `/messages` endpoint now accepts stream positions in both the `from` and `to` parameters
+* Dendrite will now log a warning at startup if the file descriptor limit is set too low
+* The federation client will now attempt to use HTTP/2 if available
+* The federation client will now attempt to resume TLS sessions if possible, to reduce handshake overheads
+* The built-in NATS Server has been updated to version 2.7.4
+* NATS streams that don't match the desired configuration will now be recreated automatically
+* When performing a graceful shutdown, Dendrite will now wait for NATS Server to shutdown completely, which should avoid some corruption of data on-disk
+* The `create-account` tool has seen a number of improvements, will now ask for passwords automatically
+
+### Fixes
+
+* The `/sync` endpoint will no longer lose state events when truncating the timeline for history visibility
+* The `/context` endpoint now works correctly with `lazy_load_members`
+* The `/directory/list/room/{roomID}` endpoint now correctly reports whether a room is published in the server room directory or not
+* Some bugs around appservice username validation have been fixed
+* Roomserver output messages are no longer unnecessarily inflated by state events, which should reduce the number of NATS message size errors
+* Stream IDs for device list updates are now always 64-bit, which should fix some problems when running Dendrite on a 32-bit system
+* Purging room state in the sync API has been fixed after a faulty database query was corrected
+* The federation client will now release host records for remote destinations after 5 minutes instead of holding them in memory forever
+* Remote media requests will now correctly return an error if the file cannot be found or downloaded
+* A panic in the media API that could happen when the remote file doesn't exist has been fixed
+* Various bugs around membership state and invites have been fixed
+* The memberships table will now be correctly updated when rejecting a federated invite
+* The client API and appservice API will now access the user database using the user API rather than accessing the database directly
+
 ## Dendrite 0.6.5 (2022-03-04)
 
 ### Features

README.md

@@ -1,4 +1,5 @@
-# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org)
+# Dendrite
+[![Build status](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml/badge.svg?event=push)](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org)
 
 Dendrite is a second-generation Matrix homeserver written in Go.
 It intends to provide an **efficient**, **reliable** and **scalable** alternative to [Synapse](https://github.com/matrix-org/synapse):

appservice/api/query.go

@@ -19,11 +19,10 @@ package api
 
 import (
 	"context"
-	"database/sql"
 	"errors"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -85,7 +84,7 @@ func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
 	asAPI AppServiceQueryAPI,
-	accountDB userdb.Database,
+	profileAPI userapi.UserProfileAPI,
 ) (*authtypes.Profile, error) {
 	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
@@ -93,10 +92,17 @@ func RetrieveUserProfile(
 	}
 
 	// Try to query the user from the local database
-	profile, err := accountDB.GetProfileByLocalpart(ctx, localpart)
+	res := &userapi.QueryProfileResponse{}
-	if err != nil && err != sql.ErrNoRows {
+	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
+	if err != nil {
 		return nil, err
-	} else if profile != nil {
+	}
+	profile := &authtypes.Profile{
+		Localpart:   localpart,
+		DisplayName: res.DisplayName,
+		AvatarURL:   res.AvatarURL,
+	}
+	if res.UserExists {
 		return profile, nil
 	}
 
@@ -113,11 +119,15 @@ func RetrieveUserProfile(
 	}
 
 	// Try to query the user from the local database again
-	profile, err = accountDB.GetProfileByLocalpart(ctx, localpart)
+	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
 	if err != nil {
 		return nil, err
 	}
 
 	// profile should not be nil at this point
-	return profile, nil
+	return &authtypes.Profile{
+		Localpart:   localpart,
+		DisplayName: res.DisplayName,
+		AvatarURL:   res.AvatarURL,
+	}, nil
 }

build/docker/Dockerfile.monolith

@@ -13,6 +13,10 @@ RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
+LABEL org.opencontainers.image.title="Dendrite (Monolith)"
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
 
 COPY --from=base /build/bin/* /usr/bin/
 

build/docker/Dockerfile.polylith

@@ -13,6 +13,10 @@ RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
+LABEL org.opencontainers.image.title="Dendrite (Polylith)"
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
 
 COPY --from=base /build/bin/* /usr/bin/
 

build/scripts/Complement.Dockerfile

@@ -21,6 +21,7 @@ WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
 
 ENV SERVER_NAME=localhost
+ENV API=0
 EXPOSE 8008 8448
 
 # At runtime, generate TLS cert based on the CA now mounted at /ca
@@ -28,4 +29,4 @@ EXPOSE 8008 8448
 CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
  ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
  cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
- ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
+ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

build/scripts/ComplementPostgres.Dockerfile

@@ -39,6 +39,7 @@ WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
 
 ENV SERVER_NAME=localhost
+ENV API=0
 EXPOSE 8008 8448
 
 
@@ -50,4 +51,4 @@ CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert s
  sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
  sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
  cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
- ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
+ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

clientapi/auth/login.go

@@ -33,7 +33,7 @@ import (
 // called after authorization has completed, with the result of the authorization.
 // If the final return value is non-nil, an error occurred and the cleanup function
 // is nil.
-func LoginFromJSONReader(ctx context.Context, r io.Reader, accountDB AccountDatabase, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserAccountAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
 	reqBytes, err := ioutil.ReadAll(r)
 	if err != nil {
 		err := &util.JSONResponse{
@@ -58,7 +58,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, accountDB AccountData
 	switch header.Type {
 	case authtypes.LoginTypePassword:
 		typ = &LoginTypePassword{
-			GetAccountByPassword: accountDB.GetAccountByPassword,
+			GetAccountByPassword: useraccountAPI.QueryAccountByPassword,
 			Config:               cfg,
 		}
 	case authtypes.LoginTypeToken:

clientapi/auth/login_test.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"context"
-	"database/sql"
 	"net/http"
 	"reflect"
 	"strings"
@@ -64,14 +63,13 @@ func TestLoginFromJSONReader(t *testing.T) {
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
-			var accountDB fakeAccountDB
 			var userAPI fakeUserInternalAPI
 			cfg := &config.ClientAPI{
 				Matrix: &config.Global{
 					ServerName: serverName,
 				},
 			}
-			login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &accountDB, &userAPI, cfg)
+			login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
 			if err != nil {
 				t.Fatalf("LoginFromJSONReader failed: %+v", err)
 			}
@@ -143,14 +141,13 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
-			var accountDB fakeAccountDB
 			var userAPI fakeUserInternalAPI
 			cfg := &config.ClientAPI{
 				Matrix: &config.Global{
 					ServerName: serverName,
 				},
 			}
-			_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &accountDB, &userAPI, cfg)
+			_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
 			if errRes == nil {
 				cleanup(ctx, nil)
 				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
@@ -161,24 +158,22 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 	}
 }
 
-type fakeAccountDB struct {
-	AccountDatabase
-}
-
-func (*fakeAccountDB) GetAccountByPassword(ctx context.Context, localpart, password string) (*uapi.Account, error) {
-	if password == "invalidpassword" {
-		return nil, sql.ErrNoRows
-	}
-
-	return &uapi.Account{}, nil
-}
-
 type fakeUserInternalAPI struct {
 	UserInternalAPIForLogin
-
+	uapi.UserAccountAPI
 	DeletedTokens []string
 }
 
+func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *uapi.QueryAccountByPasswordRequest, res *uapi.QueryAccountByPasswordResponse) error {
+	if req.PlaintextPassword == "invalidpassword" {
+		res.Account = nil
+		return nil
+	}
+	res.Exists = true
+	res.Account = &uapi.Account{}
+	return nil
+}
+
 func (ua *fakeUserInternalAPI) PerformLoginTokenDeletion(ctx context.Context, req *uapi.PerformLoginTokenDeletionRequest, res *uapi.PerformLoginTokenDeletionResponse) error {
 	ua.DeletedTokens = append(ua.DeletedTokens, req.Token)
 	return nil

clientapi/auth/password.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"context"
-	"database/sql"
 	"net/http"
 	"strings"
 
@@ -29,7 +28,7 @@ import (
 	"github.com/matrix-org/util"
 )
 
-type GetAccountByPassword func(ctx context.Context, localpart, password string) (*api.Account, error)
+type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error
 
 type PasswordRequest struct {
 	Login
@@ -62,7 +61,7 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
 
 func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
 	r := req.(*PasswordRequest)
-  username := strings.ToLower(r.Username())
+	username := strings.ToLower(r.Username())
 	if username == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
@@ -77,19 +76,33 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		}
 	}
 	// Squash username to all lowercase letters
-	_, err = t.GetAccountByPassword(ctx, strings.ToLower(localpart), r.Password)
+	res := &api.QueryAccountByPasswordResponse{}
+	err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: strings.ToLower(localpart), PlaintextPassword: r.Password}, res)
 	if err != nil {
-		if err == sql.ErrNoRows {
+		return nil, &util.JSONResponse{
-			_, err = t.GetAccountByPassword(ctx, localpart, r.Password)
+			Code: http.StatusInternalServerError,
-			if err == nil {
+			JSON: jsonerror.Unknown("unable to fetch account by password"),
-				return &r.Login, nil
+		}
+	}
+
+	if !res.Exists {
+		err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+			Localpart:         localpart,
+			PlaintextPassword: r.Password,
+		}, res)
+		if err != nil {
+			return nil, &util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: jsonerror.Unknown("unable to fetch account by password"),
 			}
 		}
 		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
 		// but that would leak the existence of the user.
-		return nil, &util.JSONResponse{
+		if !res.Exists {
-			Code: http.StatusForbidden,
+			return nil, &util.JSONResponse{
-			JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."),
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."),
+			}
 		}
 	}
 	return &r.Login, nil

clientapi/auth/user_interactive.go

@@ -110,9 +110,9 @@ type UserInteractive struct {
 	Sessions map[string][]string
 }
 
-func NewUserInteractive(accountDB AccountDatabase, cfg *config.ClientAPI) *UserInteractive {
+func NewUserInteractive(userAccountAPI api.UserAccountAPI, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
-		GetAccountByPassword: accountDB.GetAccountByPassword,
+		GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
 		Config:               cfg,
 	}
 	return &UserInteractive{

clientapi/auth/user_interactive_test.go

@@ -25,15 +25,25 @@ var (
 )
 
 type fakeAccountDatabase struct {
-	AccountDatabase
+	api.UserAccountAPI
 }
 
-func (*fakeAccountDatabase) GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error) {
+func (d *fakeAccountDatabase) PerformPasswordUpdate(ctx context.Context, req *api.PerformPasswordUpdateRequest, res *api.PerformPasswordUpdateResponse) error {
-	acc, ok := lookup[localpart+" "+plaintextPassword]
+	return nil
+}
+
+func (d *fakeAccountDatabase) PerformAccountDeactivation(ctx context.Context, req *api.PerformAccountDeactivationRequest, res *api.PerformAccountDeactivationResponse) error {
+	return nil
+}
+
+func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error {
+	acc, ok := lookup[req.Localpart+" "+req.PlaintextPassword]
 	if !ok {
-		return nil, fmt.Errorf("unknown user/password")
+		return fmt.Errorf("unknown user/password")
 	}
-	return acc, nil
+	res.Account = acc
+	res.Exists = true
+	return nil
 }
 
 func setup() *UserInteractive {

clientapi/clientapi.go

@@ -63,7 +63,7 @@ func AddPublicRoutes(
 
 	routing.Setup(
 		router, synapseAdminRouter, cfg, rsAPI, asAPI,
-		accountsDB, userAPI, federation,
+		userAPI, federation,
 		syncProducer, transactionsCache, fsAPI, keyAPI,
 		extRoomsProvider, mscCfg,
 	)

clientapi/routing/createroom.go

@@ -31,7 +31,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/config"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	log "github.com/sirupsen/logrus"
@@ -138,7 +137,7 @@ type fledglingEvent struct {
 func CreateRoom(
 	req *http.Request, device *api.Device,
 	cfg *config.ClientAPI,
-	accountDB userdb.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
+	profileAPI api.UserProfileAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	var r createRoomRequest
@@ -156,7 +155,7 @@ func CreateRoom(
 			JSON: jsonerror.InvalidArgumentValue(err.Error()),
 		}
 	}
-	return createRoom(req.Context(), r, device, cfg, accountDB, rsAPI, asAPI, evTime)
+	return createRoom(req.Context(), r, device, cfg, profileAPI, rsAPI, asAPI, evTime)
 }
 
 // createRoom implements /createRoom
@@ -165,7 +164,7 @@ func createRoom(
 	ctx context.Context,
 	r createRoomRequest, device *api.Device,
 	cfg *config.ClientAPI,
-	accountDB userdb.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
+	profileAPI api.UserProfileAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	evTime time.Time,
 ) util.JSONResponse {
@@ -201,7 +200,7 @@ func createRoom(
 		"roomVersion": roomVersion,
 	}).Info("Creating new room")
 
-	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed")
 		return jsonerror.InternalServerError()
@@ -520,7 +519,7 @@ func createRoom(
 		for _, invitee := range r.Invite {
 			// Build the invite event.
 			inviteEvent, err := buildMembershipEvent(
-				ctx, invitee, "", accountDB, device, gomatrixserverlib.Invite,
+				ctx, invitee, "", profileAPI, device, gomatrixserverlib.Invite,
 				roomID, true, cfg, evTime, rsAPI, asAPI,
 			)
 			if err != nil {

clientapi/routing/deactivate.go

@@ -15,7 +15,7 @@ import (
 func Deactivate(
 	req *http.Request,
 	userInteractiveAuth *auth.UserInteractive,
-	userAPI api.UserInternalAPI,
+	accountAPI api.UserAccountAPI,
 	deviceAPI *api.Device,
 ) util.JSONResponse {
 	ctx := req.Context()
@@ -40,7 +40,7 @@ func Deactivate(
 	}
 
 	var res api.PerformAccountDeactivationResponse
-	err = userAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{
+	err = accountAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{
 		Localpart: localpart,
 	}, &res)
 	if err != nil {

clientapi/routing/joinroom.go

@@ -18,12 +18,10 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -32,7 +30,7 @@ func JoinRoomByIDOrAlias(
 	req *http.Request,
 	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	accountDB userdb.Database,
+	profileAPI api.UserProfileAPI,
 	roomIDOrAlias string,
 ) util.JSONResponse {
 	// Prepare to ask the roomserver to perform the room join.
@@ -60,19 +58,23 @@ func JoinRoomByIDOrAlias(
 	_ = httputil.UnmarshalJSONRequest(req, &joinReq.Content)
 
 	// Work out our localpart for the client profile request.
-	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+
-	if err != nil {
+	// Request our profile content to populate the request content with.
-		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
+	res := &api.QueryProfileResponse{}
-	} else {
+	err := profileAPI.QueryProfile(req.Context(), &api.QueryProfileRequest{UserID: device.UserID}, res)
-		// Request our profile content to populate the request content with.
+	if err != nil || !res.UserExists {
-		var profile *authtypes.Profile
+		if !res.UserExists {
-		profile, err = accountDB.GetProfileByLocalpart(req.Context(), localpart)
+			util.GetLogger(req.Context()).Error("Unable to query user profile, no profile found.")
-		if err != nil {
+			return util.JSONResponse{
-			util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed")
+				Code: http.StatusInternalServerError,
-		} else {
+				JSON: jsonerror.Unknown("Unable to query user profile, no profile found."),
-			joinReq.Content["displayname"] = profile.DisplayName
+			}
-			joinReq.Content["avatar_url"] = profile.AvatarURL
 		}
+
+		util.GetLogger(req.Context()).WithError(err).Error("UserProfileAPI.QueryProfile failed")
+	} else {
+		joinReq.Content["displayname"] = res.DisplayName
+		joinReq.Content["avatar_url"] = res.AvatarURL
 	}
 
 	// Ask the roomserver to perform the join.

clientapi/routing/key_crosssigning.go

@@ -24,7 +24,6 @@ import (
 	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/util"
 )
 
@@ -36,7 +35,7 @@ type crossSigningRequest struct {
 func UploadCrossSigningDeviceKeys(
 	req *http.Request, userInteractiveAuth *auth.UserInteractive,
 	keyserverAPI api.KeyInternalAPI, device *userapi.Device,
-	accountDB userdb.Database, cfg *config.ClientAPI,
+	accountAPI userapi.UserAccountAPI, cfg *config.ClientAPI,
 ) util.JSONResponse {
 	uploadReq := &crossSigningRequest{}
 	uploadRes := &api.PerformUploadDeviceKeysResponse{}
@@ -64,7 +63,7 @@ func UploadCrossSigningDeviceKeys(
 		}
 	}
 	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: accountDB.GetAccountByPassword,
+		GetAccountByPassword: accountAPI.QueryAccountByPassword,
 		Config:               cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {

clientapi/routing/login.go

@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -54,7 +53,7 @@ func passwordLogin() flows {
 
 // Login implements GET and POST /login
 func Login(
-	req *http.Request, accountDB userdb.Database, userAPI userapi.UserInternalAPI,
+	req *http.Request, userAPI userapi.UserInternalAPI,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	if req.Method == http.MethodGet {
@@ -64,7 +63,7 @@ func Login(
 			JSON: passwordLogin(),
 		}
 	} else if req.Method == http.MethodPost {
-		login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, accountDB, userAPI, cfg)
+		login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg)
 		if authErr != nil {
 			return *authErr
 		}

clientapi/routing/membership.go

@@ -30,7 +30,6 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/util"
@@ -39,7 +38,7 @@ import (
 var errMissingUserID = errors.New("'user_id' must be supplied")
 
 func SendBan(
-	req *http.Request, accountDB userdb.Database, device *userapi.Device,
+	req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -78,16 +77,16 @@ func SendBan(
 		}
 	}
 
-	return sendMembership(req.Context(), accountDB, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+	return sendMembership(req.Context(), profileAPI, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
 }
 
-func sendMembership(ctx context.Context, accountDB userdb.Database, device *userapi.Device,
+func sendMembership(ctx context.Context, profileAPI userapi.UserProfileAPI, device *userapi.Device,
 	roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time,
 	roomVer gomatrixserverlib.RoomVersion,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
 
 	event, err := buildMembershipEvent(
-		ctx, targetUserID, reason, accountDB, device, membership,
+		ctx, targetUserID, reason, profileAPI, device, membership,
 		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
 	if err == errMissingUserID {
@@ -125,7 +124,7 @@ func sendMembership(ctx context.Context, accountDB userdb.Database, device *user
 }
 
 func SendKick(
-	req *http.Request, accountDB userdb.Database, device *userapi.Device,
+	req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -161,11 +160,11 @@ func SendKick(
 		}
 	}
 	// TODO: should we be using SendLeave instead?
-	return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+	return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
 }
 
 func SendUnban(
-	req *http.Request, accountDB userdb.Database, device *userapi.Device,
+	req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -196,11 +195,11 @@ func SendUnban(
 		}
 	}
 	// TODO: should we be using SendLeave instead?
-	return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+	return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
 }
 
 func SendInvite(
-	req *http.Request, accountDB userdb.Database, device *userapi.Device,
+	req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -210,7 +209,7 @@ func SendInvite(
 	}
 
 	inviteStored, jsonErrResp := checkAndProcessThreepid(
-		req, device, body, cfg, rsAPI, accountDB, roomID, evTime,
+		req, device, body, cfg, rsAPI, profileAPI, roomID, evTime,
 	)
 	if jsonErrResp != nil {
 		return *jsonErrResp
@@ -227,14 +226,14 @@ func SendInvite(
 	}
 
 	// We already received the return value, so no need to check for an error here.
-	response, _ := sendInvite(req.Context(), accountDB, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime)
+	response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime)
 	return response
 }
 
 // sendInvite sends an invitation to a user. Returns a JSONResponse and an error
 func sendInvite(
 	ctx context.Context,
-	accountDB userdb.Database,
+	profileAPI userapi.UserProfileAPI,
 	device *userapi.Device,
 	roomID, userID, reason string,
 	cfg *config.ClientAPI,
@@ -242,7 +241,7 @@ func sendInvite(
 	asAPI appserviceAPI.AppServiceQueryAPI, evTime time.Time,
 ) (util.JSONResponse, error) {
 	event, err := buildMembershipEvent(
-		ctx, userID, reason, accountDB, device, "invite",
+		ctx, userID, reason, profileAPI, device, "invite",
 		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
 	if err == errMissingUserID {
@@ -286,13 +285,13 @@ func sendInvite(
 
 func buildMembershipEvent(
 	ctx context.Context,
-	targetUserID, reason string, accountDB userdb.Database,
+	targetUserID, reason string, profileAPI userapi.UserProfileAPI,
 	device *userapi.Device,
 	membership, roomID string, isDirect bool,
 	cfg *config.ClientAPI, evTime time.Time,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
-	profile, err := loadProfile(ctx, targetUserID, cfg, accountDB, asAPI)
+	profile, err := loadProfile(ctx, targetUserID, cfg, profileAPI, asAPI)
 	if err != nil {
 		return nil, err
 	}
@@ -327,7 +326,7 @@ func loadProfile(
 	ctx context.Context,
 	userID string,
 	cfg *config.ClientAPI,
-	accountDB userdb.Database,
+	profileAPI userapi.UserProfileAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*authtypes.Profile, error) {
 	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
@@ -337,7 +336,7 @@ func loadProfile(
 
 	var profile *authtypes.Profile
 	if serverName == cfg.Matrix.ServerName {
-		profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB)
+		profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI)
 	} else {
 		profile = &authtypes.Profile{}
 	}
@@ -381,13 +380,13 @@ func checkAndProcessThreepid(
 	body *threepid.MembershipRequest,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	accountDB userdb.Database,
+	profileAPI userapi.UserProfileAPI,
 	roomID string,
 	evTime time.Time,
 ) (inviteStored bool, errRes *util.JSONResponse) {
 
 	inviteStored, err := threepid.CheckAndProcessInvite(
-		req.Context(), device, body, cfg, rsAPI, accountDB,
+		req.Context(), device, body, cfg, rsAPI, profileAPI,
 		roomID, evTime,
 	)
 	if err == threepid.ErrMissingParameter {

clientapi/routing/password.go

@@ -9,7 +9,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
@@ -30,7 +29,6 @@ type newPasswordAuth struct {
 func Password(
 	req *http.Request,
 	userAPI api.UserInternalAPI,
-	accountDB userdb.Database,
 	device *api.Device,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
@@ -74,7 +72,7 @@ func Password(
 
 	// Check if the existing password is correct.
 	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: accountDB.GetAccountByPassword,
+		GetAccountByPassword: userAPI.QueryAccountByPassword,
 		Config:               cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {

clientapi/routing/peekroom.go

@@ -19,7 +19,6 @@ import (
 
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -28,7 +27,6 @@ func PeekRoomByIDOrAlias(
 	req *http.Request,
 	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	accountDB userdb.Database,
 	roomIDOrAlias string,
 ) util.JSONResponse {
 	// if this is a remote roomIDOrAlias, we have to ask the roomserver (or federation sender?) to
@@ -82,7 +80,6 @@ func UnpeekRoomByID(
 	req *http.Request,
 	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	accountDB userdb.Database,
 	roomID string,
 ) util.JSONResponse {
 	unpeekReq := roomserverAPI.PerformUnpeekRequest{

clientapi/routing/profile.go

@@ -27,7 +27,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/gomatrix"
@@ -36,12 +35,12 @@ import (
 
 // GetProfile implements GET /profile/{userID}
 func GetProfile(
-	req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
-	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
+	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
 	if err != nil {
 		if err == eventutil.ErrProfileNoExists {
 			return util.JSONResponse{
@@ -65,11 +64,11 @@ func GetProfile(
 
 // GetAvatarURL implements GET /profile/{userID}/avatar_url
 func GetAvatarURL(
-	req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
-	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
+	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
 	if err != nil {
 		if err == eventutil.ErrProfileNoExists {
 			return util.JSONResponse{
@@ -92,7 +91,7 @@ func GetAvatarURL(
 
 // SetAvatarURL implements PUT /profile/{userID}/avatar_url
 func SetAvatarURL(
-	req *http.Request, accountDB userdb.Database,
+	req *http.Request, profileAPI userapi.UserProfileAPI,
 	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -127,22 +126,34 @@ func SetAvatarURL(
 		}
 	}
 
-	oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
+	res := &userapi.QueryProfileResponse{}
+	err = profileAPI.QueryProfile(req.Context(), &userapi.QueryProfileRequest{
+		UserID: userID,
+	}, res)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed")
+		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.QueryProfile failed")
+		return jsonerror.InternalServerError()
+	}
+	oldProfile := &authtypes.Profile{
+		Localpart:   localpart,
+		DisplayName: res.DisplayName,
+		AvatarURL:   res.AvatarURL,
+	}
+
+	setRes := &userapi.PerformSetAvatarURLResponse{}
+	if err = profileAPI.SetAvatarURL(req.Context(), &userapi.PerformSetAvatarURLRequest{
+		Localpart: localpart,
+		AvatarURL: r.AvatarURL,
+	}, setRes); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetAvatarURL failed")
 		return jsonerror.InternalServerError()
 	}
 
-	if err = accountDB.SetAvatarURL(req.Context(), localpart, r.AvatarURL); err != nil {
+	var roomsRes api.QueryRoomsForUserResponse
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.SetAvatarURL failed")
-		return jsonerror.InternalServerError()
-	}
-
-	var res api.QueryRoomsForUserResponse
 	err = rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{
 		UserID:         device.UserID,
 		WantMembership: "join",
-	}, &res)
+	}, &roomsRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
 		return jsonerror.InternalServerError()
@@ -155,7 +166,7 @@ func SetAvatarURL(
 	}
 
 	events, err := buildMembershipEvents(
-		req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI,
+		req.Context(), roomsRes.RoomIDs, newProfile, userID, cfg, evTime, rsAPI,
 	)
 	switch e := err.(type) {
 	case nil:
@@ -182,11 +193,11 @@ func SetAvatarURL(
 
 // GetDisplayName implements GET /profile/{userID}/displayname
 func GetDisplayName(
-	req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
-	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
+	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
 	if err != nil {
 		if err == eventutil.ErrProfileNoExists {
 			return util.JSONResponse{
@@ -209,7 +220,7 @@ func GetDisplayName(
 
 // SetDisplayName implements PUT /profile/{userID}/displayname
 func SetDisplayName(
-	req *http.Request, accountDB userdb.Database,
+	req *http.Request, profileAPI userapi.UserProfileAPI,
 	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -244,14 +255,26 @@ func SetDisplayName(
 		}
 	}
 
-	oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
+	pRes := &userapi.QueryProfileResponse{}
+	err = profileAPI.QueryProfile(req.Context(), &userapi.QueryProfileRequest{
+		UserID: userID,
+	}, pRes)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed")
+		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.QueryProfile failed")
 		return jsonerror.InternalServerError()
 	}
+	oldProfile := &authtypes.Profile{
+		Localpart:   localpart,
+		DisplayName: pRes.DisplayName,
+		AvatarURL:   pRes.AvatarURL,
+	}
 
-	if err = accountDB.SetDisplayName(req.Context(), localpart, r.DisplayName); err != nil {
+	err = profileAPI.SetDisplayName(req.Context(), &userapi.PerformUpdateDisplayNameRequest{
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.SetDisplayName failed")
+		Localpart:   localpart,
+		DisplayName: r.DisplayName,
+	}, &struct{}{})
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetDisplayName failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -302,7 +325,7 @@ func SetDisplayName(
 // Returns an error when something goes wrong or specifically
 // eventutil.ErrProfileNoExists when the profile doesn't exist.
 func getProfile(
-	ctx context.Context, accountDB userdb.Database, cfg *config.ClientAPI,
+	ctx context.Context, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
@@ -331,7 +354,7 @@ func getProfile(
 		}, nil
 	}
 
-	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI)
 	if err != nil {
 		return nil, err
 	}

clientapi/routing/register.go

@@ -44,7 +44,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 )
 
 var (
@@ -523,8 +522,7 @@ func validateApplicationService(
 // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
 func Register(
 	req *http.Request,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.UserRegisterAPI,
-	accountDB userdb.Database,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var r registerRequest
@@ -552,13 +550,12 @@ func Register(
 	}
 	// Auto generate a numeric username if r.Username is empty
 	if r.Username == "" {
-		id, err := accountDB.GetNewNumericLocalpart(req.Context())
+		res := &userapi.QueryNumericLocalpartResponse{}
-		if err != nil {
+		if err := userAPI.QueryNumericLocalpart(req.Context(), res); err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetNewNumericLocalpart failed")
+			util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryNumericLocalpart failed")
 			return jsonerror.InternalServerError()
 		}
-
+		r.Username = strconv.FormatInt(res.ID, 10)
-		r.Username = strconv.FormatInt(id, 10)
 	}
 
 	// Is this an appservice registration? It will be if the access
@@ -606,7 +603,7 @@ func handleGuestRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.ClientAPI,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.UserRegisterAPI,
 ) util.JSONResponse {
 	if cfg.RegistrationDisabled || cfg.GuestsDisabled {
 		return util.JSONResponse{
@@ -671,7 +668,7 @@ func handleRegistrationFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.ClientAPI,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.UserRegisterAPI,
 	accessToken string,
 	accessTokenErr error,
 ) util.JSONResponse {
@@ -760,7 +757,7 @@ func handleApplicationServiceRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.ClientAPI,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.UserRegisterAPI,
 ) util.JSONResponse {
 	// Check if we previously had issues extracting the access token from the
 	// request.
@@ -798,7 +795,7 @@ func checkAndCompleteFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.ClientAPI,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.UserRegisterAPI,
 ) util.JSONResponse {
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
@@ -825,7 +822,7 @@ func checkAndCompleteFlow(
 // not all
 func completeRegistration(
 	ctx context.Context,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.UserRegisterAPI,
 	username, password, appserviceID, ipAddr, userAgent, sessionID string,
 	inhibitLogin eventutil.WeakBoolean,
 	displayName, deviceID *string,
@@ -991,7 +988,7 @@ type availableResponse struct {
 func RegisterAvailable(
 	req *http.Request,
 	cfg *config.ClientAPI,
-	accountDB userdb.Database,
+	registerAPI userapi.UserRegisterAPI,
 ) util.JSONResponse {
 	username := req.URL.Query().Get("username")
 
@@ -1013,14 +1010,18 @@ func RegisterAvailable(
 		}
 	}
 
-	availability, availabilityErr := accountDB.CheckAccountAvailability(req.Context(), username)
+	res := &userapi.QueryAccountAvailabilityResponse{}
-	if availabilityErr != nil {
+	err := registerAPI.QueryAccountAvailability(req.Context(), &userapi.QueryAccountAvailabilityRequest{
+		Localpart: username,
+	}, res)
+	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: jsonerror.Unknown("failed to check availability: " + availabilityErr.Error()),
+			JSON: jsonerror.Unknown("failed to check availability:" + err.Error()),
 		}
 	}
-	if !availability {
+
+	if !res.Available {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UserInUse("Desired User ID is already taken."),

clientapi/routing/routing.go

@@ -49,7 +49,6 @@ func Setup(
 	publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
-	accountDB userdb.Database,
 	userAPI userapi.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	syncProducer *producers.SyncAPIProducer,
@@ -60,7 +59,7 @@ func Setup(
 	mscCfg *config.MSCs,
 ) {
 	rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
-	userInteractiveAuth := auth.NewUserInteractive(accountDB, cfg)
+	userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg)
 
 	unstableFeatures := map[string]bool{
 		"org.matrix.e2e_cross_signing": true,
@@ -118,7 +117,7 @@ func Setup(
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
-		serverNotificationSender, err := getSenderDevice(context.Background(), userAPI, accountDB, cfg)
+		serverNotificationSender, err := getSenderDevice(context.Background(), userAPI, cfg)
 		if err != nil {
 			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
 		}
@@ -136,7 +135,7 @@ func Setup(
 				txnID := vars["txnID"]
 				return SendServerNotice(
 					req, &cfg.Matrix.ServerNotices,
-					cfg, userAPI, rsAPI, accountDB, asAPI,
+					cfg, userAPI, rsAPI, asAPI,
 					device, serverNotificationSender,
 					&txnID, transactionsCache,
 				)
@@ -151,7 +150,7 @@ func Setup(
 				}
 				return SendServerNotice(
 					req, &cfg.Matrix.ServerNotices,
-					cfg, userAPI, rsAPI, accountDB, asAPI,
+					cfg, userAPI, rsAPI, asAPI,
 					device, serverNotificationSender,
 					nil, transactionsCache,
 				)
@@ -171,7 +170,7 @@ func Setup(
 
 	v3mux.Handle("/createRoom",
 		httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI)
+			return CreateRoom(req, device, cfg, userAPI, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/join/{roomIDOrAlias}",
@@ -184,7 +183,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return JoinRoomByIDOrAlias(
-				req, device, rsAPI, accountDB, vars["roomIDOrAlias"],
+				req, device, rsAPI, userAPI, vars["roomIDOrAlias"],
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -200,7 +199,7 @@ func Setup(
 					return util.ErrorResponse(err)
 				}
 				return PeekRoomByIDOrAlias(
-					req, device, rsAPI, accountDB, vars["roomIDOrAlias"],
+					req, device, rsAPI, vars["roomIDOrAlias"],
 				)
 			}),
 		).Methods(http.MethodPost, http.MethodOptions)
@@ -220,7 +219,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return JoinRoomByIDOrAlias(
-				req, device, rsAPI, accountDB, vars["roomID"],
+				req, device, rsAPI, userAPI, vars["roomID"],
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -245,7 +244,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return UnpeekRoomByID(
-				req, device, rsAPI, accountDB, vars["roomID"],
+				req, device, rsAPI, vars["roomID"],
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -255,7 +254,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendBan(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+			return SendBan(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/invite",
@@ -267,7 +266,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendInvite(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+			return SendInvite(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/kick",
@@ -276,7 +275,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendKick(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+			return SendKick(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/unban",
@@ -285,7 +284,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendUnban(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+			return SendUnban(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/send/{eventType}",
@@ -381,14 +380,14 @@ func Setup(
 		if r := rateLimits.Limit(req); r != nil {
 			return *r
 		}
-		return Register(req, userAPI, accountDB, cfg)
+		return Register(req, userAPI, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
 	v3mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
 		if r := rateLimits.Limit(req); r != nil {
 			return *r
 		}
-		return RegisterAvailable(req, cfg, accountDB)
+		return RegisterAvailable(req, cfg, userAPI)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	v3mux.Handle("/directory/room/{roomAlias}",
@@ -527,7 +526,7 @@ func Setup(
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
-			return Password(req, userAPI, accountDB, device, cfg)
+			return Password(req, userAPI, device, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -547,7 +546,7 @@ func Setup(
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
-			return Login(req, accountDB, userAPI, cfg)
+			return Login(req, userAPI, cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
@@ -702,7 +701,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetProfile(req, accountDB, cfg, vars["userID"], asAPI, federation)
+			return GetProfile(req, userAPI, cfg, vars["userID"], asAPI, federation)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -712,7 +711,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetAvatarURL(req, accountDB, cfg, vars["userID"], asAPI, federation)
+			return GetAvatarURL(req, userAPI, cfg, vars["userID"], asAPI, federation)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -725,7 +724,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetAvatarURL(req, accountDB, device, vars["userID"], cfg, rsAPI)
+			return SetAvatarURL(req, userAPI, device, vars["userID"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
@@ -737,7 +736,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetDisplayName(req, accountDB, cfg, vars["userID"], asAPI, federation)
+			return GetDisplayName(req, userAPI, cfg, vars["userID"], asAPI, federation)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -750,7 +749,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetDisplayName(req, accountDB, device, vars["userID"], cfg, rsAPI)
+			return SetDisplayName(req, userAPI, device, vars["userID"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
@@ -758,25 +757,25 @@ func Setup(
 
 	v3mux.Handle("/account/3pid",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return GetAssociated3PIDs(req, accountDB, device)
+			return GetAssociated3PIDs(req, userAPI, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	v3mux.Handle("/account/3pid",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return CheckAndSave3PIDAssociation(req, accountDB, device, cfg)
+			return CheckAndSave3PIDAssociation(req, userAPI, device, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	unstableMux.Handle("/account/3pid/delete",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return Forget3PID(req, accountDB)
+			return Forget3PID(req, userAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	v3mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken",
 		httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse {
-			return RequestEmailToken(req, accountDB, cfg)
+			return RequestEmailToken(req, userAPI, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -1251,7 +1250,7 @@ func Setup(
 	// Cross-signing device keys
 
 	postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, keyAPI, device, accountDB, cfg)
+		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, keyAPI, device, userAPI, cfg)
 	})
 
 	postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {

clientapi/routing/server_notices.go

@@ -21,7 +21,6 @@ import (
 	"net/http"
 	"time"
 
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/tokens"
@@ -58,7 +57,6 @@ func SendServerNotice(
 	cfgClient *config.ClientAPI,
 	userAPI userapi.UserInternalAPI,
 	rsAPI api.RoomserverInternalAPI,
-	accountsDB userdb.Database,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	device *userapi.Device,
 	senderDevice *userapi.Device,
@@ -175,7 +173,7 @@ func SendServerNotice(
 			PowerLevelContentOverride: pl,
 		}
 
-		roomRes := createRoom(ctx, crReq, senderDevice, cfgClient, accountsDB, rsAPI, asAPI, time.Now())
+		roomRes := createRoom(ctx, crReq, senderDevice, cfgClient, userAPI, rsAPI, asAPI, time.Now())
 
 		switch data := roomRes.JSON.(type) {
 		case createRoomResponse:
@@ -201,7 +199,7 @@ func SendServerNotice(
 		// we've found a room in common, check the membership
 		roomID = commonRooms[0]
 		// re-invite the user
-		res, err := sendInvite(ctx, accountsDB, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
+		res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
 		if err != nil {
 			return res
 		}
@@ -284,7 +282,6 @@ func (r sendServerNoticeRequest) valid() (ok bool) {
 func getSenderDevice(
 	ctx context.Context,
 	userAPI userapi.UserInternalAPI,
-	accountDB userdb.Database,
 	cfg *config.ClientAPI,
 ) (*userapi.Device, error) {
 	var accRes userapi.PerformAccountCreationResponse
@@ -299,8 +296,12 @@ func getSenderDevice(
 	}
 
 	// set the avatarurl for the user
-	if err = accountDB.SetAvatarURL(ctx, cfg.Matrix.ServerNotices.LocalPart, cfg.Matrix.ServerNotices.AvatarURL); err != nil {
+	res := &userapi.PerformSetAvatarURLResponse{}
-		util.GetLogger(ctx).WithError(err).Error("accountDB.SetAvatarURL failed")
+	if err = userAPI.SetAvatarURL(ctx, &userapi.PerformSetAvatarURLRequest{
+		Localpart: cfg.Matrix.ServerNotices.LocalPart,
+		AvatarURL: cfg.Matrix.ServerNotices.AvatarURL,
+	}, res); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("userAPI.SetAvatarURL failed")
 		return nil, err
 	}
 

clientapi/routing/threepid.go

@@ -40,7 +40,7 @@ type threePIDsResponse struct {
 // RequestEmailToken implements:
 //     POST /account/3pid/email/requestToken
 //     POST /register/email/requestToken
-func RequestEmailToken(req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI) util.JSONResponse {
+func RequestEmailToken(req *http.Request, threePIDAPI api.UserThreePIDAPI, cfg *config.ClientAPI) util.JSONResponse {
 	var body threepid.EmailAssociationRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr
@@ -50,13 +50,18 @@ func RequestEmailToken(req *http.Request, accountDB userdb.Database, cfg *config
 	var err error
 
 	// Check if the 3PID is already in use locally
-	localpart, err := accountDB.GetLocalpartForThreePID(req.Context(), body.Email, "email")
+	res := &api.QueryLocalpartForThreePIDResponse{}
+	err = threePIDAPI.QueryLocalpartForThreePID(req.Context(), &api.QueryLocalpartForThreePIDRequest{
+		ThreePID: body.Email,
+		Medium:   "email",
+	}, res)
+
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetLocalpartForThreePID failed")
+		util.GetLogger(req.Context()).WithError(err).Error("threePIDAPI.QueryLocalpartForThreePID failed")
 		return jsonerror.InternalServerError()
 	}
 
-	if len(localpart) > 0 {
+	if len(res.Localpart) > 0 {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.MatrixError{
@@ -85,7 +90,7 @@ func RequestEmailToken(req *http.Request, accountDB userdb.Database, cfg *config
 
 // CheckAndSave3PIDAssociation implements POST /account/3pid
 func CheckAndSave3PIDAssociation(
-	req *http.Request, accountDB userdb.Database, device *api.Device,
+	req *http.Request, threePIDAPI api.UserThreePIDAPI, device *api.Device,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var body threepid.EmailAssociationCheckRequest
@@ -136,8 +141,12 @@ func CheckAndSave3PIDAssociation(
 		return jsonerror.InternalServerError()
 	}
 
-	if err = accountDB.SaveThreePIDAssociation(req.Context(), address, localpart, medium); err != nil {
+	if err = threePIDAPI.PerformSaveThreePIDAssociation(req.Context(), &api.PerformSaveThreePIDAssociationRequest{
-		util.GetLogger(req.Context()).WithError(err).Error("accountsDB.SaveThreePIDAssociation failed")
+		ThreePID:  address,
+		Localpart: localpart,
+		Medium:    medium,
+	}, &struct{}{}); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("threePIDAPI.PerformSaveThreePIDAssociation failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -149,7 +158,7 @@ func CheckAndSave3PIDAssociation(
 
 // GetAssociated3PIDs implements GET /account/3pid
 func GetAssociated3PIDs(
-	req *http.Request, accountDB userdb.Database, device *api.Device,
+	req *http.Request, threepidAPI api.UserThreePIDAPI, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
@@ -157,27 +166,30 @@ func GetAssociated3PIDs(
 		return jsonerror.InternalServerError()
 	}
 
-	threepids, err := accountDB.GetThreePIDsForLocalpart(req.Context(), localpart)
+	res := &api.QueryThreePIDsForLocalpartResponse{}
+	err = threepidAPI.QueryThreePIDsForLocalpart(req.Context(), &api.QueryThreePIDsForLocalpartRequest{
+		Localpart: localpart,
+	}, res)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetThreePIDsForLocalpart failed")
+		util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.QueryThreePIDsForLocalpart failed")
 		return jsonerror.InternalServerError()
 	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: threePIDsResponse{threepids},
+		JSON: threePIDsResponse{res.ThreePIDs},
 	}
 }
 
 // Forget3PID implements POST /account/3pid/delete
-func Forget3PID(req *http.Request, accountDB userdb.Database) util.JSONResponse {
+func Forget3PID(req *http.Request, threepidAPI api.UserThreePIDAPI) util.JSONResponse {
 	var body authtypes.ThreePID
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr
 	}
 
-	if err := accountDB.RemoveThreePIDAssociation(req.Context(), body.Address, body.Medium); err != nil {
+	if err := threepidAPI.PerformForgetThreePID(req.Context(), &api.PerformForgetThreePIDRequest{}, &struct{}{}); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.RemoveThreePIDAssociation failed")
+		util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.PerformForgetThreePID failed")
 		return jsonerror.InternalServerError()
 	}
 

clientapi/threepid/invites.go

@@ -29,7 +29,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -87,7 +86,7 @@ var (
 func CheckAndProcessInvite(
 	ctx context.Context,
 	device *userapi.Device, body *MembershipRequest, cfg *config.ClientAPI,
-	rsAPI api.RoomserverInternalAPI, db userdb.Database,
+	rsAPI api.RoomserverInternalAPI, db userapi.UserProfileAPI,
 	roomID string,
 	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
@@ -137,7 +136,7 @@ func CheckAndProcessInvite(
 // Returns an error if a check or a request failed.
 func queryIDServer(
 	ctx context.Context,
-	db userdb.Database, cfg *config.ClientAPI, device *userapi.Device,
+	db userapi.UserProfileAPI, cfg *config.ClientAPI, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) {
 	if err = isTrusted(body.IDServer, cfg); err != nil {
@@ -206,7 +205,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe
 // Returns an error if the request failed to send or if the response couldn't be parsed.
 func queryIDServerStoreInvite(
 	ctx context.Context,
-	db userdb.Database, cfg *config.ClientAPI, device *userapi.Device,
+	db userapi.UserProfileAPI, cfg *config.ClientAPI, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (*idServerStoreInviteResponse, error) {
 	// Retrieve the sender's profile to get their display name
@@ -217,10 +216,17 @@ func queryIDServerStoreInvite(
 
 	var profile *authtypes.Profile
 	if serverName == cfg.Matrix.ServerName {
-		profile, err = db.GetProfileByLocalpart(ctx, localpart)
+		res := &userapi.QueryProfileResponse{}
+		err = db.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res)
 		if err != nil {
 			return nil, err
 		}
+		profile = &authtypes.Profile{
+			Localpart:   localpart,
+			DisplayName: res.DisplayName,
+			AvatarURL:   res.AvatarURL,
+		}
+
 	} else {
 		profile = &authtypes.Profile{}
 	}

cmd/create-account/main.go

@@ -24,12 +24,11 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/term"
-
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/userapi/api"
 )
 
 const usage = `Usage: %s
@@ -43,7 +42,7 @@ Example:
 	# use password from file
   	%s --config dendrite.yaml -username alice -passwordfile my.pass
 	# ask user to provide password
-	%s --config dendrite.yaml -username alice -ask-pass
+	%s --config dendrite.yaml -username alice
 	# read password from stdin
 	%s --config dendrite.yaml -username alice -passwordstdin < my.pass
 	cat my.pass | %s --config dendrite.yaml -username alice -passwordstdin
@@ -56,10 +55,10 @@ Arguments:
 
 var (
 	username           = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
-	password           = flag.String("password", "", "The password to associate with the account (optional, account will be password-less if not specified)")
+	password           = flag.String("password", "", "The password to associate with the account")
 	pwdFile            = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
 	pwdStdin           = flag.Bool("passwordstdin", false, "Reads the password from stdin")
-	askPass            = flag.Bool("ask-pass", false, "Ask for the password to use")
+	pwdLess            = flag.Bool("passwordless", false, "Create a passwordless account, e.g. if only an accesstoken is required")
 	isAdmin            = flag.Bool("admin", false, "Create an admin account")
 	resetPassword      = flag.Bool("reset-password", false, "Resets the password for the given username")
 	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
@@ -78,22 +77,44 @@ func main() {
 		os.Exit(1)
 	}
 
+	if *pwdLess && *resetPassword {
+		logrus.Fatalf("Can not reset to an empty password, unable to login afterwards.")
+	}
+
 	if !validUsernameRegex.MatchString(*username) {
 		logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='")
 		os.Exit(1)
 	}
 
-	pass := getPassword(password, pwdFile, pwdStdin, askPass, os.Stdin)
+	if len(fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName)) > 255 {
+		logrus.Fatalf("Username can not be longer than 255 characters: %s", fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName))
+	}
 
-	b := base.NewBaseDendrite(cfg, "create-account")
+	var pass string
+	var err error
+	if !*pwdLess {
+		pass, err = getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
+		if err != nil {
+			logrus.Fatalln(err)
+		}
+	}
+
+	b := base.NewBaseDendrite(cfg, "Monolith")
 	accountDB := b.CreateAccountsDB()
 
 	accType := api.AccountTypeUser
 	if *isAdmin {
 		accType = api.AccountTypeAdmin
 	}
-	var err error
+
+	available, err := accountDB.CheckAccountAvailability(context.Background(), *username)
+	if err != nil {
+		logrus.Fatalln("Unable check username existence.")
+	}
 	if *resetPassword {
+		if available {
+			logrus.Fatalln("Username could not be found.")
+		}
 		err = accountDB.SetPassword(context.Background(), *username, pass)
 		if err != nil {
 			logrus.Fatalf("Failed to update password for user %s: %s", *username, err.Error())
@@ -104,6 +125,9 @@ func main() {
 		logrus.Infof("Updated password for user %s and invalidated all logins\n", *username)
 		return
 	}
+	if !available {
+		logrus.Fatalln("Username is already in use.")
+	}
 
 	_, err = accountDB.CreateAccount(context.Background(), *username, pass, "", accType)
 	if err != nil {
@@ -113,53 +137,44 @@ func main() {
 	logrus.Infoln("Created account", *username)
 }
 
-func getPassword(password, pwdFile *string, pwdStdin, askPass *bool, r io.Reader) string {
+func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string, error) {
-	// no password option set, use empty password
-	if password == nil && pwdFile == nil && pwdStdin == nil && askPass == nil {
-		return ""
-	}
-	// password defined as parameter
-	if password != nil && *password != "" {
-		return *password
-	}
-
 	// read password from file
-	if pwdFile != nil && *pwdFile != "" {
+	if pwdFile != "" {
-		pw, err := ioutil.ReadFile(*pwdFile)
+		pw, err := ioutil.ReadFile(pwdFile)
 		if err != nil {
-			logrus.Fatalln("Unable to read password from file:", err)
+			return "", fmt.Errorf("Unable to read password from file: %v", err)
 		}
-		return strings.TrimSpace(string(pw))
+		return strings.TrimSpace(string(pw)), nil
 	}
 
 	// read password from stdin
-	if pwdStdin != nil && *pwdStdin {
+	if pwdStdin {
 		data, err := ioutil.ReadAll(r)
 		if err != nil {
-			logrus.Fatalln("Unable to read password from stdin:", err)
+			return "", fmt.Errorf("Unable to read password from stdin: %v", err)
 		}
-		return strings.TrimSpace(string(data))
+		return strings.TrimSpace(string(data)), nil
 	}
 
-	// ask the user to provide the password
+	// If no parameter was set, ask the user to provide the password
-	if *askPass {
+	if password == "" {
 		fmt.Print("Enter Password: ")
 		bytePassword, err := term.ReadPassword(int(os.Stdin.Fd()))
 		if err != nil {
-			logrus.Fatalln("Unable to read password:", err)
+			return "", fmt.Errorf("Unable to read password: %v", err)
 		}
 		fmt.Println()
 		fmt.Print("Confirm Password: ")
 		bytePassword2, err := term.ReadPassword(int(os.Stdin.Fd()))
 		if err != nil {
-			logrus.Fatalln("Unable to read password:", err)
+			return "", fmt.Errorf("Unable to read password: %v", err)
 		}
 		fmt.Println()
 		if strings.TrimSpace(string(bytePassword)) != strings.TrimSpace(string(bytePassword2)) {
-			logrus.Fatalln("Entered passwords don't match")
+			return "", fmt.Errorf("Entered passwords don't match")
 		}
-		return strings.TrimSpace(string(bytePassword))
+		return strings.TrimSpace(string(bytePassword)), nil
 	}
 
-	return ""
+	return password, nil
 }

cmd/create-account/main_test.go

@@ -8,45 +8,48 @@ import (
 
 func Test_getPassword(t *testing.T) {
 	type args struct {
-		password *string
+		password string
-		pwdFile  *string
+		pwdFile  string
-		pwdStdin *bool
+		pwdStdin bool
-		askPass  *bool
 		reader   io.Reader
 	}
 
 	pass := "mySecretPass"
 	passwordFile := "testdata/my.pass"
-	passwordStdin := true
 	reader := &bytes.Buffer{}
 	_, err := reader.WriteString(pass)
 	if err != nil {
 		t.Errorf("unable to write to buffer: %+v", err)
 	}
 	tests := []struct {
-		name string
+		name    string
-		args args
+		args    args
-		want string
+		want    string
+		wantErr bool
 	}{
-		{
-			name: "no password defined",
-			args: args{},
-			want: "",
-		},
 		{
 			name: "password defined",
-			args: args{password: &pass},
+			args: args{
+				password: pass,
+			},
 			want: pass,
 		},
 		{
 			name: "pwdFile defined",
-			args: args{pwdFile: &passwordFile},
+			args: args{
+				pwdFile: passwordFile,
+			},
 			want: pass,
 		},
+		{
+			name:    "pwdFile does not exist",
+			args:    args{pwdFile: "iDontExist"},
+			wantErr: true,
+		},
 		{
 			name: "read pass from stdin defined",
 			args: args{
-				pwdStdin: &passwordStdin,
+				pwdStdin: true,
 				reader:   reader,
 			},
 			want: pass,
@@ -54,7 +57,11 @@ func Test_getPassword(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := getPassword(tt.args.password, tt.args.pwdFile, tt.args.pwdStdin, tt.args.askPass, tt.args.reader); got != tt.want {
+			got, err := getPassword(tt.args.password, tt.args.pwdFile, tt.args.pwdStdin, tt.args.reader)
+			if !tt.wantErr && err != nil {
+				t.Errorf("expected no error, but got %v", err)
+			}
+			if got != tt.want {
 				t.Errorf("getPassword() = '%v', want '%v'", got, tt.want)
 			}
 		})

internal/httputil/paths.go

@@ -21,4 +21,6 @@ const (
 	PublicMediaPathPrefix      = "/_matrix/media/"
 	PublicWellKnownPrefix      = "/.well-known/matrix/"
 	InternalPathPrefix         = "/api/"
+	DendriteAdminPathPrefix    = "/_dendrite/"
+	SynapseAdminPathPrefix     = "/_synapse/"
 )

internal/version.go

@@ -16,8 +16,8 @@ var build string
 
 const (
 	VersionMajor = 0
-	VersionMinor = 6
+	VersionMinor = 7
-	VersionPatch = 5
+	VersionPatch = 0
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/internal/internal.go

@@ -223,6 +223,7 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query
 	res.StreamID = maxStreamID
 }
 
+// nolint:gocyclo
 func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) {
 	res.DeviceKeys = make(map[string]map[string]json.RawMessage)
 	res.MasterKeys = make(map[string]gomatrixserverlib.CrossSigningKey)

roomserver/api/alias.go

@@ -102,4 +102,3 @@ func (a AliasEvent) Valid() bool {
 	}
 	return a.Alias == "" || validateAliasRegex.MatchString(a.Alias)
 }
-

roomserver/api/alias_test.go

@@ -22,29 +22,29 @@ func TestAliasEvent_Valid(t *testing.T) {
 		{
 			name: "empty alias, invalid alt aliases",
 			fields: fields{
-				Alias: "",
+				Alias:      "",
-				AltAliases: []string{ "%not:valid.local"},
+				AltAliases: []string{"%not:valid.local"},
 			},
 		},
 		{
 			name: "valid alias, invalid alt aliases",
 			fields: fields{
-				Alias: "#valid:test.local",
+				Alias:      "#valid:test.local",
-				AltAliases: []string{ "%not:valid.local"},
+				AltAliases: []string{"%not:valid.local"},
 			},
 		},
 		{
 			name: "empty alias, invalid alt aliases",
 			fields: fields{
-				Alias: "",
+				Alias:      "",
-				AltAliases: []string{ "%not:valid.local"},
+				AltAliases: []string{"%not:valid.local"},
 			},
 		},
 		{
 			name: "invalid alias",
 			fields: fields{
-				Alias: "%not:valid.local",
+				Alias:      "%not:valid.local",
-				AltAliases: []string{ },
+				AltAliases: []string{},
 			},
 		},
 	}

roomserver/internal/alias.go

@@ -173,12 +173,15 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias(
 	}
 
 	if creatorID != request.UserID {
-		plEvent, err := r.DB.GetStateEvent(ctx, roomID, gomatrixserverlib.MRoomPowerLevels, "")
+		var plEvent *gomatrixserverlib.HeaderedEvent
+		var pls *gomatrixserverlib.PowerLevelContent
+
+		plEvent, err = r.DB.GetStateEvent(ctx, roomID, gomatrixserverlib.MRoomPowerLevels, "")
 		if err != nil {
 			return fmt.Errorf("r.DB.GetStateEvent: %w", err)
 		}
 
-		pls, err := plEvent.PowerLevels()
+		pls, err = plEvent.PowerLevels()
 		if err != nil {
 			return fmt.Errorf("plEvent.PowerLevels: %w", err)
 		}
@@ -223,7 +226,7 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias(
 			}
 
 			stateRes := &api.QueryLatestEventsAndStateResponse{}
-			if err := helpers.QueryLatestEventsAndState(ctx, r.DB, &api.QueryLatestEventsAndStateRequest{RoomID: roomID, StateToFetch: eventsNeeded.Tuples()}, stateRes); err != nil {
+			if err = helpers.QueryLatestEventsAndState(ctx, r.DB, &api.QueryLatestEventsAndStateRequest{RoomID: roomID, StateToFetch: eventsNeeded.Tuples()}, stateRes); err != nil {
 				return err
 			}
 

roomserver/internal/perform/perform_leave.go

@@ -212,12 +212,34 @@ func (r *Leaver) performFederatedRejectInvite(
 		ServerNames: []gomatrixserverlib.ServerName{domain},
 	}
 	leaveRes := fsAPI.PerformLeaveResponse{}
-	if err := r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil {
+	if err = r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil {
 		// failures in PerformLeave should NEVER stop us from telling other components like the
 		// sync API that the invite was withdrawn. Otherwise we can end up with stuck invites.
 		util.GetLogger(ctx).WithError(err).Errorf("failed to PerformLeave, still retiring invite event")
 	}
 
+	info, err := r.DB.RoomInfo(ctx, req.RoomID)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Errorf("failed to get RoomInfo, still retiring invite event")
+	}
+
+	updater, err := r.DB.MembershipUpdater(ctx, req.RoomID, req.UserID, true, info.RoomVersion)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Errorf("failed to get MembershipUpdater, still retiring invite event")
+	}
+	if updater != nil {
+		if _, err = updater.SetToLeave(req.UserID, eventID); err != nil {
+			util.GetLogger(ctx).WithError(err).Errorf("failed to set membership to leave, still retiring invite event")
+			if err = updater.Rollback(); err != nil {
+				util.GetLogger(ctx).WithError(err).Errorf("failed to rollback membership leave, still retiring invite event")
+			}
+		} else {
+			if err = updater.Commit(); err != nil {
+				util.GetLogger(ctx).WithError(err).Errorf("failed to commit membership update, still retiring invite event")
+			}
+		}
+	}
+
 	// Withdraw the invite, so that the sync API etc are
 	// notified that we rejected it.
 	return []api.OutputEvent{

roomserver/storage/sqlite3/event_state_keys_table.go

@@ -151,7 +151,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey(
 	if err != nil {
 		return nil, err
 	}
-	defer selectPrep.Close()
+	defer internal.CloseAndLogIfError(ctx, selectPrep, "selectPrep.close() failed")
 	stmt := sqlutil.TxStmt(txn, selectPrep)
 	rows, err := stmt.QueryContext(ctx, iEventStateKeyNIDs...)
 	if err != nil {

roomserver/storage/sqlite3/event_types_table.go

@@ -128,7 +128,7 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID(
 	if err != nil {
 		return nil, err
 	}
-	defer selectPrep.Close()
+	defer internal.CloseAndLogIfError(ctx, selectPrep, "selectPrep.close() failed")
 	stmt := sqlutil.TxStmt(txn, selectPrep)
 	///////////////
 

roomserver/storage/sqlite3/events_table.go

@@ -567,7 +567,7 @@ func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx,
 	if err != nil {
 		return 0, err
 	}
-	defer sqlPrep.Close()
+	defer internal.CloseAndLogIfError(ctx, sqlPrep, "sqlPrep.close() failed")
 	err = sqlutil.TxStmt(txn, sqlPrep).QueryRowContext(ctx, iEventIDs...).Scan(&result)
 	if err != nil {
 		return 0, fmt.Errorf("sqlutil.TxStmt.QueryRowContext: %w", err)
@@ -583,7 +583,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs(
 	if err != nil {
 		return nil, err
 	}
-	defer sqlPrep.Close()
+	defer internal.CloseAndLogIfError(ctx, sqlPrep, "sqlPrep.close() failed")
 	sqlStmt := sqlutil.TxStmt(txn, sqlPrep)
 	iEventNIDs := make([]interface{}, len(eventNIDs))
 	for i, v := range eventNIDs {

setup/base/base.go

@@ -74,6 +74,7 @@ type BaseDendrite struct {
 	PublicMediaAPIMux      *mux.Router
 	PublicWellKnownAPIMux  *mux.Router
 	InternalAPIMux         *mux.Router
+	DendriteAdminMux       *mux.Router
 	SynapseAdminMux        *mux.Router
 	UseHTTPAPIs            bool
 	apiHttpClient          *http.Client
@@ -206,7 +207,8 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		PublicMediaAPIMux:      mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(),
 		PublicWellKnownAPIMux:  mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicWellKnownPrefix).Subrouter().UseEncodedPath(),
 		InternalAPIMux:         mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(),
-		SynapseAdminMux:        mux.NewRouter().SkipClean(true).PathPrefix("/_synapse/").Subrouter().UseEncodedPath(),
+		DendriteAdminMux:       mux.NewRouter().SkipClean(true).PathPrefix(httputil.DendriteAdminPathPrefix).Subrouter().UseEncodedPath(),
+		SynapseAdminMux:        mux.NewRouter().SkipClean(true).PathPrefix(httputil.SynapseAdminPathPrefix).Subrouter().UseEncodedPath(),
 		apiHttpClient:          &apiClient,
 	}
 }
@@ -366,6 +368,17 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		internalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth))
 	}
 
+	b.DendriteAdminMux.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+	})
+	b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) {
+		if b.ProcessContext.IsDegraded() {
+			w.WriteHeader(503)
+			return
+		}
+		w.WriteHeader(200)
+	})
+
 	var clientHandler http.Handler
 	clientHandler = b.PublicClientAPIMux
 	if b.Cfg.Global.Sentry.Enabled {
@@ -382,12 +395,13 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		})
 		federationHandler = sentryHandler.Handle(b.PublicFederationAPIMux)
 	}
+	internalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(b.DendriteAdminMux)
 	externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(clientHandler)
 	if !b.Cfg.Global.DisableFederation {
 		externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.PublicKeyAPIMux)
 		externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(federationHandler)
 	}
-	externalRouter.PathPrefix("/_synapse/").Handler(b.SynapseAdminMux)
+	externalRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(b.SynapseAdminMux)
 	externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux)
 	externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(b.PublicWellKnownAPIMux)
 

setup/config/config_appservice.go

@@ -209,13 +209,14 @@ func setupRegexps(asAPI *AppServiceAPI, derived *Derived) (err error) {
 	for _, appservice := range derived.ApplicationServices {
 		// The sender_localpart can be considered an exclusive regex for a single user, so let's do that
 		// to simplify the code
-		var senderUserIDSlice = []string{fmt.Sprintf("@%s:%s", appservice.SenderLocalpart, asAPI.Matrix.ServerName)}
+		users, found := appservice.NamespaceMap["users"]
-		usersSlice, found := appservice.NamespaceMap["users"]
 		if !found {
-			usersSlice = []ApplicationServiceNamespace{}
+			users = []ApplicationServiceNamespace{}
-			appservice.NamespaceMap["users"] = usersSlice
 		}
-		appendExclusiveNamespaceRegexs(&senderUserIDSlice, usersSlice)
+		appservice.NamespaceMap["users"] = append(users, ApplicationServiceNamespace{
+			Exclusive: true,
+			Regex:     regexp.QuoteMeta(fmt.Sprintf("@%s:%s", appservice.SenderLocalpart, asAPI.Matrix.ServerName)),
+		})
 
 		for key, namespaceSlice := range appservice.NamespaceMap {
 			switch key {

setup/jetstream/nats.go

@@ -1,11 +1,13 @@
 package jetstream
 
 import (
+	"fmt"
 	"reflect"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/sirupsen/logrus"
@@ -20,7 +22,7 @@ var natsServerMutex sync.Mutex
 func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) {
 	// check if we need an in-process NATS Server
 	if len(cfg.Addresses) != 0 {
-		return setupNATS(cfg, nil)
+		return setupNATS(process, cfg, nil)
 	}
 	natsServerMutex.Lock()
 	if natsServer == nil {
@@ -56,10 +58,10 @@ func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient
 	if err != nil {
 		logrus.Fatalln("Failed to create NATS client")
 	}
-	return setupNATS(cfg, nc)
+	return setupNATS(process, cfg, nc)
 }
 
-func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
+func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
 	if nc == nil {
 		var err error
 		nc, err = natsclient.Connect(strings.Join(cfg.Addresses, ","))
@@ -117,7 +119,40 @@ func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStream
 			namespaced.Name = name
 			namespaced.Subjects = subjects
 			if _, err = s.AddStream(&namespaced); err != nil {
-				logrus.WithError(err).WithField("stream", name).WithField("subjects", subjects).Fatal("Unable to add stream")
+				logger := logrus.WithError(err).WithFields(logrus.Fields{
+					"stream":   namespaced.Name,
+					"subjects": namespaced.Subjects,
+				})
+
+				// If the stream was supposed to be in-memory to begin with
+				// then an error here is fatal so we'll give up.
+				if namespaced.Storage == natsclient.MemoryStorage {
+					logger.WithError(err).Fatal("Unable to add in-memory stream")
+				}
+
+				// The stream was supposed to be on disk. Let's try starting
+				// Dendrite with the stream in-memory instead. That'll mean that
+				// we can't recover anything that was queued on the disk but we
+				// will still be able to start and run hopefully in the meantime.
+				logger.WithError(err).Error("Unable to add stream")
+				sentry.CaptureException(fmt.Errorf("Unable to add stream %q: %w", namespaced.Name, err))
+
+				namespaced.Storage = natsclient.MemoryStorage
+				if _, err = s.AddStream(&namespaced); err != nil {
+					// We tried to add the stream in-memory instead but something
+					// went wrong. That's an unrecoverable situation so we will
+					// give up at this point.
+					logger.WithError(err).Fatal("Unable to add in-memory stream")
+				}
+
+				if stream.Storage != namespaced.Storage {
+					// We've managed to add the stream in memory.  What's on the
+					// disk will be left alone, but our ability to recover from a
+					// future crash will be limited. Yell about it.
+					sentry.CaptureException(fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible", namespaced.Name))
+					logrus.Warn("Stream is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible")
+					process.Degraded()
+				}
 			}
 		}
 	}

setup/mscs/msc2946/msc2946.go

@@ -283,11 +283,7 @@ func (w *walker) walk() util.JSONResponse {
 		if !roomExists {
 			// attempt to query this room over federation, as either we've never heard of it before
 			// or we've left it and hence are not authorised (but info may be exposed regardless)
-			fedRes, err := w.federatedRoomInfo(rv.roomID, rv.vias)
+			fedRes := w.federatedRoomInfo(rv.roomID, rv.vias)
-			if err != nil {
-				util.GetLogger(w.ctx).WithError(err).WithField("room_id", rv.roomID).Errorf("failed to query federated spaces")
-				continue
-			}
 			if fedRes != nil {
 				discoveredChildEvents = fedRes.Room.ChildrenState
 				discoveredRooms = append(discoveredRooms, fedRes.Room)
@@ -420,15 +416,15 @@ func (w *walker) publicRoomsChunk(roomID string) *gomatrixserverlib.PublicRoom {
 
 // federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was
 // unsuccessful.
-func (w *walker) federatedRoomInfo(roomID string, vias []string) (*gomatrixserverlib.MSC2946SpacesResponse, error) {
+func (w *walker) federatedRoomInfo(roomID string, vias []string) *gomatrixserverlib.MSC2946SpacesResponse {
 	// only do federated requests for client requests
 	if w.caller == nil {
-		return nil, nil
+		return nil
 	}
 	resp, ok := w.cache.GetSpaceSummary(roomID)
 	if ok {
 		util.GetLogger(w.ctx).Debugf("Returning cached response for %s", roomID)
-		return &resp, nil
+		return &resp
 	}
 	util.GetLogger(w.ctx).Debugf("Querying %s via %+v", roomID, vias)
 	ctx := context.Background()
@@ -455,9 +451,9 @@ func (w *walker) federatedRoomInfo(roomID string, vias []string) (*gomatrixserve
 		}
 		w.cache.StoreSpaceSummary(roomID, res)
 
-		return &res, nil
+		return &res
 	}
-	return nil, nil
+	return nil
 }
 
 func (w *walker) roomExists(roomID string) bool {
@@ -717,23 +713,6 @@ func stripped(ev *gomatrixserverlib.Event) *gomatrixserverlib.MSC2946StrippedEve
 	}
 }
 
-func eventKey(event *gomatrixserverlib.MSC2946StrippedEvent) string {
-	return event.RoomID + "|" + event.Type + "|" + event.StateKey
-}
-
-func spaceTargetStripped(event *gomatrixserverlib.MSC2946StrippedEvent) string {
-	if event.StateKey == "" {
-		return "" // no-op
-	}
-	switch event.Type {
-	case ConstSpaceParentEventType:
-		return event.StateKey
-	case ConstSpaceChildEventType:
-		return event.StateKey
-	}
-	return ""
-}
-
 func parseInt(intstr string, defaultVal int) int {
 	i, err := strconv.ParseInt(intstr, 10, 32)
 	if err != nil {

setup/process/process.go

@@ -2,13 +2,19 @@ package process
 
 import (
 	"context"
+	"fmt"
 	"sync"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/sirupsen/logrus"
+	"go.uber.org/atomic"
 )
 
 type ProcessContext struct {
 	wg       *sync.WaitGroup    // used to wait for components to shutdown
 	ctx      context.Context    // cancelled when Stop is called
 	shutdown context.CancelFunc // shut down Dendrite
+	degraded atomic.Bool
 }
 
 func NewProcessContext() *ProcessContext {
@@ -43,3 +49,14 @@ func (b *ProcessContext) WaitForShutdown() <-chan struct{} {
 func (b *ProcessContext) WaitForComponentsToFinish() {
 	b.wg.Wait()
 }
+
+func (b *ProcessContext) Degraded() {
+	if b.degraded.CAS(false, true) {
+		logrus.Warn("Dendrite is running in a degraded state")
+		sentry.CaptureException(fmt.Errorf("Process is running in a degraded state"))
+	}
+}
+
+func (b *ProcessContext) IsDegraded() bool {
+	return b.degraded.Load()
+}

syncapi/storage/postgres/output_room_events_table.go

@@ -472,7 +472,7 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
 	if err != nil {
 		return
 	}
-	defer rows.Close()
+	defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed")
 
 	for rows.Next() {
 		var (
@@ -504,7 +504,7 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent(
 	if err != nil {
 		return
 	}
-	defer rows.Close()
+	defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed")
 
 	for rows.Next() {
 		var (

syncapi/storage/sqlite3/output_room_events_table.go

@@ -514,7 +514,7 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
 	if err != nil {
 		return
 	}
-	defer rows.Close()
+	defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed")
 
 	for rows.Next() {
 		var (
@@ -550,7 +550,7 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent(
 	if err != nil {
 		return
 	}
-	defer rows.Close()
+	defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed")
 
 	for rows.Next() {
 		var (

syncapi/streams/stream_pdu.go

@@ -147,7 +147,6 @@ func (p *PDUStreamProvider) IncrementalSync(
 		To:        to,
 		Backwards: from > to,
 	}
-	newPos = to
 
 	var err error
 	var stateDeltas []types.StateDelta
@@ -172,14 +171,26 @@ func (p *PDUStreamProvider) IncrementalSync(
 		req.Rooms[roomID] = gomatrixserverlib.Join
 	}
 
+	if len(stateDeltas) == 0 {
+		return to
+	}
+
+	newPos = from
 	for _, delta := range stateDeltas {
-		if err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, req.Response); err != nil {
+		var pos types.StreamPosition
+		if pos, err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, req.Response); err != nil {
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
-			return newPos
+			return to
+		}
+		switch {
+		case r.Backwards && pos < newPos:
+			fallthrough
+		case !r.Backwards && pos > newPos:
+			newPos = pos
 		}
 	}
 
-	return r.To
+	return newPos
 }
 
 func (p *PDUStreamProvider) addRoomDeltaToResponse(
@@ -189,7 +200,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	delta types.StateDelta,
 	eventFilter *gomatrixserverlib.RoomEventFilter,
 	res *types.Response,
-) error {
+) (types.StreamPosition, error) {
 	if delta.MembershipPos > 0 && delta.Membership == gomatrixserverlib.Leave {
 		// make sure we don't leak recent events after the leave event.
 		// TODO: History visibility makes this somewhat complex to handle correctly. For example:
@@ -204,19 +215,42 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		eventFilter, true, true,
 	)
 	if err != nil {
-		return err
+		return r.From, err
 	}
 	recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents)
 	delta.StateEvents = removeDuplicates(delta.StateEvents, recentEvents) // roll back
 	prevBatch, err := p.DB.GetBackwardTopologyPos(ctx, recentStreamEvents)
 	if err != nil {
-		return err
+		return r.From, err
 	}
 
-	// XXX: should we ever get this far if we have no recent events or state in this room?
+	// If we didn't return any events at all then don't bother doing anything else.
-	// in practice we do for peeks, but possibly not joins?
 	if len(recentEvents) == 0 && len(delta.StateEvents) == 0 {
-		return nil
+		return r.To, nil
+	}
+
+	// Sort the events so that we can pick out the latest events from both sections.
+	recentEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(recentEvents, gomatrixserverlib.TopologicalOrderByPrevEvents)
+	delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(delta.StateEvents, gomatrixserverlib.TopologicalOrderByAuthEvents)
+
+	// Work out what the highest stream position is for all of the events in this
+	// room that were returned.
+	latestPosition := r.To
+	updateLatestPosition := func(mostRecentEventID string) {
+		if _, pos, err := p.DB.PositionInTopology(ctx, mostRecentEventID); err == nil {
+			switch {
+			case r.Backwards && pos > latestPosition:
+				fallthrough
+			case !r.Backwards && pos < latestPosition:
+				latestPosition = pos
+			}
+		}
+	}
+	if len(recentEvents) > 0 {
+		updateLatestPosition(recentEvents[len(recentEvents)-1].EventID())
+	}
+	if len(delta.StateEvents) > 0 {
+		updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID())
 	}
 
 	switch delta.Membership {
@@ -250,7 +284,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		res.Rooms.Leave[delta.RoomID] = *lr
 	}
 
-	return nil
+	return latestPosition, nil
 }
 
 func (p *PDUStreamProvider) getJoinResponseForCompleteSync(

userapi/api/api.go

@@ -27,16 +27,16 @@ import (
 // UserInternalAPI is the internal API for information about users and devices.
 type UserInternalAPI interface {
 	LoginTokenInternalAPI
+	UserProfileAPI
+	UserRegisterAPI
+	UserAccountAPI
+	UserThreePIDAPI
 
 	InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error
 
-	PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error
-	PerformPasswordUpdate(ctx context.Context, req *PerformPasswordUpdateRequest, res *PerformPasswordUpdateResponse) error
-	PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error
 	PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error
 	PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error
 	PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error
-	PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error
 	PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error
 	PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error
 	PerformPusherSet(ctx context.Context, req *PerformPusherSetRequest, res *struct{}) error
@@ -44,18 +44,47 @@ type UserInternalAPI interface {
 	PerformPushRulesPut(ctx context.Context, req *PerformPushRulesPutRequest, res *struct{}) error
 
 	QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse)
-	QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error
 	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
 	QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error
 	QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error
 	QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error
-	QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error
 	QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error
 	QueryPushers(ctx context.Context, req *QueryPushersRequest, res *QueryPushersResponse) error
 	QueryPushRules(ctx context.Context, req *QueryPushRulesRequest, res *QueryPushRulesResponse) error
 	QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error
 }
 
+// UserProfileAPI provides functions for getting user profiles
+type UserProfileAPI interface {
+	QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error
+	QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error
+	SetAvatarURL(ctx context.Context, req *PerformSetAvatarURLRequest, res *PerformSetAvatarURLResponse) error
+	SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *struct{}) error
+}
+
+// UserRegisterAPI defines functions for registering accounts
+type UserRegisterAPI interface {
+	QueryNumericLocalpart(ctx context.Context, res *QueryNumericLocalpartResponse) error
+	QueryAccountAvailability(ctx context.Context, req *QueryAccountAvailabilityRequest, res *QueryAccountAvailabilityResponse) error
+	PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error
+	PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error
+}
+
+// UserAccountAPI defines functions for changing an account
+type UserAccountAPI interface {
+	PerformPasswordUpdate(ctx context.Context, req *PerformPasswordUpdateRequest, res *PerformPasswordUpdateResponse) error
+	PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error
+	QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error
+}
+
+// UserThreePIDAPI defines functions for 3PID
+type UserThreePIDAPI interface {
+	QueryLocalpartForThreePID(ctx context.Context, req *QueryLocalpartForThreePIDRequest, res *QueryLocalpartForThreePIDResponse) error
+	QueryThreePIDsForLocalpart(ctx context.Context, req *QueryThreePIDsForLocalpartRequest, res *QueryThreePIDsForLocalpartResponse) error
+	PerformForgetThreePID(ctx context.Context, req *PerformForgetThreePIDRequest, res *struct{}) error
+	PerformSaveThreePIDAssociation(ctx context.Context, req *PerformSaveThreePIDAssociationRequest, res *struct{}) error
+}
+
 type PerformKeyBackupRequest struct {
 	UserID       string
 	Version      string // optional if modifying a key backup
@@ -507,3 +536,55 @@ type Notification struct {
 	RoomID     string                        `json:"room_id"`     // Required.
 	TS         gomatrixserverlib.Timestamp   `json:"ts"`          // Required.
 }
+
+type PerformSetAvatarURLRequest struct {
+	Localpart, AvatarURL string
+}
+type PerformSetAvatarURLResponse struct{}
+
+type QueryNumericLocalpartResponse struct {
+	ID int64
+}
+
+type QueryAccountAvailabilityRequest struct {
+	Localpart string
+}
+
+type QueryAccountAvailabilityResponse struct {
+	Available bool
+}
+
+type QueryAccountByPasswordRequest struct {
+	Localpart, PlaintextPassword string
+}
+
+type QueryAccountByPasswordResponse struct {
+	Account *Account
+	Exists  bool
+}
+
+type PerformUpdateDisplayNameRequest struct {
+	Localpart, DisplayName string
+}
+
+type QueryLocalpartForThreePIDRequest struct {
+	ThreePID, Medium string
+}
+
+type QueryLocalpartForThreePIDResponse struct {
+	Localpart string
+}
+
+type QueryThreePIDsForLocalpartRequest struct {
+	Localpart string
+}
+
+type QueryThreePIDsForLocalpartResponse struct {
+	ThreePIDs []authtypes.ThreePID
+}
+
+type PerformForgetThreePIDRequest QueryLocalpartForThreePIDRequest
+
+type PerformSaveThreePIDAssociationRequest struct {
+	ThreePID, Localpart, Medium string
+}

userapi/api/api_trace.go

@@ -149,6 +149,60 @@ func (t *UserInternalAPITrace) QueryNotifications(ctx context.Context, req *Quer
 	return err
 }
 
+func (t *UserInternalAPITrace) SetAvatarURL(ctx context.Context, req *PerformSetAvatarURLRequest, res *PerformSetAvatarURLResponse) error {
+	err := t.Impl.SetAvatarURL(ctx, req, res)
+	util.GetLogger(ctx).Infof("SetAvatarURL req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
+func (t *UserInternalAPITrace) QueryNumericLocalpart(ctx context.Context, res *QueryNumericLocalpartResponse) error {
+	err := t.Impl.QueryNumericLocalpart(ctx, res)
+	util.GetLogger(ctx).Infof("QueryNumericLocalpart req= res=%+v", js(res))
+	return err
+}
+
+func (t *UserInternalAPITrace) QueryAccountAvailability(ctx context.Context, req *QueryAccountAvailabilityRequest, res *QueryAccountAvailabilityResponse) error {
+	err := t.Impl.QueryAccountAvailability(ctx, req, res)
+	util.GetLogger(ctx).Infof("QueryAccountAvailability req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
+func (t *UserInternalAPITrace) SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *struct{}) error {
+	err := t.Impl.SetDisplayName(ctx, req, res)
+	util.GetLogger(ctx).Infof("SetDisplayName req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
+func (t *UserInternalAPITrace) QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error {
+	err := t.Impl.QueryAccountByPassword(ctx, req, res)
+	util.GetLogger(ctx).Infof("QueryAccountByPassword req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
+func (t *UserInternalAPITrace) QueryLocalpartForThreePID(ctx context.Context, req *QueryLocalpartForThreePIDRequest, res *QueryLocalpartForThreePIDResponse) error {
+	err := t.Impl.QueryLocalpartForThreePID(ctx, req, res)
+	util.GetLogger(ctx).Infof("QueryLocalpartForThreePID req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
+func (t *UserInternalAPITrace) QueryThreePIDsForLocalpart(ctx context.Context, req *QueryThreePIDsForLocalpartRequest, res *QueryThreePIDsForLocalpartResponse) error {
+	err := t.Impl.QueryThreePIDsForLocalpart(ctx, req, res)
+	util.GetLogger(ctx).Infof("QueryThreePIDsForLocalpart req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
+func (t *UserInternalAPITrace) PerformForgetThreePID(ctx context.Context, req *PerformForgetThreePIDRequest, res *struct{}) error {
+	err := t.Impl.PerformForgetThreePID(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformForgetThreePID req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
+func (t *UserInternalAPITrace) PerformSaveThreePIDAssociation(ctx context.Context, req *PerformSaveThreePIDAssociationRequest, res *struct{}) error {
+	err := t.Impl.PerformSaveThreePIDAssociation(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformSaveThreePIDAssociation req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
 func js(thing interface{}) string {
 	b, err := json.Marshal(thing)
 	if err != nil {

userapi/internal/api.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/crypto/bcrypt"
 
 	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
@@ -761,4 +762,71 @@ func (a *UserInternalAPI) QueryPushRules(ctx context.Context, req *api.QueryPush
 	return nil
 }
 
+func (a *UserInternalAPI) SetAvatarURL(ctx context.Context, req *api.PerformSetAvatarURLRequest, res *api.PerformSetAvatarURLResponse) error {
+	return a.DB.SetAvatarURL(ctx, req.Localpart, req.AvatarURL)
+}
+
+func (a *UserInternalAPI) QueryNumericLocalpart(ctx context.Context, res *api.QueryNumericLocalpartResponse) error {
+	id, err := a.DB.GetNewNumericLocalpart(ctx)
+	if err != nil {
+		return err
+	}
+	res.ID = id
+	return nil
+}
+
+func (a *UserInternalAPI) QueryAccountAvailability(ctx context.Context, req *api.QueryAccountAvailabilityRequest, res *api.QueryAccountAvailabilityResponse) error {
+	_, err := a.DB.CheckAccountAvailability(ctx, req.Localpart)
+	if err == sql.ErrNoRows {
+		res.Available = true
+		return nil
+	}
+	res.Available = false
+	return err
+}
+
+func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error {
+	acc, err := a.DB.GetAccountByPassword(ctx, req.Localpart, req.PlaintextPassword)
+	switch err {
+	case sql.ErrNoRows: // user does not exist
+		return nil
+	case bcrypt.ErrMismatchedHashAndPassword: // user exists, but password doesn't match
+		return nil
+	default:
+		res.Exists = true
+		res.Account = acc
+		return nil
+	}
+}
+
+func (a *UserInternalAPI) SetDisplayName(ctx context.Context, req *api.PerformUpdateDisplayNameRequest, _ *struct{}) error {
+	return a.DB.SetDisplayName(ctx, req.Localpart, req.DisplayName)
+}
+
+func (a *UserInternalAPI) QueryLocalpartForThreePID(ctx context.Context, req *api.QueryLocalpartForThreePIDRequest, res *api.QueryLocalpartForThreePIDResponse) error {
+	localpart, err := a.DB.GetLocalpartForThreePID(ctx, req.ThreePID, req.Medium)
+	if err != nil {
+		return err
+	}
+	res.Localpart = localpart
+	return nil
+}
+
+func (a *UserInternalAPI) QueryThreePIDsForLocalpart(ctx context.Context, req *api.QueryThreePIDsForLocalpartRequest, res *api.QueryThreePIDsForLocalpartResponse) error {
+	r, err := a.DB.GetThreePIDsForLocalpart(ctx, req.Localpart)
+	if err != nil {
+		return err
+	}
+	res.ThreePIDs = r
+	return nil
+}
+
+func (a *UserInternalAPI) PerformForgetThreePID(ctx context.Context, req *api.PerformForgetThreePIDRequest, res *struct{}) error {
+	return a.DB.RemoveThreePIDAssociation(ctx, req.ThreePID, req.Medium)
+}
+
+func (a *UserInternalAPI) PerformSaveThreePIDAssociation(ctx context.Context, req *api.PerformSaveThreePIDAssociationRequest, res *struct{}) error {
+	return a.DB.SaveThreePIDAssociation(ctx, req.ThreePID, req.Localpart, req.Medium)
+}
+
 const pushRulesAccountDataType = "m.push_rules"

userapi/inthttp/client.go

@@ -28,30 +28,39 @@ import (
 const (
 	InputAccountDataPath = "/userapi/inputAccountData"
 
-	PerformDeviceCreationPath      = "/userapi/performDeviceCreation"
+	PerformDeviceCreationPath          = "/userapi/performDeviceCreation"
-	PerformAccountCreationPath     = "/userapi/performAccountCreation"
+	PerformAccountCreationPath         = "/userapi/performAccountCreation"
-	PerformPasswordUpdatePath      = "/userapi/performPasswordUpdate"
+	PerformPasswordUpdatePath          = "/userapi/performPasswordUpdate"
-	PerformDeviceDeletionPath      = "/userapi/performDeviceDeletion"
+	PerformDeviceDeletionPath          = "/userapi/performDeviceDeletion"
-	PerformLastSeenUpdatePath      = "/userapi/performLastSeenUpdate"
+	PerformLastSeenUpdatePath          = "/userapi/performLastSeenUpdate"
-	PerformDeviceUpdatePath        = "/userapi/performDeviceUpdate"
+	PerformDeviceUpdatePath            = "/userapi/performDeviceUpdate"
-	PerformAccountDeactivationPath = "/userapi/performAccountDeactivation"
+	PerformAccountDeactivationPath     = "/userapi/performAccountDeactivation"
-	PerformOpenIDTokenCreationPath = "/userapi/performOpenIDTokenCreation"
+	PerformOpenIDTokenCreationPath     = "/userapi/performOpenIDTokenCreation"
-	PerformKeyBackupPath           = "/userapi/performKeyBackup"
+	PerformKeyBackupPath               = "/userapi/performKeyBackup"
-	PerformPusherSetPath           = "/pushserver/performPusherSet"
+	PerformPusherSetPath               = "/pushserver/performPusherSet"
-	PerformPusherDeletionPath      = "/pushserver/performPusherDeletion"
+	PerformPusherDeletionPath          = "/pushserver/performPusherDeletion"
-	PerformPushRulesPutPath        = "/pushserver/performPushRulesPut"
+	PerformPushRulesPutPath            = "/pushserver/performPushRulesPut"
+	PerformSetAvatarURLPath            = "/userapi/performSetAvatarURL"
+	PerformSetDisplayNamePath          = "/userapi/performSetDisplayName"
+	PerformForgetThreePIDPath          = "/userapi/performForgetThreePID"
+	PerformSaveThreePIDAssociationPath = "/userapi/performSaveThreePIDAssociation"
 
-	QueryKeyBackupPath      = "/userapi/queryKeyBackup"
+	QueryKeyBackupPath             = "/userapi/queryKeyBackup"
-	QueryProfilePath        = "/userapi/queryProfile"
+	QueryProfilePath               = "/userapi/queryProfile"
-	QueryAccessTokenPath    = "/userapi/queryAccessToken"
+	QueryAccessTokenPath           = "/userapi/queryAccessToken"
-	QueryDevicesPath        = "/userapi/queryDevices"
+	QueryDevicesPath               = "/userapi/queryDevices"
-	QueryAccountDataPath    = "/userapi/queryAccountData"
+	QueryAccountDataPath           = "/userapi/queryAccountData"
-	QueryDeviceInfosPath    = "/userapi/queryDeviceInfos"
+	QueryDeviceInfosPath           = "/userapi/queryDeviceInfos"
-	QuerySearchProfilesPath = "/userapi/querySearchProfiles"
+	QuerySearchProfilesPath        = "/userapi/querySearchProfiles"
-	QueryOpenIDTokenPath    = "/userapi/queryOpenIDToken"
+	QueryOpenIDTokenPath           = "/userapi/queryOpenIDToken"
-	QueryPushersPath        = "/pushserver/queryPushers"
+	QueryPushersPath               = "/pushserver/queryPushers"
-	QueryPushRulesPath      = "/pushserver/queryPushRules"
+	QueryPushRulesPath             = "/pushserver/queryPushRules"
-	QueryNotificationsPath  = "/pushserver/queryNotifications"
+	QueryNotificationsPath         = "/pushserver/queryNotifications"
+	QueryNumericLocalpartPath      = "/userapi/queryNumericLocalpart"
+	QueryAccountAvailabilityPath   = "/userapi/queryAccountAvailability"
+	QueryAccountByPasswordPath     = "/userapi/queryAccountByPassword"
+	QueryLocalpartForThreePIDPath  = "/userapi/queryLocalpartForThreePID"
+	QueryThreePIDsForLocalpartPath = "/userapi/queryThreePIDsForLocalpart"
 )
 
 // NewUserAPIClient creates a UserInternalAPI implemented by talking to a HTTP POST API.
@@ -310,3 +319,75 @@ func (h *httpUserInternalAPI) QueryPushRules(ctx context.Context, req *api.Query
 	apiURL := h.apiURL + QueryPushRulesPath
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
 }
+
+func (h *httpUserInternalAPI) SetAvatarURL(ctx context.Context, req *api.PerformSetAvatarURLRequest, res *api.PerformSetAvatarURLResponse) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, PerformSetAvatarURLPath)
+	defer span.Finish()
+
+	apiURL := h.apiURL + PerformSetAvatarURLPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
+func (h *httpUserInternalAPI) QueryNumericLocalpart(ctx context.Context, res *api.QueryNumericLocalpartResponse) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, QueryNumericLocalpartPath)
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryNumericLocalpartPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, struct{}{}, res)
+}
+
+func (h *httpUserInternalAPI) QueryAccountAvailability(ctx context.Context, req *api.QueryAccountAvailabilityRequest, res *api.QueryAccountAvailabilityResponse) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, QueryAccountAvailabilityPath)
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryAccountAvailabilityPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
+func (h *httpUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, QueryAccountByPasswordPath)
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryAccountByPasswordPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
+func (h *httpUserInternalAPI) SetDisplayName(ctx context.Context, req *api.PerformUpdateDisplayNameRequest, res *struct{}) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, PerformSetDisplayNamePath)
+	defer span.Finish()
+
+	apiURL := h.apiURL + PerformSetDisplayNamePath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
+func (h *httpUserInternalAPI) QueryLocalpartForThreePID(ctx context.Context, req *api.QueryLocalpartForThreePIDRequest, res *api.QueryLocalpartForThreePIDResponse) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, QueryLocalpartForThreePIDPath)
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryLocalpartForThreePIDPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
+func (h *httpUserInternalAPI) QueryThreePIDsForLocalpart(ctx context.Context, req *api.QueryThreePIDsForLocalpartRequest, res *api.QueryThreePIDsForLocalpartResponse) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, QueryThreePIDsForLocalpartPath)
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryThreePIDsForLocalpartPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
+func (h *httpUserInternalAPI) PerformForgetThreePID(ctx context.Context, req *api.PerformForgetThreePIDRequest, res *struct{}) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, PerformForgetThreePIDPath)
+	defer span.Finish()
+
+	apiURL := h.apiURL + PerformForgetThreePIDPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
+func (h *httpUserInternalAPI) PerformSaveThreePIDAssociation(ctx context.Context, req *api.PerformSaveThreePIDAssociationRequest, res *struct{}) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, PerformSaveThreePIDAssociationPath)
+	defer span.Finish()
+
+	apiURL := h.apiURL + PerformSaveThreePIDAssociationPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}

userapi/inthttp/server.go

@@ -347,4 +347,101 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(PerformSetAvatarURLPath,
+		httputil.MakeInternalAPI("performSetAvatarURL", func(req *http.Request) util.JSONResponse {
+			request := api.PerformSetAvatarURLRequest{}
+			response := api.PerformSetAvatarURLResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := s.SetAvatarURL(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(QueryNumericLocalpartPath,
+		httputil.MakeInternalAPI("queryNumericLocalpart", func(req *http.Request) util.JSONResponse {
+			response := api.QueryNumericLocalpartResponse{}
+			if err := s.QueryNumericLocalpart(req.Context(), &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(QueryAccountByPasswordPath,
+		httputil.MakeInternalAPI("queryAccountByPassword", func(req *http.Request) util.JSONResponse {
+			request := api.QueryAccountByPasswordRequest{}
+			response := api.QueryAccountByPasswordResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := s.QueryAccountByPassword(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(PerformSetDisplayNamePath,
+		httputil.MakeInternalAPI("performSetDisplayName", func(req *http.Request) util.JSONResponse {
+			request := api.PerformUpdateDisplayNameRequest{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := s.SetDisplayName(req.Context(), &request, &struct{}{}); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &struct{}{}}
+		}),
+	)
+	internalAPIMux.Handle(QueryLocalpartForThreePIDPath,
+		httputil.MakeInternalAPI("queryLocalpartForThreePID", func(req *http.Request) util.JSONResponse {
+			request := api.QueryLocalpartForThreePIDRequest{}
+			response := api.QueryLocalpartForThreePIDResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := s.QueryLocalpartForThreePID(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(QueryThreePIDsForLocalpartPath,
+		httputil.MakeInternalAPI("queryThreePIDsForLocalpart", func(req *http.Request) util.JSONResponse {
+			request := api.QueryThreePIDsForLocalpartRequest{}
+			response := api.QueryThreePIDsForLocalpartResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := s.QueryThreePIDsForLocalpart(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(PerformForgetThreePIDPath,
+		httputil.MakeInternalAPI("performForgetThreePID", func(req *http.Request) util.JSONResponse {
+			request := api.PerformForgetThreePIDRequest{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := s.PerformForgetThreePID(req.Context(), &request, &struct{}{}); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &struct{}{}}
+		}),
+	)
+	internalAPIMux.Handle(PerformSaveThreePIDAssociationPath,
+		httputil.MakeInternalAPI("performSaveThreePIDAssociation", func(req *http.Request) util.JSONResponse {
+			request := api.PerformSaveThreePIDAssociationRequest{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := s.PerformSaveThreePIDAssociation(req.Context(), &request, &struct{}{}); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &struct{}{}}
+		}),
+	)
 }

userapi/storage/interface.go

@@ -24,12 +24,17 @@ import (
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 )
 
-type Database interface {
+type Profile interface {
-	GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error)
 	GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error)
+	SearchProfiles(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error)
 	SetPassword(ctx context.Context, localpart string, plaintextPassword string) error
 	SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error
 	SetDisplayName(ctx context.Context, localpart string, displayName string) error
+}
+
+type Database interface {
+	Profile
+	GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error)
 	// CreateAccount makes a new account with the given login name and password, and creates an empty profile
 	// for this account. If no password is supplied, the account will be a passwordless account. If the
 	// account already exists, it will return nil, ErrUserExists.
@@ -48,7 +53,6 @@ type Database interface {
 	GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error)
 	CheckAccountAvailability(ctx context.Context, localpart string) (bool, error)
 	GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error)
-	SearchProfiles(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error)
 	DeactivateAccount(ctx context.Context, localpart string) (err error)
 	CreateOpenIDToken(ctx context.Context, token, localpart string) (exp int64, err error)
 	GetOpenIDTokenAttributes(ctx context.Context, token string) (*api.OpenIDTokenAttributes, error)