Update uses

Signed-off-by: Skyler Mäntysaari <sm+git@skym.fi>

.dockerignore

@@ -1,2 +1,3 @@
 bin
-*.wasm
+*.wasm
+.git

.forgejo/workflows/docker.yml

@@ -1,59 +0,0 @@
-on:
-  push:
-    tags:
-      - 'v*'
-
-env:
-  GHCR_NAMESPACE: sigb.us
-  PLATFORMS: linux/amd64
-  FORGEJO_USER: signaryk
-
-jobs:
-  monolith:
-    name: Monolith image
-    runs-on: docker
-    container:
-      image: ghcr.io/catthehacker/ubuntu:act-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Get release tag & build flags
-        if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to sigb.us container registry
-        uses: docker/login-action@v3
-        with:
-          registry: git.sigb.us
-          username: ${{ env.FORGEJO_USER }}
-          password: ${{ secrets.FORGEJO_TOKEN }}
-
-      - name: Build main monolith image
-        id: docker_build_monolith
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ github.ref_name }}
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:devel
-
-      - name: Build release monolith image
-        if: github.event_name == 'release' # Only for GitHub releases
-        id: docker_build_monolith_release
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:stable
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ env.RELEASE_VERSION }}

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -17,6 +17,7 @@ see: https://www.matrix.org/security-disclosure-policy/
 ### Background information
 <!-- Please include versions of all software when known e.g database versions, docker versions, client versions -->
 - **Dendrite version or git SHA**:
+- **Monolith or Polylith?**:
 - **SQLite3 or Postgres?**:
 - **Running in Docker?**:
 - **`go version`**:
@@ -62,6 +63,6 @@ If you can identify any relevant log snippets from server logs, please include
 those (please be careful to remove any personal or private data). Please surround them with
 ``` (three backticks, on a line on their own), so that they are formatted legibly.
 
-Alternatively, please send logs to @kegan:matrix.org, @s7evink:matrix.org or @devonh:one.ems.host
+Alternatively, please send logs to @kegan:matrix.org or @neilalexander:matrix.org
 with a link to the respective Github issue, thanks!
 -->

.github/actions/collect-changes/action.yaml

@@ -0,0 +1,45 @@
+name: "Collect changes"
+description: "Collects and stores changed files/charts"
+
+outputs:
+  changesDetected:
+    description: "Whether or not changes to charts have been detected"
+    value: ${{ steps.filter.outputs.addedOrModified }}
+  addedOrModifiedFiles:
+    description: "A list of the files changed"
+    value: ${{ steps.filter.outputs.addedOrModified_files }}
+  addedOrModifiedCharts:
+    description: "A list of the charts changed"
+    value: ${{ steps.filter-charts.outputs.addedOrModified }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Collect changed files
+      uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        list-files: shell
+        filters: |
+          addedOrModified:
+            - added|modified: 'charts/*/**'
+
+    - name: Collect changed charts
+      if: |
+        steps.filter.outputs.addedOrModified == 'true'
+      id: filter-charts
+      shell: bash
+      run: |
+        CHARTS=()
+        PATHS=(${{ steps.filter.outputs.addedOrModified_files }})
+        # Get only the chart paths
+        for CHARTPATH in "${PATHS[@]}"
+        do
+          IFS='/' read -r -a path_parts <<< "${CHARTPATH}"
+          CHARTS+=("${path_parts[1]}/${path_parts[2]}")
+        done
+
+        # Remove duplicates
+        CHARTS=( `printf "%s\n" "${CHARTS[@]}" | sort -u` )
+        # Set output to changed charts
+        printf "::set-output name=addedOrModified::%s\n" "${CHARTS[*]}"

.github/codecov.yaml

@@ -1,20 +0,0 @@
-flag_management:
-  default_rules:
-    carryforward: true
-
-coverage:
-  status:
-    project:
-      default:
-        target: auto
-        threshold: 0.1%
-        base: auto
-        flags:
-          - unittests
-    patch:
-      default:
-        target: 75%
-        threshold: 0%
-        base: auto
-        flags:
-          - unittests

.github/scripts/check-releasenotes.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Check if release notes have been changed
+# Usage ./check-releasenotes.sh path
+
+# require yq
+command -v yq >/dev/null 2>&1 || {
+    printf >&2 "%s\n" "yq (https://github.com/mikefarah/yq) is not installed. Aborting."
+    exit 1
+}
+
+# Absolute path of repository
+repository=$(git rev-parse --show-toplevel)
+
+# Allow for a specific chart to be passed in as a argument
+if [ $# -ge 1 ] && [ -n "$1" ]; then
+    root="$1"
+    chart_file="${1}/Chart.yaml"
+    if [ ! -f "$chart_file" ]; then
+        printf >&2 "File %s\n does not exist.\n" "${chart_file}"
+        exit 1
+    fi
+
+    cd $root
+
+    if [ -z "$DEFAULT_BRANCH" ]; then
+      DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}')
+    fi
+
+    CURRENT=$(cat Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -)
+
+    if [ "$CURRENT" == "" ] || [ "$CURRENT" == "null" ]; then
+      printf >&2 "Changelog annotation has not been set in %s!\n" "$chart_file"
+      exit 1
+    fi
+
+    DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}')
+    ORIGINAL=$(git show origin/$DEFAULT_BRANCH:./Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -)
+
+    if [ "$CURRENT" == "$ORIGINAL" ]; then
+      printf >&2 "Changelog annotation has not been updated in %s!\n" "$chart_file"
+      exit 1
+    fi
+else
+    printf >&2 "%s\n" "No chart folder has been specified."
+    exit 1
+fi

.github/scripts/gen-helm-docs.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -eu
+
+# Generate helm-docs for Helm charts
+# Usage ./gen-helm-docs.sh [stable/incubator] [chart]
+
+# require helm-docs
+command -v helm-docs >/dev/null 2>&1 || {
+    echo >&2 "helm-docs (https://github.com/k8s-at-home/helm-docs) is not installed. Aborting."
+    exit 1
+}
+
+# Absolute path of repository
+repository=$(git rev-parse --show-toplevel)
+
+# Templates to copy into each chart directory
+readme_template="${repository}/hack/templates/README.md.gotmpl"
+readme_config_template="${repository}/hack/templates/README_CONFIG.md.gotmpl"
+
+# Gather all charts using the common library, excluding common-test
+charts=$(find "${repository}" -name "Chart.yaml")
+
+# Allow for a specific chart to be passed in as a argument
+if [ $# -ge 1 ] && [ -n "$1" ] && [ -n "$2" ]; then
+    charts="${repository}/charts/$1/$2/Chart.yaml"
+    root="$(dirname "${charts}")"
+    if [ ! -f "$charts" ]; then
+        echo "File ${charts} does not exist."
+        exit 1
+    fi
+else
+    root="${repository}/charts/stable"
+fi
+
+for chart in ${charts}; do
+    chart_directory="$(dirname "${chart}")"
+    echo "-] Copying templates to ${chart_directory}"
+    # Copy CONFIG template to each Chart directory, do not overwrite if exists
+    cp -n "${readme_config_template}" "${chart_directory}" || true
+done
+
+# Run helm-docs for charts using the common library and the common library itself
+helm-docs \
+    --ignore-file="${repository}/.helmdocsignore" \
+    --template-files="${readme_template}" \
+    --template-files="$(basename "${readme_config_template}")" \
+    --chart-search-root="${root}"

.github/scripts/renovate-releasenotes.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import typer
+
+from git import Repo
+from loguru import logger
+from pathlib import Path
+
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
+from ruamel.yaml.scalarstring import LiteralScalarString
+from typing import List
+
+app = typer.Typer(add_completion=False)
+
+
+def _setup_logging(debug):
+    """
+    Setup the log formatter for this script
+    """
+
+    log_level = "INFO"
+    if debug:
+        log_level = "DEBUG"
+
+    logger.remove()
+    logger.add(
+        sys.stdout,
+        colorize=True,
+        format="<level>{message}</level>",
+        level=log_level,
+    )
+
+
+@app.command()
+def main(
+    chart_folders: List[Path] = typer.Argument(
+        ..., help="Folders containing the chart to process"),
+    check_branch: str = typer.Option(
+        None, help="The branch to compare against."),
+    chart_base_folder: Path = typer.Option(
+        "charts", help="The base folder where the charts reside."),
+    debug: bool = False,
+):
+    _setup_logging(debug)
+
+    git_repository = Repo(search_parent_directories=True)
+
+    if check_branch:
+        logger.info(f"Trying to find branch {check_branch}...")
+        branch = next(
+            (ref for ref in git_repository.remotes.origin.refs if ref.name == check_branch),
+            None
+        )
+    else:
+        logger.info(f"Trying to determine default branch...")
+        branch = next(
+            (ref for ref in git_repository.remotes.origin.refs if ref.name == "origin/HEAD"),
+            None
+        )
+
+    if not branch:
+        logger.error(
+            f"Could not find branch {check_branch} to compare against.")
+        raise typer.Exit(1)
+
+    logger.info(f"Comparing against branch {branch}")
+
+    for chart_folder in chart_folders:
+        chart_folder = chart_base_folder.joinpath(chart_folder)
+        if not chart_folder.is_dir():
+            logger.error(f"Could not find folder {str(chart_folder)}")
+            raise typer.Exit(1)
+
+        chart_metadata_file = chart_folder.joinpath('Chart.yaml')
+
+        if not chart_metadata_file.is_file():
+            logger.error(f"Could not find file {str(chart_metadata_file)}")
+            raise typer.Exit(1)
+
+        logger.info(f"Updating changelog annotation for chart {chart_folder}")
+
+        yaml = YAML(typ=['rt', 'string'])
+        yaml.indent(mapping=2, sequence=4, offset=2)
+        yaml.explicit_start = True
+        yaml.preserve_quotes = True
+        yaml.width = 4096
+
+        old_chart_metadata = yaml.load(
+            git_repository.git.show(f"{branch}:{chart_metadata_file}")
+        )
+        new_chart_metadata = yaml.load(chart_metadata_file.read_text())
+
+        try:
+            old_chart_dependencies = old_chart_metadata["dependencies"]
+        except KeyError:
+            old_chart_dependencies = []
+
+        try:
+            new_chart_dependencies = new_chart_metadata["dependencies"]
+        except KeyError:
+            new_chart_dependencies = []
+
+        annotations = []
+        for dependency in new_chart_dependencies:
+            old_dep = None
+            if "alias" in dependency.keys():
+                old_dep = next(
+                    (old_dep for old_dep in old_chart_dependencies if "alias" in old_dep.keys(
+                    ) and old_dep["alias"] == dependency["alias"]),
+                    None
+                )
+            else:
+                old_dep = next(
+                    (old_dep for old_dep in old_chart_dependencies if old_dep["name"] == dependency["name"]),
+                    None
+                )
+
+            add_annotation = False
+            if old_dep:
+                if dependency["version"] != old_dep["version"]:
+                    add_annotation = True
+            else:
+                add_annotation = True
+
+            if add_annotation:
+                if "alias" in dependency.keys():
+                    annotations.append({
+                        "kind": "changed",
+                        "description": f"Upgraded `{dependency['name']}` chart dependency to version {dependency['version']} for alias '{dependency['alias']}'"
+                    })
+                else:
+                    annotations.append({
+                        "kind": "changed",
+                        "description": f"Upgraded `{dependency['name']}` chart dependency to version {dependency['version']}"
+                    })
+
+        if annotations:
+            annotations = YAML(typ=['rt', 'string']
+                               ).dump_to_string(annotations)
+
+            if not "annotations" in new_chart_metadata:
+                new_chart_metadata["annotations"] = CommentedMap()
+
+            new_chart_metadata["annotations"]["artifacthub.io/changes"] = LiteralScalarString(
+                annotations)
+            yaml.dump(new_chart_metadata, chart_metadata_file)
+
+
+if __name__ == "__main__":
+    app()

.github/scripts/requirements.txt

@@ -0,0 +1,5 @@
+GitPython==3.1.27
+loguru==0.6.0
+ruamel.yaml==0.17.21
+ruamel.yaml.string==0.1.0
+typer==0.6.1

.github/workflows/dendrite.yml

@@ -2,20 +2,20 @@ name: Dendrite
 
 on:
   push:
+    paths-ignore:
+      - 'charts/**' # ignore helm chart changes
     branches:
       - main
-    paths:
-      - '**.go'                 # only execute on changes to go files
-      - 'go.sum'                # or dependency updates
-      - '.github/workflows/**'  # or workflow changes
   pull_request:
-    paths:
-      - '**.go'
-      - 'go.sum'                # or dependency updates
-      - '.github/workflows/**'
+    paths-ignore:
+      - 'charts/**' # ignore helm chart changes
   release:
+    paths-ignore:
+      - 'charts/**' # ignore helm chart changes
     types: [published]
   workflow_dispatch:
+    paths-ignore:
+      - 'charts/**' # ignore helm chart changes
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -28,12 +28,12 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ false }} # disable for now
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v3
         with:
-          go-version: "stable"
+          go-version: 1.18
           cache: true
 
       - name: Install Node
@@ -41,7 +41,7 @@ jobs:
         with:
           node-version: 14
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v3
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -66,20 +66,18 @@ jobs:
     name: Linting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - name: Install libolm
-        run: sudo apt-get install libolm-dev libolm3
+      - uses: actions/checkout@v3
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v3
         with:
-          go-version: "stable"
+          go-version: 1.18
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
 
   # run go test with different go versions
   test:
     timeout-minutes: 10
-    name: Unit tests
+    name: Unit tests (Go ${{ matrix.go }})
     runs-on: ubuntu-latest
     # Service containers to run with `container-job`
     services:
@@ -101,29 +99,31 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.19"]
     steps:
-      - uses: actions/checkout@v4
-      - name: Install libolm
-        run: sudo apt-get install libolm-dev libolm3
+      - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v3
         with:
-          go-version: "stable"
-      - uses: actions/cache@v4
+          go-version: ${{ matrix.go }}
+      - uses: actions/cache@v3
         # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go${{ matrix.go }}-unit-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go-stable-unit-
+            ${{ runner.os }}-go${{ matrix.go }}-unit-
       - name: Set up gotestfmt
         uses: gotesttools/gotestfmt-action@v2
         with:
           # Optional: pass GITHUB_TOKEN to avoid rate limiting.
           token: ${{ secrets.GITHUB_TOKEN }}
-      - run: go test -json -v ./... 2>&1 | gotestfmt -hide all
+      - run: go test -json -v ./... 2>&1 | gotestfmt
         env:
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
@@ -138,22 +138,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        go: ["1.18", "1.19"]
         goos: ["linux"]
         goarch: ["amd64", "386"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v3
         with:
-          go-version: "stable"
-      - uses: actions/cache@v4
+          go-version: ${{ matrix.go }}
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
+            key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
       - name: Install dependencies x86
         if: ${{ matrix.goarch == '386' }}
         run: sudo apt update && sudo apt-get install -y gcc-multilib
@@ -171,22 +172,23 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+        go: ["1.18", "1.19"]
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
-      - uses: actions/checkout@v4
-      - name: Setup Go
-        uses: actions/setup-go@v4
+      - uses: actions/checkout@v3
+      - name: Setup Go ${{ matrix.go }}
+        uses: actions/setup-go@v3
         with:
-          go-version: "stable"
-      - uses: actions/cache@v4
+          go-version: ${{ matrix.go }}
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
+            key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
       - env:
@@ -212,7 +214,7 @@ jobs:
   integration:
     timeout-minutes: 20
     needs: initial-tests-done
-    name: Integration tests
+    name: Integration tests (Go ${{ matrix.go }})
     runs-on: ubuntu-latest
     # Service containers to run with `container-job`
     services:
@@ -234,39 +236,39 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.19"]
     steps:
-      - uses: actions/checkout@v4
-      - name: Install libolm
-        run: sudo apt-get install libolm-dev libolm3
+      - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v3
         with:
-          go-version: "stable"
+          go-version: ${{ matrix.go }}
       - name: Set up gotestfmt
         uses: gotesttools/gotestfmt-action@v2
         with:
           # Optional: pass GITHUB_TOKEN to avoid rate limiting.
           token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/cache@v4
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go-stable-test-race-
-      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all
+            ${{ runner.os }}-go${{ matrix.go }}-test-race-
+      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
         env:
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: dendrite
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v3
         with:
           flags: unittests
-          fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
 
   # run database upgrade tests
   upgrade_test:
@@ -275,22 +277,12 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v3
         with:
-          go-version: "stable"
+          go-version: "1.18"
           cache: true
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-upgrade-test-
-      - name: Docker version
-        run: docker version
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
       - name: Test upgrade (PostgreSQL)
@@ -305,22 +297,12 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v3
         with:
-          go-version: "stable"
+          go-version: "1.18"
           cache: true
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-upgrade-direct-test-
-      - name: Docker version
-        run: docker version
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
       - name: Test upgrade (PostgreSQL)
@@ -343,22 +325,33 @@ jobs:
           - label: SQLite Cgo
             cgo: 1
 
+          - label: SQLite native, full HTTP APIs
+            api: full-http
+
+          - label: SQLite Cgo, full HTTP APIs
+            api: full-http
+            cgo: 1
+
           - label: PostgreSQL
             postgres: postgres
 
+          - label: PostgreSQL, full HTTP APIs
+            postgres: postgres
+            api: full-http
     container:
-      image: matrixdotorg/sytest-dendrite
+      image: matrixdotorg/sytest-dendrite:latest
       volumes:
         - ${{ github.workspace }}:/src
         - /root/.cache/go-build:/github/home/.cache/go-build
         - /root/.cache/go-mod:/gopath/pkg/mod
       env:
         POSTGRES: ${{ matrix.postgres && 1}}
+        API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
         CGO_ENABLED: ${{ matrix.cgo && 1 }}
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/cache@v4
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -381,7 +374,7 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
@@ -405,9 +398,22 @@ jobs:
           - label: SQLite Cgo
             cgo: 1
 
+          - label: SQLite native, full HTTP APIs
+            api: full-http
+            cgo: 0
+
+          - label: SQLite Cgo, full HTTP APIs
+            api: full-http
+            cgo: 1
+
           - label: PostgreSQL
             postgres: Postgres
             cgo: 0
+
+          - label: PostgreSQL, full HTTP APIs
+            postgres: Postgres
+            api: full-http
+            cgo: 0
     steps:
       # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
       # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
@@ -420,9 +426,9 @@ jobs:
         # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v4 for dendrite
-        uses: actions/checkout@v4
+          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+      - name: Run actions/checkout@v3 for dendrite
+        uses: actions/checkout@v3
         with:
           path: dendrite
 
@@ -449,7 +455,7 @@ jobs:
             (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
           done
       # Build initial Dendrite image
-      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
+      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
         working-directory: dendrite
         env:
           DOCKER_BUILDKIT: 1
@@ -457,12 +463,12 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
+          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
         shell: bash
         name: Run Complement Tests
         env:
-          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
-          COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
+          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
+          API: ${{ matrix.api && 1 }}
         working-directory: complement
 
   integration-tests-done:

.github/workflows/docker.yml

@@ -4,6 +4,8 @@ name: "Docker"
 
 on:
   release: # A GitHub release was published
+    paths-ignore:
+      - 'charts/**' # ignore helm chart changes
     types: [published]
   workflow_dispatch: # A build was manually requested
   workflow_call: # Another pipeline called us
@@ -27,22 +29,26 @@ jobs:
       security-events: write # To upload Trivy sarif files
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -53,9 +59,11 @@ jobs:
         id: docker_build_monolith
         uses: docker/build-push-action@v3
         with:
-          cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache
-          cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           context: .
+          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
+          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -70,6 +78,8 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
+          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
+          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -90,6 +100,86 @@ jobs:
         with:
           sarif_file: "trivy-results.sarif"
 
+  polylith:
+    name: Polylith image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      security-events: write # To upload Trivy sarif files
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Get release tag & build flags
+        if: github.event_name == 'release' # Only for GitHub releases
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to GitHub Containers
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build main polylith image
+        if: github.ref_name == 'main'
+        id: docker_build_polylith
+        uses: docker/build-push-action@v3
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: .
+          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
+          target: polylith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
+
+      - name: Build release polylith image
+        if: github.event_name == 'release' # Only for GitHub releases
+        id: docker_build_polylith_release
+        uses: docker/build-push-action@v3
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: .
+          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
+          target: polylith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results.sarif"
+
   demo-pinecone:
     name: Pinecone demo image
     runs-on: ubuntu-latest
@@ -98,22 +188,26 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -127,6 +221,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
+          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -142,6 +237,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
+          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -159,22 +255,26 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -188,6 +288,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
+          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -203,6 +304,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
+          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true

.github/workflows/gh-pages.yml

@@ -1,52 +0,0 @@
-# Sample workflow for building and deploying a Jekyll site to GitHub Pages
-name: Deploy GitHub Pages dependencies preinstalled
-
-on:
-  # Runs on pushes targeting the default branch
-  push:
-    branches: ["gh-pages"]
-    paths:
-      - 'docs/**' # only execute if we have docs changes
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow one concurrent deployment
-concurrency:
-  group: "pages"
-  cancel-in-progress: true
-
-jobs:
-  # Build job
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Setup Pages
-        uses: actions/configure-pages@v2
-      - name: Build with Jekyll
-        uses: actions/jekyll-build-pages@v1
-        with:
-          source: ./docs
-          destination: ./_site
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
-
-  # Deployment job
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v1

.github/workflows/helm-charts-changelog.yaml

@@ -0,0 +1,81 @@
+name: "Charts: Update README"
+
+on:
+  workflow_call:
+    inputs:
+      modifiedCharts:
+        required: true
+        type: string
+      isRenovatePR:
+        required: true
+        type: string
+    outputs:
+      commitHash:
+        description: "The most recent commit hash at the end of this workflow"
+        value: ${{ jobs.generate-changelog.outputs.commitHash }}
+
+jobs:
+  validate-changelog:
+    name: Validate changelog
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Check changelog annotations
+        if: inputs.isRenovatePR != 'true'
+        run: |
+          CHARTS=(${{ inputs.modifiedCharts }})
+          for i in "${CHARTS[@]}"
+          do
+              IFS='/' read -r -a chart_parts <<< "$i"
+              ./.github/scripts/check-releasenotes.sh "charts/${chart_parts[0]}/${chart_parts[1]}"
+              echo ""
+          done
+
+  generate-changelog:
+    name: Generate changelog annotations
+    runs-on: ubuntu-latest
+    needs:
+      - validate-changelog
+    outputs:
+      commitHash: ${{ steps.save-commit-hash.outputs.commit_hash }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Annotate Charts.yaml for Renovate PR's
+        if: inputs.isRenovatePR == 'true'
+        env:
+          CHECK_BRANCH: "origin/${{ github.event.repository.default_branch }}"
+        run: |
+          pip install -r ./.github/scripts/requirements.txt
+          ./.github/scripts/renovate-releasenotes.py --check-branch "$CHECK_BRANCH" ${{ inputs.modifiedCharts }}
+
+      - name: Create commit
+        id: create-commit
+        if: inputs.isRenovatePR == 'true'
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          file_pattern: charts/**/
+          commit_message: "chore: Auto-update chart metadata"
+          commit_user_name: ${{ github.actor }}
+          commit_user_email: ${{ github.actor }}@users.noreply.github.com
+
+      - name: Save commit hash
+        id: save-commit-hash
+        run: |
+          if [ "${{ steps.create-commit.outputs.changes_detected || 'unknown' }}" == "true" ]; then
+            echo '::set-output name=commit_hash::${{ steps.create-commit.outputs.commit_hash }}'
+          else
+            echo "::set-output name=commit_hash::${GITHUB_SHA}"
+          fi

.github/workflows/helm-charts-lint.yaml

@@ -0,0 +1,54 @@
+name: "Charts: Lint"
+
+on:
+  workflow_call:
+    inputs:
+      checkoutCommit:
+        required: true
+        type: string
+      chartChangesDetected:
+        required: true
+        type: string
+
+jobs:
+  lint:
+    name: Lint charts
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.checkoutCommit }}
+
+      - name: Install Kubernetes tools
+        uses: yokawasa/action-setup-kube-tools@v0.8.2
+        with:
+          setup-tools: |
+            helmv3
+          helm: "3.8.0"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.3.0
+
+      - name: Collect changes
+        id: list-changed
+        if: inputs.chartChangesDetected == 'true'
+        run: |
+          EXCLUDED=$(yq eval -o=json '.excluded-charts // []' .github/ct-lint.yaml)
+          CHARTS=$(ct list-changed --config .github/ct-lint.yaml)
+          CHARTS_JSON=$(echo "${CHARTS}" | jq -R -s -c 'split("\n")[:-1]')
+          OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED}, \"all\": ${CHARTS_JSON}}" | jq -c '.all-.excluded')
+          echo ::set-output name=charts::${OUTPUT_JSON}
+          if [[ $(echo ${OUTPUT_JSON} | jq -c '. | length') -gt 0 ]]; then
+            echo "::set-output name=detected::true"
+          fi
+
+      - name: Run chart-testing (lint)
+        id: lint
+        if: steps.list-changed.outputs.detected == 'true'
+        run: ct lint --config .github/ct-lint.yaml

.github/workflows/helm-charts-test.yaml

@@ -0,0 +1,134 @@
+name: "Charts: Test"
+
+on:
+  workflow_call:
+    inputs:
+      checkoutCommit:
+        required: true
+        type: string
+      chartChangesDetected:
+        required: true
+        type: string
+
+jobs:
+  unit-test:
+    name: Run unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.checkoutCommit }}
+
+      - name: Install Kubernetes tools
+        uses: yokawasa/action-setup-kube-tools@v0.8.2
+        with:
+          setup-tools: |
+            helmv3
+          helm: "3.8.0"
+
+      - name: Install Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+
+      - name: Install dependencies
+        env:
+          RUBYJQ_USE_SYSTEM_LIBRARIES: 1
+        run: |
+          sudo apt-get update
+          sudo apt-get install libjq-dev
+          bundle install
+
+      - name: Run tests
+        run: |
+          bundle exec m -r ./test/
+
+  generate-install-matrix:
+    name: Generate matrix for install
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: |
+        {
+          "chart": ${{ steps.list-changed.outputs.charts }}
+        }
+      detected: ${{ steps.list-changed.outputs.detected }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.checkoutCommit }}
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.3.0
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        if: inputs.chartChangesDetected == 'true'
+        run: |
+          EXCLUDED=$(yq eval -o=json '.excluded-charts // []' .github/ct-install.yaml)
+          CHARTS=$(ct list-changed --config .github/ct-install.yaml)
+          CHARTS_JSON=$(echo "${CHARTS}" | jq -R -s -c 'split("\n")[:-1]')
+          OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED}, \"all\": ${CHARTS_JSON}}" | jq -c '.all-.excluded')
+          echo ::set-output name=charts::${OUTPUT_JSON}
+          if [[ $(echo ${OUTPUT_JSON} | jq -c '. | length') -gt 0 ]]; then
+            echo "::set-output name=detected::true"
+          fi
+
+  install-charts:
+    needs:
+      - generate-install-matrix
+    if: needs.generate-install-matrix.outputs.detected == 'true'
+    name: Install charts
+    strategy:
+      matrix: ${{ fromJson(needs.generate-install-matrix.outputs.matrix) }}
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.checkoutCommit }}
+
+      - name: Install Kubernetes tools
+        uses: yokawasa/action-setup-kube-tools@v0.8.2
+        with:
+          setup-tools: |
+            helmv3
+          helm: "3.6.3"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.3.0
+
+      - name: Create k3d cluster
+        uses: nolar/setup-k3d-k3s@v1
+        with:
+          version: v1.19
+
+      - name: Remove node taints
+        run: |
+          kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true
+
+      - name: Run chart-testing (install)
+        run: ct install --config .github/ct-install.yaml --charts ${{ matrix.chart }}
+
+  # Summarize matrix https://github.community/t/status-check-for-a-matrix-jobs/127354/7
+  install_success:
+    needs:
+      - generate-install-matrix
+      - install-charts
+    if: |
+      always()
+    name: Install successful
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check install matrix status
+        if: ${{ (needs.generate-install-matrix.outputs.detected == 'true') && (needs.install-charts.result != 'success') }}
+        run: exit 1

.github/workflows/helm-pr-metadata.yaml

@@ -0,0 +1,60 @@
+name: "Pull Request: Get metadata"
+
+on:
+  workflow_call:
+    outputs:
+      isRenovatePR:
+        description: "Is the PR coming from Renovate?"
+        value: ${{ jobs.pr-metadata.outputs.isRenovatePR }}
+      isFork:
+        description: "Is the PR coming from a forked repo?"
+        value: ${{ jobs.pr-metadata.outputs.isFork }}
+      addedOrModified:
+        description: "Does the PR contain any changes?"
+        value: ${{ jobs.pr-changes.outputs.addedOrModified }}
+      addedOrModifiedFiles:
+        description: "A list of the files changed in this PR"
+        value: ${{ jobs.pr-changes.outputs.addedOrModifiedFiles }}
+      addedOrModifiedCharts:
+        description: "A list of the charts changed in this PR"
+        value: ${{ jobs.pr-changes.outputs.addedOrModifiedCharts }}
+
+jobs:
+  pr-metadata:
+    name: Collect PR metadata
+    runs-on: ubuntu-latest
+    outputs:
+      isRenovatePR: ${{ startsWith(steps.branch-name.outputs.current_branch, 'renovate/') }}
+      isFork: ${{ github.event.pull_request.head.repo.full_name != github.repository }}
+    steps:
+      - name: Get branch name
+        id: branch-name
+        uses: tj-actions/branch-names@v5.4
+
+      - name: Save PR data to file
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+
+      - name: Store pr data in artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: pr_metadata
+          path: ./pr_number.txt
+          retention-days: 5
+
+  pr-changes:
+    name: Collect PR changes
+    runs-on: ubuntu-latest
+    outputs:
+      addedOrModified: ${{ steps.collect-changes.outputs.changesDetected }}
+      addedOrModifiedFiles: ${{ steps.collect-changes.outputs.addedOrModifiedFiles }}
+      addedOrModifiedCharts: ${{ steps.collect-changes.outputs.addedOrModifiedCharts }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Collect changes
+        id: collect-changes
+        uses: ./.github/actions/collect-changes

.github/workflows/helm-pre-commit-check.yaml

@@ -0,0 +1,21 @@
+name: "Pre-commit consistency check"
+
+on:
+  workflow_call:
+    inputs:
+      modifiedFiles:
+        required: true
+        type: string
+
+jobs:
+  pre-commit-check:
+    name: Run pre-commit checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Run against changes
+        uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: --files ${{ inputs.modifiedFiles }}

.github/workflows/helm.yml

@@ -1,41 +1,56 @@
-name: Release Charts
+name: "Pull Request: Validate"
 
 on:
-  push:
+  pull_request:
+    paths:
+      - 'charts/**' # only execute if we have helm chart changes
     branches:
       - main
-    paths:
-      - 'helm/**' # only execute if we have helm chart changes
-  workflow_dispatch:
+    types:
+      - opened
+      - edited
+      - reopened
+      - ready_for_review
+      - synchronize
+
+concurrency:
+  group: ${{ github.head_ref }}-pr-validate
+  cancel-in-progress: true
 
 jobs:
-  release:
-    # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions
-    # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
-    permissions:
-      contents: write
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
+  pr-metadata:
+    uses: S7evinK/dendrite/.github/workflows/helm-pr-metadata.yaml@main
 
-      - name: Configure Git
-        run: |
-          git config user.name "$GITHUB_ACTOR"
-          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+  pre-commit-check:
+    uses: S7evinK/dendrite/.github/workflows/helm-pre-commit-check.yaml@main
+    needs:
+      - pr-metadata
+    with:
+      modifiedFiles: ${{ needs.pr-metadata.outputs.addedOrModifiedFiles }}
 
-      - name: Install Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.10.0
+  charts-changelog:
+    uses: S7evinK/dendrite/.github/workflows/helm-charts-changelog.yaml@main
+    needs:
+      - pr-metadata
+      - pre-commit-check
+    with:
+      isRenovatePR: ${{ needs.pr-metadata.outputs.isRenovatePR }}
+      modifiedCharts: ${{ needs.pr-metadata.outputs.addedOrModifiedCharts }}
 
-      - name: Run chart-releaser
-        uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release
-        env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-        with:
-          config: helm/cr.yaml
-          charts_dir: helm/
-          mark_as_latest: false
+  charts-lint:
+    uses: S7evinK/dendrite/.github/workflows/helm-charts-lint.yaml@main
+    needs:
+      - pr-metadata
+      - charts-changelog
+    with:
+      checkoutCommit: ${{ needs.charts-changelog.outputs.commitHash }}
+      chartChangesDetected: ${{ needs.pr-metadata.outputs.addedOrModified }}
+
+  charts-test:
+    uses: S7evinK/dendrite/.github/workflows/helm-charts-test.yaml@main
+    needs:
+      - pr-metadata
+      - charts-changelog
+    with:
+      checkoutCommit: ${{ needs.charts-changelog.outputs.commitHash }}
+      chartChangesDetected: ${{ needs.pr-metadata.outputs.addedOrModified }}

.github/workflows/k8s.yml

@@ -1,91 +0,0 @@
-name: k8s
-
-on:
-  push:
-    branches: ["main"]
-    paths:
-      - 'helm/**' # only execute if we have helm chart changes
-  pull_request:
-    branches: ["main"]
-    paths:
-      - 'helm/**'
-
-jobs:
-  lint:
-    name: Lint Helm chart
-    runs-on: ubuntu-latest
-    outputs:
-      changed: ${{ steps.list-changed.outputs.changed }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - uses: azure/setup-helm@v3
-        with:
-          version: v3.10.0
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.11
-          check-latest: true
-      - uses: helm/chart-testing-action@v2.3.1
-      - name: Get changed status
-        id: list-changed
-        run: |
-          changed=$(ct list-changed --config helm/ct.yaml --target-branch ${{ github.event.repository.default_branch }})
-          if [[ -n "$changed" ]]; then
-              echo "::set-output name=changed::true"
-          fi
-
-      - name: Run lint
-        run: ct lint --config helm/ct.yaml
-
-  # only bother to run if lint step reports a change to the helm chart
-  install:
-    needs:
-      - lint
-    if: ${{ needs.lint.outputs.changed == 'true' }}
-    name: Install Helm charts
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ inputs.checkoutCommit }}
-      - name: Install Kubernetes tools
-        uses: yokawasa/action-setup-kube-tools@v0.8.2
-        with:
-          setup-tools: |
-            helmv3
-          helm: "3.10.3"
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.3.1
-      - name: Create k3d cluster
-        uses: nolar/setup-k3d-k3s@v1
-        with:
-          version: v1.28
-      - name: Remove node taints
-        run: |
-          kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true
-      - name: Run chart-testing (install)
-        run: ct install --config helm/ct.yaml
-
-      # Install the chart using helm directly and test with create-account
-      - name: Install chart
-        run: |
-          helm install --values helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml dendrite helm/dendrite
-      - name: Wait for Postgres and Dendrite to be up
-        run: |
-          kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=postgresql || kubectl get pods -A
-          kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=dendrite || kubectl get pods -A
-          kubectl get pods -A
-          kubectl get services
-          kubectl get ingress
-          kubectl logs -l app.kubernetes.io/name=dendrite
-      - name: Run create account
-        run: |
-          podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name)
-          kubectl exec "${podName}" -- /usr/bin/create-account -username alice -password somerandompassword

.github/workflows/schedules.yaml

@@ -10,26 +10,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check_date: # https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where
-    runs-on: ubuntu-latest
-    name: Check latest commit
-    outputs:
-      should_run: ${{ steps.should_run.outputs.should_run }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: print latest_commit
-        run: echo ${{ github.sha }}
-
-      - id: should_run
-        continue-on-error: true
-        name: check latest commit is less than a day
-        if: ${{ github.event_name == 'schedule' }}
-        run: test -z $(git rev-list  --after="24 hours"  ${{ github.sha }}) && echo "::set-output name=should_run::false"
-
   # run Sytest in different variations
   sytest:
-    needs: check_date
-    if: ${{ needs.check_date.outputs.should_run != 'false' }}
     timeout-minutes: 60
     name: "Sytest (${{ matrix.label }})"
     runs-on: ubuntu-latest
@@ -37,13 +19,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - label: SQLite native
+          - label: SQLite
 
-          - label: SQLite Cgo
-            cgo: 1
+          - label: SQLite, full HTTP APIs
+            api: full-http
 
           - label: PostgreSQL
             postgres: postgres
+
+          - label: PostgreSQL, full HTTP APIs
+            postgres: postgres
+            api: full-http
     container:
       image: matrixdotorg/sytest-dendrite:latest
       volumes:
@@ -52,12 +38,12 @@ jobs:
         - /root/.cache/go-mod:/gopath/pkg/mod
       env:
         POSTGRES: ${{ matrix.postgres && 1}}
+        API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
         RACE_DETECTION: 1
-        COVER: 1
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/cache@v4
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -80,243 +66,10 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }})
+          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
           path: |
             /logs/results.tap
             /logs/**/*.log*
-            /logs/**/covdatafiles/**
-
-  sytest-coverage:
-    timeout-minutes: 5
-    name: "Sytest Coverage"
-    runs-on: ubuntu-latest
-    needs: [ sytest, check_date ] # only run once Sytest is done and there was a commit
-    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 'stable'
-          cache: true
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
-      - name: Collect coverage
-        run: |
-          go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov
-          grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov
-          go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        with:
-          files: ./final.cov
-          flags: sytest
-          fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
-
-  # run Complement
-  complement:
-    needs: check_date
-    if: ${{ needs.check_date.outputs.should_run != 'false' }}
-    name: "Complement (${{ matrix.label }})"
-    timeout-minutes: 60
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - label: SQLite native
-            cgo: 0
-
-          - label: SQLite Cgo
-            cgo: 1
-
-          - label: PostgreSQL
-            postgres: Postgres
-            cgo: 0
-    steps:
-      # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          echo "~/go/bin" >> $GITHUB_PATH
-      - name: "Install Complement Dependencies"
-        # We don't need to install Go because it is included on the Ubuntu 20.04 image:
-        # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v4 for dendrite
-        uses: actions/checkout@v4
-        with:
-          path: dendrite
-
-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to main.
-      - name: Checkout complement
-        shell: bash
-        run: |
-          mkdir -p complement
-          # Attempt to use the version of complement which best matches the current
-          # build. Depending on whether this is a PR or release, etc. we need to
-          # use different fallbacks.
-          #
-          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
-          #    for pull requests, otherwise GITHUB_REF).
-          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
-          #    (GITHUB_BASE_REF for pull requests).
-          # 3. Use the default complement branch ("master").
-          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
-            # Skip empty branch names and merge commits.
-            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
-              continue
-            fi
-            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
-          done
-      # Build initial Dendrite image
-      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
-        working-directory: dendrite
-        env:
-          DOCKER_BUILDKIT: 1
-
-      - name: Create post test script
-        run: |
-          cat <<EOF > /tmp/posttest.sh
-          #!/bin/bash
-          mkdir -p /tmp/Complement/logs/\$2/\$1/
-          docker cp \$1:/tmp/covdatafiles/. /tmp/Complement/logs/\$2/\$1/
-          EOF
-          
-          chmod +x /tmp/posttest.sh
-      # Run Complement
-      - run: |
-          set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
-        shell: bash
-        name: Run Complement Tests
-        env:
-          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
-          COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
-          COMPLEMENT_DENDRITE_COVER: 1
-          COMPLEMENT_POST_TEST_SCRIPT: /tmp/posttest.sh
-        working-directory: complement
-
-      - name: Upload Complement logs
-        uses: actions/upload-artifact@v4
-        if: ${{ always() }}
-        with:
-          name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }})
-          path: |
-            /tmp/Complement/logs/**
-
-  complement-coverage:
-    timeout-minutes: 5
-    name: "Complement Coverage"
-    runs-on: ubuntu-latest
-    needs: [ complement, check_date ] # only run once Complements is done and there was a commit
-    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 'stable'
-          cache: true
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
-      - name: Collect coverage
-        run: |
-          go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov
-          grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
-          go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        with:
-          files: ./final.cov
-          flags: complement
-          fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }} # required
-
-  element-web:
-    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
-    timeout-minutes: 120
-    runs-on: ubuntu-latest
-    steps:
-      - uses: tecolicom/actions-use-apt-tools@v1
-        with:
-          # Our test suite includes some screenshot tests with unusual diacritics, which are
-          # supposed to be covered by STIXGeneral.
-          tools: fonts-stix
-      - uses: actions/checkout@v4
-        with:
-          repository: matrix-org/matrix-react-sdk
-      - uses: actions/setup-node@v3
-        with:
-          cache: 'yarn'
-      - name: Fetch layered build
-        run: scripts/ci/layered.sh
-      - name: Copy config
-        run: cp element.io/develop/config.json config.json
-        working-directory: ./element-web
-      - name: Build
-        env:
-          CI_PACKAGE: true
-          NODE_OPTIONS: "--openssl-legacy-provider"
-        run: yarn build
-        working-directory: ./element-web
-      - name: Edit Test Config
-        run: |
-          sed -i '/HOMESERVER/c\        HOMESERVER: "dendrite",' cypress.config.ts
-      - name: "Run cypress tests"
-        uses: cypress-io/github-action@v4.1.1
-        with:
-          browser: chrome
-          start: npx serve -p 8080 ./element-web/webapp
-          wait-on: 'http://localhost:8080'
-        env:
-          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
-          TMPDIR: ${{ runner.temp }}
-
-  element-web-pinecone:
-    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
-    timeout-minutes: 120
-    runs-on: ubuntu-latest
-    steps:
-      - uses: tecolicom/actions-use-apt-tools@v1
-        with:
-          # Our test suite includes some screenshot tests with unusual diacritics, which are
-          # supposed to be covered by STIXGeneral.
-          tools: fonts-stix
-      - uses: actions/checkout@v4
-        with:
-          repository: matrix-org/matrix-react-sdk
-      - uses: actions/setup-node@v3
-        with:
-          cache: 'yarn'
-      - name: Fetch layered build
-        run: scripts/ci/layered.sh
-      - name: Copy config
-        run: cp element.io/develop/config.json config.json
-        working-directory: ./element-web
-      - name: Build
-        env:
-          CI_PACKAGE: true
-          NODE_OPTIONS: "--openssl-legacy-provider"
-        run: yarn build
-        working-directory: ./element-web
-      - name: Edit Test Config
-        run: |
-          sed -i '/HOMESERVER/c\        HOMESERVER: "dendritePinecone",' cypress.config.ts
-      - name: "Run cypress tests"
-        uses: cypress-io/github-action@v4.1.1
-        with:
-          browser: chrome
-          start: npx serve -p 8080 ./element-web/webapp
-          wait-on: 'http://localhost:8080'
-        env:
-          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
-          TMPDIR: ${{ runner.temp }}

.gitignore

@@ -5,7 +5,6 @@
 
 # Allow GitHub config
 !.github
-!.forgejo
 
 # Downloads
 /.downloads
@@ -57,7 +56,6 @@ dendrite.yaml
 
 # Database files
 *.db
-*.db-journal
 
 # Log files
 *.log*
@@ -75,10 +73,3 @@ complement/
 docs/_site
 
 media_store/
-build
-
-# golang workspaces
-go.work*
-
-# helm chart
-helm/dendrite/charts/

.golangci.yml

@@ -6,7 +6,7 @@ run:
   concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  timeout: 5m
+  deadline: 30m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
@@ -18,6 +18,24 @@ run:
   #build-tags:
   #  - mytag
 
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs:
+    - bin
+    - docs
+
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  skip-files:
+    - ".*\\.md$"
+    - ".*\\.sh$"
+    - "^cmd/syncserver-integration-tests/testdata.go$"
+
   # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
   # If invoked with -mod=readonly, the go command is disallowed from the implicit
   # automatic updating of go.mod described above. Instead, it fails when any changes
@@ -32,8 +50,7 @@ run:
 # output configuration options
 output:
   # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
-  formats:
-    - format: colored-line-number
+  format: colored-line-number
 
   # print lines of code with issue, default is true
   print-issued-lines: true
@@ -62,8 +79,9 @@ linters-settings:
     # see https://github.com/kisielk/errcheck#excluding-functions for details
     #exclude: /path/to/file.txt
   govet:
-    enable:
-      - shadow
+    # report about shadowed variables
+    check-shadowing: true
+
     # settings per analyzer
     settings:
       printf: # analyzer name, run `go tool vet help` to see all analyzers
@@ -161,7 +179,9 @@ linters-settings:
 
 linters:
   enable:
+    - deadcode
     - errcheck
+    - goconst
     - gocyclo
     - goimports # Does everything gofmt does
     - gosimple
@@ -171,8 +191,10 @@ linters:
     - misspell # Check code comments, whereas misspell in CI checks *.md files
     - nakedret
     - staticcheck
+    - structcheck
     - unparam
     - unused
+    - varcheck
   enable-all: false
   disable:
     - bodyclose
@@ -192,31 +214,12 @@ linters:
     - stylecheck
     - typecheck # Should turn back on soon
     - unconvert # Should turn back on soon
-    - goconst # Slightly annoying, as it reports "issues" in SQL statements
   disable-all: false
   presets:
   fast: false
 
 
 issues:
-  # which files to skip: they will be analyzed, but issues from them
-  # won't be reported. Default value is empty list, but there is
-  # no need to include all autogenerated files, we confidently recognize
-  # autogenerated files. If it's not please let us know.
-  exclude-files:
-    - ".*\\.md$"
-    - ".*\\.sh$"
-    - "^cmd/syncserver-integration-tests/testdata.go$"
-
-  # which dirs to skip: they won't be analyzed;
-  # can use regexp here: generated.*, regexp is applied on full path;
-  # default value is empty list, but next dirs are always skipped independently
-  # from this option's value:
-  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  exclude-dirs:
-    - bin
-    - docs
-
   # List of regexps of issue texts to exclude, empty list by default.
   # But independently from this option we use default exclude patterns,
   # it can be disabled by `exclude-use-default: false`. To list all

CHANGES.md

@@ -1,260 +1,5 @@
 # Changelog
 
-## Dendrite 0.13.7 (2024-04-09)
-
-### Fixes
-
-- Fixed an issue where the displayname/avatar of an invited user was replaced with the inviter's details
-- Improved server startup performance by avoiding unnecessary room ACL queries
-  - This change reduces memory footprint as it caches ACL regex patterns once instead of for each room
-  - Unnecessary Relay related queries have been removed. **Note**: To use relays, you now need to explicitly enable them using the `federation_api.enable_relays` config
-- Fixed space summaries over federation
-- Improved usage of external NATS JetStream by reusing existing connections instead of opening new ones unnecessarily
-
-### Features
-
-- Modernized Appservices (contributed by [tulir](https://github.com/tulir))
-- Added event reporting with Synapse Admin endpoints for querying them
-- Updated dependencies
-
-## Dendrite 0.13.6 (2024-01-26)
-
-Upgrading to this version is **highly** recommended, as it contains several QoL improvements.
-
-### Fixes
-
-- Use `AckExplicitPolicy` for JetStream consumers, so messages don't pile up in NATS
-- A rare panic when assigning a state key NID has been fixed
-- A rare panic when checking powerlevels has been fixed
-- Notary keys requests for all keys now work correctly
-- Spec compliance:
-  - Return `M_INVALID_PARAM` when querying room aliases
-  - Handle empty `from` parameter when requesting `/messages`
-  - Add CORP headers on media endpoints
-  - Remove `aliases` from `/publicRooms` responses
-  - Allow `+` in MXIDs (Contributed by [RosstheRoss](https://github.com/RosstheRoss))
-- Fixes membership transitions from `knock` to `join` in `knock_restricted` rooms
-- Incremental syncs now batch querying events (Contributed by [recht](https://github.com/recht))
-- Move `/joined_members` back to the clientAPI/roomserver, which should make bridges happier again
-- Backfilling from other servers now only uses at max 100 events instead of potentially thousands
-
-## Dendrite 0.13.5 (2023-12-12)
-
-Upgrading to this version is **highly** recommended, as it fixes several long-standing bugs in
-our CanonicalJSON implementation.
-
-### Fixes
-
-- Convert unicode escapes to lowercase (gomatrixserverlib)
-- Fix canonical json utf-16 surrogate pair detection logic (gomatrixserverlib)
-- Handle negative zero and exponential numbers in Canonical JSON verification (gomatrixserverlib)
-- Avoid logging unnecessary messages when unable to fetch server keys if multiple fetchers are used (gomatrixserverlib)
-- Issues around the device list updater have been fixed, which should ensure that there are always
-  workers available to process incoming device list updates.
-- A panic in the `/hierarchy` endpoints used for spaces has been fixed (client-server and server-server API)
-- Fixes around the way we handle database transactions (including a potential connection leak)
-- ACLs are now updated when received as outliers
-- A race condition, which could lead to bridges instantly leaving a room after joining it, between the SyncAPI and
-  Appservices has been fixed
-
-### Features
-
-- **Appservice login is now supported!**
-- Users can now kick themselves (used by some bridges)
-
-## Dendrite 0.13.4 (2023-10-25)
-
-Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution
-algorithm.
-
-### Fixes:
-
-- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also 
-  avoids spamming the logs when shutting down.)
-- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset"
-  out of rooms and other missing state events due to calculating the wrong state.
-- A bug when setting notifications from Element Android has been fixed by implementing MSC3987
-
-### Features
-
-- Updated dependencies
-  - Internal NATS Server has been updated from v2.9.19 to v2.9.23
-
-## Dendrite 0.13.3 (2023-09-28)
-
-### Fixes:
-
-- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir))
-- Invitations are now correctly pushed to devices
-- A bug which could result in the corruption of `m.direct` account data has been fixed 
-
-### Features
-
-- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response
-- Room version 11 is now supported
-- Clients can request the `federation` `event_format` when creating filters
-- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
-
-### Other
-
-- Dendrite now requires Go 1.20 if building from source
-
-## Dendrite 0.13.2 (2023-08-23)
-
-### Fixes:
-
-- Migrations in SQLite are now prepared on the correct context (transaction or database)
-- The `InputRoomEvent` stream now has a maximum age of 24h, which should help with slow start up times of NATS JetStream (contributed by [neilalexander](https://github.com/neilalexander))
-- Event size checks are more in line with Synapse
-- Requests to `/messages` have been optimized, possibly reducing database round trips
-- Re-add the revision of Dendrite when building from source (Note: This only works if git is installed)
-- Getting local members to notify has been optimized, which should significantly reduce memory allocation and cache usage
-- When getting queried about user profiles, we now return HTTP404 if the user/profiles does not exist
-- Background federated joins should now be fixed and not timeout after a short time
-- Database connections are now correctly re-used
-- Restored the old behavior of the `/purgeRoom` admin endpoint (does not evacuate the room before purging)
-- Don't expose information about the system when trying to download files that don't exist
-
-### Features
-
-- Further improvements and fixes for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
-  - Lookup correct prev events in the sync API
-  - Populate `prev_sender` correctly in the sync API
-  - Event federation should work better
-- Added new `dendrite_up` Prometheus metric, containing the version of Dendrite
-- Space summaries ([MSC2946](https://github.com/matrix-org/matrix-spec-proposals/pull/2946)) have been moved from MSC to being natively supported
-- For easier issue investigation, logs for application services now contain the application service ID (contributed by [maxberger](https://github.com/maxberger))
-- The default room version to use when creating rooms can now be configured using `room_server.default_room_version`
-
-## Dendrite 0.13.1 (2023-07-06)
-
-This releases fixes a long-standing "off-by-one" error which could result in state resets. Upgrading to this version is **highly** recommended.
-
-When deduplicating state events, we were checking if the event in question was already in a state snapshot. If it was in a previous state snapshot, we would 
-then remove it from the list of events to store. If this happened, we were, unfortunately, skipping the next event to check. This resulted in 
-events getting stored in state snapshots where they may not be needed. When we now compared two of those state snapshots, one of them
-contained the skipped event, while the other didn't. This difference possibly shouldn't exist, resulting in unexpected state resets and explains
-reports of missing state events as well.
-
-Rooms where a state reset occurred earlier should, hopefully, reconcile over time.
-
-### Fixes:
-
-- A long-standing "off-by-one" error has been fixed, which could result in state resets
-- Roomserver Prometheus Metrics are available again
-
-### Features
-
-- Updated dependencies
-  - Internal NATS Server has been updated from v2.9.15 to v2.9.19
-
-## Dendrite 0.13.0 (2023-06-30)
-
-### Features
-
-- Results in responses to `/search` now highlight words more accurately and not only the search terms as before
-- Support for connecting to appservices listening on unix sockets has been added (contributed by [cyberb](https://github.com/cyberb))
-- Admin APIs for token authenticated registration have been added (contributed by [santhoshivan23](https://github.com/santhoshivan23))
-- Initial support for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
-  - This is **highly experimental**, things like changing usernames/avatars, inviting users, upgrading rooms isn't working
-
-### Fixes
-
-- `m.upload.size` is now optional, finally allowing uploads with unlimited file size
-- A bug while resolving server names has been fixed (contributed by [anton-molyboha](https://github.com/anton-molyboha))
-- Application services should only receive one invitation instead of 2 (or worse), which could result in state resets previously
-- Several admin endpoints are now using `POST` instead of `GET`
-- `/delete_devices` now uses user-interactive authentication
-- Several "membership" (e.g `/kick`, `/ban`) endpoints are using less heavy database queries to check if the user is allowed to perform this action
-- `/3pid` endpoints are now available on `/v3` instead of the `/unstable` prefix
-- Upgrading rooms ignores state events of other users, which could result in failed upgrades before
-- Uploading key backups with a wrong version now returns `M_WRONG_ROOM_KEYS_VERSION`
-- A potential state reset when joining the same room multiple times in short sequence has been fixed
-- A bug where we returned the full event as `redacted_because` in redaction events has been fixed
-- The `displayname` and `avatar_url` can now be set to empty strings
-- Unsafe hotserving of files has been fixed (contributed by [joshqou](https://github.com/joshqou))
-- Joining new rooms would potentially return "redacted" events, due to history visibility not being set correctly, this could result in events being rejected 
-- Backfilling resulting in `unsuported room version ''` should now be solved
-
-### Other
-
-- Huge refactoring of Dendrite and gomatrixserverlib
-
-## Dendrite 0.12.0 (2023-03-13)
-
-### Features
-
-- The userapi and keyserver have been merged (no actions needed regarding the database)
-- The internal NATS JetStream server is now using logrus for logging (contributed by [dvob](https://github.com/dvob))
-- The roomserver database has been refactored to have separate interfaces when working with rooms and events. Also includes increased usage of the cache to avoid database round trips. (database is unchanged)
-- The pinecone demo now shuts down more cleanly
-- The Helm chart now has the ability to deploy a Grafana chart as well (contributed by [genofire](https://github.com/genofire))
-- Support for listening on unix sockets has been added (contributed by [cyberb](https://github.com/cyberb))
-- The internal NATS server was updated to v2.9.15
-- Initial support for `runtime/trace` has been added, to further track down long-running tasks
-
-### Fixes
-
-- The `session_id` is now correctly set when using SQLite
-- An issue where device keys could be removed if a device ID is reused has been fixed
-- A possible DoS issue related to relations has been fixed (reported by [sleroq](https://github.com/sleroq))
-- When backfilling events, errors are now ignored if we still could fetch events
-
-### Other
-
-- **⚠️ DEPRECATION: Polylith/HTTP API mode has been removed**
-- The default endpoint to report usages stats to has been updated
-
-## Dendrite 0.11.1 (2023-02-10)
-
-**⚠️ DEPRECATION WARNING: This is the last release to have polylith and HTTP API mode. Future releases are monolith only.**
-
-### Features
-
-* Dendrite can now be compiled against Go 1.20
-* Initial store and forward support has been added
-* A landing page showing that Dendrite is running has been added (contributed by [LukasLJL](https://github.com/LukasLJL))
-
-### Fixes
-
-- `/sync` is now using significantly less database round trips when using Postgres, resulting in faster initial syncs, allowing larger accounts to login again
-- Many under the hood pinecone improvements
-- Publishing rooms is now possible again
-
-## Dendrite 0.11.0 (2023-01-20)
-
-The last three missing federation API Sytests have been fixed - bringing us to 100% server-server Synapse parity, with client-server parity at 93% 🎉
-
-### Features
-
-* Added `/_dendrite/admin/purgeRoom/{roomID}` to clean up the database
-* The default room version was updated to 10 (contributed by [FSG-Cat](https://github.com/FSG-Cat))
-
-### Fixes
-
-* An oversight in the `create-config` binary, which now correctly sets the media path if specified (contributed by [BieHDC](https://github.com/BieHDC))
-* The Helm chart now uses the `$.Chart.AppVersion` as the default image version to pull, with the possibility to override it (contributed by [genofire](https://github.com/genofire))
-
-## Dendrite 0.10.9 (2023-01-17)
-
-### Features
-
-* Stale device lists are now cleaned up on startup, removing entries for users the server doesn't share a room with anymore
-* Dendrite now has its own Helm chart
-* Guest access is now handled correctly (disallow joins, kick guests on revocation of guest access, as well as over federation)
-
-### Fixes
-
-* Push rules have seen several tweaks and fixes, which should, for example, fix notifications for `m.read_receipts`
-* Outgoing presence will now correctly be sent to newly joined hosts
-* Fixes the `/_dendrite/admin/resetPassword/{userID}` admin endpoint to use the correct variable
-* Federated backfilling for medium/large rooms has been fixed
-* `/login` causing wrong device list updates has been resolved
-* `/sync` should now return the correct room summary heroes
-* The default config options for `recaptcha_sitekey_class` and `recaptcha_form_field` are now set correctly 
-* `/messages` now omits empty `state` to be more spec compliant (contributed by [handlerug](https://github.com/handlerug))
-* `/sync` has been optimised to only query state events for history visibility if they are really needed
-
 ## Dendrite 0.10.8 (2022-11-29)
 
 ### Features

Dockerfile

@@ -3,9 +3,8 @@
 #
 # base installs required dependencies and runs go mod download to cache dependencies
 #
-# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine3.18 AS base
-RUN apk --update --no-cache add bash build-base curl git
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base
+RUN apk --update --no-cache add bash build-base curl
 
 #
 # build creates all needed binaries
@@ -14,6 +13,7 @@ FROM --platform=${BUILDPLATFORM} base AS build
 WORKDIR /src
 ARG TARGETOS
 ARG TARGETARCH
+ARG FLAGS
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
@@ -21,29 +21,72 @@ RUN --mount=target=. \
     GOARCH="$TARGETARCH" \
     GOOS="linux" \
     CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
-    go build -v -trimpath -o /out/ ./cmd/...
-
+    go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
 
 #
-# Builds the Dendrite image containing all required binaries
+# The dendrite base image
 #
-FROM alpine:latest
-RUN apk --update --no-cache add curl
-LABEL org.opencontainers.image.title="Dendrite"
+FROM alpine:latest AS dendrite-base
 LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
 LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
 LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
 LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
 
-COPY --from=build /out/create-account /usr/bin/create-account
-COPY --from=build /out/generate-config /usr/bin/generate-config
-COPY --from=build /out/generate-keys /usr/bin/generate-keys
-COPY --from=build /out/dendrite /usr/bin/dendrite
+#
+# Builds the polylith image and only contains the polylith binary
+#
+FROM dendrite-base AS polylith
+LABEL org.opencontainers.image.title="Dendrite (Polylith)"
+
+COPY --from=build /out/dendrite-polylith-multi /usr/bin/
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
 
-ENTRYPOINT ["/usr/bin/dendrite"]
+ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
+
+#
+# Builds the monolith image and contains all required binaries
+#
+FROM dendrite-base AS monolith
+LABEL org.opencontainers.image.title="Dendrite (Monolith)"
+
+COPY --from=build /out/create-account /usr/bin/create-account
+COPY --from=build /out/generate-config /usr/bin/generate-config
+COPY --from=build /out/generate-keys /usr/bin/generate-keys
+COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
 EXPOSE 8008 8448
 
+#
+# Builds the Complement image, used for integration tests
+#
+FROM base AS complement
+LABEL org.opencontainers.image.title="Dendrite (Complement)"
+RUN apk add --no-cache sqlite openssl ca-certificates
+
+COPY --from=build /out/generate-config /usr/bin/generate-config
+COPY --from=build /out/generate-keys /usr/bin/generate-keys
+COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
+
+WORKDIR /dendrite
+RUN /usr/bin/generate-keys --private-key matrix_key.pem && \
+    mkdir /ca && \
+    openssl genrsa -out /ca/ca.key 2048 && \
+    openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt
+
+ENV SERVER_NAME=localhost
+ENV API=0
+EXPOSE 8008 8448
+
+# At runtime, generate TLS cert based on the CA now mounted at /ca
+# At runtime, replace the SERVER_NAME with what we are told
+CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \
+    /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
+    cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
+    /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

README.md

@@ -13,7 +13,7 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ
 
 Dendrite is **beta** software, which means:
 
-- Dendrite is ready for early adopters. We recommend running Dendrite with a PostgreSQL database.
+- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database.
 - Dendrite has periodic releases. We intend to release new versions as we fix bugs and land significant features.
 - Dendrite supports database schema upgrades between releases. This means you should never lose your messages when upgrading Dendrite.
 
@@ -21,9 +21,10 @@ This does not mean:
 
 - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
 - Dendrite is feature-complete. There may be client or federation APIs that are not implemented.
-- Dendrite is ready for massive homeserver deployments. There is no high-availability/clustering support.
+- Dendrite is ready for massive homeserver deployments. There is no sharding of microservices (although it is possible to run them on separate machines) and there is no high-availability/clustering support.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
+In the future, we will be able to scale up to gigantic servers (equivalent to `matrix.org`) via polylith mode.
 
 If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or join us in:
 
@@ -36,7 +37,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for
 more information on requirements.
 
-To build Dendrite, you will need Go 1.20 or later.
+To build Dendrite, you will need Go 1.18 or later.
 
 For a usable federating Dendrite deployment, you will also need:
 
@@ -47,7 +48,7 @@ For a usable federating Dendrite deployment, you will also need:
 Also recommended are:
 
 - A PostgreSQL database engine, which will perform better than SQLite with many users and/or larger rooms
-- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/dendrite-sample.conf)
+- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/master/docs/nginx/monolith-sample.conf)
 
 The [Federation Tester](https://federationtester.matrix.org) can be used to verify your deployment.
 
@@ -60,7 +61,7 @@ The following instructions are enough to get Dendrite started as a non-federatin
 ```bash
 $ git clone https://github.com/matrix-org/dendrite
 $ cd dendrite
-$ go build -o bin/ ./cmd/...
+$ ./build.sh
 
 # Generate a Matrix signing key for federation (required)
 $ ./bin/generate-keys --private-key matrix_key.pem
@@ -71,10 +72,10 @@ $ ./bin/generate-keys --tls-cert server.crt --tls-key server.key
 
 # Copy and modify the config file - you'll need to set a server name and paths to the keys
 # at the very least, along with setting up the database connection strings.
-$ cp dendrite-sample.yaml dendrite.yaml
+$ cp dendrite-sample.monolith.yaml dendrite.yaml
 
 # Build and run the server:
-$ ./bin/dendrite --tls-cert server.crt --tls-key server.key --config dendrite.yaml
+$ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
 
 # Create an user account (add -admin for an admin user).
 # Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
@@ -85,9 +86,9 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l
 
 ## Progress
 
-We use a script called "Are We Synapse Yet" which checks Sytest compliance rates. Sytest is a black-box homeserver
+We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of January 2023, we have 100% server-server parity with Synapse, and the client-server parity is at 93% , though check
+updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs).
 

appservice/api/query.go

@@ -22,6 +22,8 @@ import (
 	"encoding/json"
 	"errors"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
@@ -82,17 +84,9 @@ type UserIDExistsResponse struct {
 }
 
 const (
-	ASProtocolLegacyPath        = "/_matrix/app/unstable/thirdparty/protocol/"
-	ASUserLegacyPath            = "/_matrix/app/unstable/thirdparty/user"
-	ASLocationLegacyPath        = "/_matrix/app/unstable/thirdparty/location"
-	ASRoomAliasExistsLegacyPath = "/rooms/"
-	ASUserExistsLegacyPath      = "/users/"
-
-	ASProtocolPath        = "/_matrix/app/v1/thirdparty/protocol/"
-	ASUserPath            = "/_matrix/app/v1/thirdparty/user"
-	ASLocationPath        = "/_matrix/app/v1/thirdparty/location"
-	ASRoomAliasExistsPath = "/_matrix/app/v1/rooms/"
-	ASUserExistsPath      = "/_matrix/app/v1/users/"
+	ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
+	ASUserPath     = "/_matrix/app/unstable/thirdparty/user"
+	ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
 )
 
 type ProtocolRequest struct {
@@ -156,10 +150,6 @@ type ASLocationResponse struct {
 	Fields   json.RawMessage `json:"fields"`
 }
 
-// ErrProfileNotExists is returned when trying to lookup a user's profile that
-// doesn't exist locally.
-var ErrProfileNotExists = errors.New("no known profile for given user ID")
-
 // RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
 // TODO: Remove this, it's called from federationapi and clientapi but is a pure function
@@ -167,11 +157,25 @@ func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
 	asAPI AppServiceInternalAPI,
-	profileAPI userapi.ProfileAPI,
+	profileAPI userapi.ClientUserAPI,
 ) (*authtypes.Profile, error) {
+	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return nil, err
+	}
+
 	// Try to query the user from the local database
-	profile, err := profileAPI.QueryProfile(ctx, userID)
-	if err == nil {
+	res := &userapi.QueryProfileResponse{}
+	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
+	if err != nil {
+		return nil, err
+	}
+	profile := &authtypes.Profile{
+		Localpart:   localpart,
+		DisplayName: res.DisplayName,
+		AvatarURL:   res.AvatarURL,
+	}
+	if res.UserExists {
 		return profile, nil
 	}
 
@@ -184,15 +188,19 @@ func RetrieveUserProfile(
 
 	// If no user exists, return
 	if !userResp.UserIDExists {
-		return nil, ErrProfileNotExists
+		return nil, errors.New("no known profile for given user ID")
 	}
 
 	// Try to query the user from the local database again
-	profile, err = profileAPI.QueryProfile(ctx, userID)
+	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
 	if err != nil {
 		return nil, err
 	}
 
 	// profile should not be nil at this point
-	return profile, nil
+	return &authtypes.Profile{
+		Localpart:   localpart,
+		DisplayName: res.DisplayName,
+		AvatarURL:   res.AvatarURL,
+	}, nil
 }

appservice/appservice.go

@@ -16,49 +16,66 @@ package appservice
 
 import (
 	"context"
+	"crypto/tls"
+	"net/http"
 	"sync"
+	"time"
 
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/appservice/consumers"
+	"github.com/matrix-org/dendrite/appservice/inthttp"
 	"github.com/matrix-org/dendrite/appservice/query"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
+// AddInternalRoutes registers HTTP handlers for internal API calls
+func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI) {
+	inthttp.AddRoutes(queryAPI, router)
+}
+
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	processContext *process.ProcessContext,
-	cfg *config.Dendrite,
-	natsInstance *jetstream.NATSInstance,
-	userAPI userapi.AppserviceUserAPI,
+	base *base.BaseDendrite,
+	userAPI userapi.UserInternalAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceInternalAPI {
-
+	client := &http.Client{
+		Timeout: time.Second * 30,
+		Transport: &http.Transport{
+			DisableKeepAlives: true,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation,
+			},
+			Proxy: http.ProxyFromEnvironment,
+		},
+	}
 	// Create appserivce query API with an HTTP client that will be used for all
 	// outbound and inbound requests (inbound only for the internal API)
 	appserviceQueryAPI := &query.AppServiceQueryAPI{
-		Cfg:           &cfg.AppServiceAPI,
+		HTTPClient:    client,
+		Cfg:           &base.Cfg.AppServiceAPI,
 		ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{},
 		CacheMu:       sync.Mutex{},
 	}
 
-	if len(cfg.Derived.ApplicationServices) == 0 {
+	if len(base.Cfg.Derived.ApplicationServices) == 0 {
 		return appserviceQueryAPI
 	}
 
 	// Wrap application services in a type that relates the application service and
 	// a sync.Cond object that can be used to notify workers when there are new
 	// events to be sent out.
-	for _, appservice := range cfg.Derived.ApplicationServices {
+	for _, appservice := range base.Cfg.Derived.ApplicationServices {
 		// Create bot account for this AS if it doesn't already exist
-		if err := generateAppServiceAccount(userAPI, appservice, cfg.Global.ServerName); err != nil {
+		if err := generateAppServiceAccount(userAPI, appservice, base.Cfg.Global.ServerName); err != nil {
 			logrus.WithFields(logrus.Fields{
 				"appservice": appservice.ID,
 			}).WithError(err).Panicf("failed to generate bot account for appservice")
@@ -67,10 +84,10 @@ func NewInternalAPI(
 
 	// Only consume if we actually have ASes to track, else we'll just chew cycles needlessly.
 	// We can't add ASes at runtime so this is safe to do.
-	js, _ := natsInstance.Prepare(processContext, &cfg.Global.JetStream)
+	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 	consumer := consumers.NewOutputRoomEventConsumer(
-		processContext, &cfg.AppServiceAPI,
-		js, rsAPI,
+		base.ProcessContext, &base.Cfg.AppServiceAPI,
+		client, js, rsAPI,
 	)
 	if err := consumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
@@ -85,7 +102,7 @@ func NewInternalAPI(
 func generateAppServiceAccount(
 	userAPI userapi.AppserviceUserAPI,
 	as config.ApplicationService,
-	serverName spec.ServerName,
+	serverName gomatrixserverlib.ServerName,
 ) error {
 	var accRes userapi.PerformAccountCreationResponse
 	err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{

appservice/appservice_test.go

@@ -3,50 +3,27 @@ package appservice_test
 import (
 	"context"
 	"encoding/json"
-	"fmt"
-	"net"
 	"net/http"
 	"net/http/httptest"
-	"path"
 	"reflect"
 	"regexp"
 	"strings"
 	"testing"
-	"time"
 
-	"github.com/matrix-org/dendrite/clientapi"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/federationapi/statistics"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/dendrite/syncapi"
-	uapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/tidwall/gjson"
+	"github.com/gorilla/mux"
 
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/appservice/consumers"
-	"github.com/matrix-org/dendrite/internal/caching"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/appservice/inthttp"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver"
-	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/test/testrig"
 )
 
-var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
-	return &statistics.ServerStatistics{}, nil
-}
-
 func TestAppserviceInternalAPI(t *testing.T) {
 
 	// Set expected results
@@ -100,6 +77,32 @@ func TestAppserviceInternalAPI(t *testing.T) {
 		}
 	}))
 
+	// TODO: use test.WithAllDatabases
+	// only one DBType, since appservice.AddInternalRoutes complains about multiple prometheus counters added
+	base, closeBase := testrig.CreateBaseDendrite(t, test.DBTypeSQLite)
+	defer closeBase()
+
+	// Create a dummy application service
+	base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
+		{
+			ID:              "someID",
+			URL:             srv.URL,
+			ASToken:         "",
+			HSToken:         "",
+			SenderLocalpart: "senderLocalPart",
+			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+				"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
+				"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
+			},
+			Protocols: []string{existingProtocol},
+		},
+	}
+
+	// Create required internal APIs
+	rsAPI := roomserver.NewInternalAPI(base)
+	usrAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, nil, rsAPI, nil)
+	asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
+
 	// The test cases to run
 	runCases := func(t *testing.T, testAPI api.AppServiceInternalAPI) {
 		t.Run("UserIDExists", func(t *testing.T) {
@@ -130,137 +133,23 @@ func TestAppserviceInternalAPI(t *testing.T) {
 		})
 	}
 
-	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		cfg, ctx, close := testrig.CreateConfig(t, dbType)
-		defer close()
+	// Finally execute the tests
+	t.Run("HTTP API", func(t *testing.T) {
+		router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
+		appservice.AddInternalRoutes(router, asAPI)
+		apiURL, cancel := test.ListenAndServe(t, router, false)
+		defer cancel()
 
-		// Create a dummy application service
-		as := &config.ApplicationService{
-			ID:              "someID",
-			URL:             srv.URL,
-			ASToken:         "",
-			HSToken:         "",
-			SenderLocalpart: "senderLocalPart",
-			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
-				"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
-				"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
-			},
-			Protocols: []string{existingProtocol},
+		asHTTPApi, err := inthttp.NewAppserviceClient(apiURL, &http.Client{})
+		if err != nil {
+			t.Fatalf("failed to create HTTP client: %s", err)
 		}
-		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
-		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
-		t.Cleanup(func() {
-			ctx.ShutdownDendrite()
-			ctx.WaitForShutdown()
-		})
-		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
-		// Create required internal APIs
-		natsInstance := jetstream.NATSInstance{}
-		cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
-		rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
-		asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
+		runCases(t, asHTTPApi)
+	})
 
+	t.Run("Monolith", func(t *testing.T) {
 		runCases(t, asAPI)
 	})
-}
-
-func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) {
-
-	// Set expected results
-	existingProtocol := "irc"
-	wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
-	wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
-	wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}}
-
-	// create a dummy AS url, handling some cases
-	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		switch {
-		case strings.Contains(r.URL.Path, "location"):
-			// Check if we've got an existing protocol, if so, return a proper response.
-			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
-				if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil {
-					t.Fatalf("failed to encode response: %s", err)
-				}
-				return
-			}
-			if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil {
-				t.Fatalf("failed to encode response: %s", err)
-			}
-			return
-		case strings.Contains(r.URL.Path, "user"):
-			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
-				if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil {
-					t.Fatalf("failed to encode response: %s", err)
-				}
-				return
-			}
-			if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil {
-				t.Fatalf("failed to encode response: %s", err)
-			}
-			return
-		case strings.Contains(r.URL.Path, "protocol"):
-			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
-				if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil {
-					t.Fatalf("failed to encode response: %s", err)
-				}
-				return
-			}
-			if err := json.NewEncoder(w).Encode(nil); err != nil {
-				t.Fatalf("failed to encode response: %s", err)
-			}
-			return
-		default:
-			t.Logf("hit location: %s", r.URL.Path)
-		}
-	}))
-
-	tmpDir := t.TempDir()
-	socket := path.Join(tmpDir, "socket")
-	l, err := net.Listen("unix", socket)
-	assert.NoError(t, err)
-	_ = srv.Listener.Close()
-	srv.Listener = l
-	srv.Start()
-	defer srv.Close()
-
-	cfg, ctx, tearDown := testrig.CreateConfig(t, test.DBTypeSQLite)
-	defer tearDown()
-
-	// Create a dummy application service
-	as := &config.ApplicationService{
-		ID:              "someID",
-		URL:             fmt.Sprintf("unix://%s", socket),
-		ASToken:         "",
-		HSToken:         "",
-		SenderLocalpart: "senderLocalPart",
-		NamespaceMap: map[string][]config.ApplicationServiceNamespace{
-			"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
-			"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
-		},
-		Protocols: []string{existingProtocol},
-	}
-	as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
-	cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
-
-	t.Cleanup(func() {
-		ctx.ShutdownDendrite()
-		ctx.WaitForShutdown()
-	})
-	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
-	// Create required internal APIs
-	natsInstance := jetstream.NATSInstance{}
-	cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
-	rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-	rsAPI.SetFederationAPI(nil, nil)
-	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
-	asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
-
-	t.Run("UserIDExists", func(t *testing.T) {
-		testUserIDExists(t, asAPI, "@as-testing:test", true)
-		testUserIDExists(t, asAPI, "@as1-testing:test", false)
-	})
 
 }
 
@@ -333,274 +222,3 @@ func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, w
 		t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
 	}
 }
-
-// Tests that the roomserver consumer only receives one invite
-func TestRoomserverConsumerOneInvite(t *testing.T) {
-
-	alice := test.NewUser(t)
-	bob := test.NewUser(t)
-	room := test.NewRoom(t, alice)
-
-	// Invite Bob
-	room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
-		"membership": "invite",
-	}, test.WithStateKey(bob.ID))
-
-	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
-		defer closeDB()
-		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
-		natsInstance := &jetstream.NATSInstance{}
-
-		evChan := make(chan struct{})
-		// create a dummy AS url, handling the events
-		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			var txn consumers.ApplicationServiceTransaction
-			err := json.NewDecoder(r.Body).Decode(&txn)
-			if err != nil {
-				t.Fatal(err)
-			}
-			for _, ev := range txn.Events {
-				if ev.Type != spec.MRoomMember {
-					continue
-				}
-				// Usually we would check the event content for the membership, but since
-				// we only invited bob, this should be fine for this test.
-				if ev.StateKey != nil && *ev.StateKey == bob.ID {
-					evChan <- struct{}{}
-				}
-			}
-		}))
-		defer srv.Close()
-
-		as := &config.ApplicationService{
-			ID:              "someID",
-			URL:             srv.URL,
-			ASToken:         "",
-			HSToken:         "",
-			SenderLocalpart: "senderLocalPart",
-			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
-				"users":   {{RegexpObject: regexp.MustCompile(bob.ID)}},
-				"aliases": {{RegexpObject: regexp.MustCompile(room.ID)}},
-			},
-		}
-		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
-
-		// Create a dummy application service
-		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
-
-		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
-		// Create required internal APIs
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
-		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
-		// start the consumer
-		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
-
-		// Create the room
-		if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
-			t.Fatalf("failed to send events: %v", err)
-		}
-		var seenInvitesForBob int
-	waitLoop:
-		for {
-			select {
-			case <-time.After(time.Millisecond * 50): // wait for the AS to process the events
-				break waitLoop
-			case <-evChan:
-				seenInvitesForBob++
-				if seenInvitesForBob != 1 {
-					t.Fatalf("received unexpected invites: %d", seenInvitesForBob)
-				}
-			}
-		}
-		close(evChan)
-	})
-}
-
-// Note: If this test panics, it is because we timed out waiting for the
-// join event to come through to the appservice and we close the DB/shutdown Dendrite. This makes the
-// syncAPI unhappy, as it is unable to write to the database.
-func TestOutputAppserviceEvent(t *testing.T) {
-	alice := test.NewUser(t)
-	bob := test.NewUser(t)
-
-	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
-		defer closeDB()
-		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
-		natsInstance := &jetstream.NATSInstance{}
-
-		evChan := make(chan struct{})
-
-		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
-		// Create required internal APIs
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
-		rsAPI.SetFederationAPI(nil, nil)
-
-		// Create the router, so we can hit `/joined_members`
-		routers := httputil.NewRouters()
-
-		accessTokens := map[*test.User]userDevice{
-			bob: {},
-		}
-
-		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
-		clientapi.AddPublicRoutes(processCtx, routers, cfg, natsInstance, nil, rsAPI, nil, nil, nil, usrAPI, nil, nil, caching.DisableMetrics)
-		createAccessTokens(t, accessTokens, usrAPI, processCtx.Context(), routers)
-
-		room := test.NewRoom(t, alice)
-
-		// Invite Bob
-		room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
-			"membership": "invite",
-		}, test.WithStateKey(bob.ID))
-
-		// create a dummy AS url, handling the events
-		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			var txn consumers.ApplicationServiceTransaction
-			err := json.NewDecoder(r.Body).Decode(&txn)
-			if err != nil {
-				t.Fatal(err)
-			}
-			for _, ev := range txn.Events {
-				if ev.Type != spec.MRoomMember {
-					continue
-				}
-				if ev.StateKey != nil && *ev.StateKey == bob.ID {
-					membership := gjson.GetBytes(ev.Content, "membership").Str
-					t.Logf("Processing membership: %s", membership)
-					switch membership {
-					case spec.Invite:
-						// Accept the invite
-						joinEv := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
-							"membership": "join",
-						}, test.WithStateKey(bob.ID))
-
-						if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, []*types.HeaderedEvent{joinEv}, "test", "test", "test", nil, false); err != nil {
-							t.Fatalf("failed to send events: %v", err)
-						}
-					case spec.Join: // the AS has received the join event, now hit `/joined_members` to validate that
-						rec := httptest.NewRecorder()
-						req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/rooms/"+room.ID+"/joined_members", nil)
-						req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
-						routers.Client.ServeHTTP(rec, req)
-						if rec.Code != http.StatusOK {
-							t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
-						}
-
-						// Both Alice and Bob should be joined. If not, we have a race condition
-						if !gjson.GetBytes(rec.Body.Bytes(), "joined."+alice.ID).Exists() {
-							t.Errorf("Alice is not joined to the room") // in theory should not happen
-						}
-						if !gjson.GetBytes(rec.Body.Bytes(), "joined."+bob.ID).Exists() {
-							t.Errorf("Bob is not joined to the room")
-						}
-						evChan <- struct{}{}
-					default:
-						t.Fatalf("Unexpected membership: %s", membership)
-					}
-				}
-			}
-		}))
-		defer srv.Close()
-
-		as := &config.ApplicationService{
-			ID:              "someID",
-			URL:             srv.URL,
-			ASToken:         "",
-			HSToken:         "",
-			SenderLocalpart: "senderLocalPart",
-			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
-				"users":   {{RegexpObject: regexp.MustCompile(bob.ID)}},
-				"aliases": {{RegexpObject: regexp.MustCompile(room.ID)}},
-			},
-		}
-		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
-
-		// Create a dummy application service
-		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
-
-		// Prepare AS Streams on the old topic to validate that they get deleted
-		jsCtx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
-
-		token := jetstream.Tokenise(as.ID)
-		if err := jetstream.JetStreamConsumer(
-			processCtx.Context(), jsCtx, cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
-			cfg.Global.JetStream.Durable("Appservice_"+token),
-			50, // maximum number of events to send in a single transaction
-			func(ctx context.Context, msgs []*nats.Msg) bool {
-				return true
-			},
-		); err != nil {
-			t.Fatal(err)
-		}
-
-		// Start the syncAPI to have `/joined_members` available
-		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, natsInstance, usrAPI, rsAPI, caches, caching.DisableMetrics)
-
-		// start the consumer
-		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
-
-		// At this point, the old JetStream consumers should be deleted
-		for consumer := range jsCtx.Consumers(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) {
-			if consumer.Name == cfg.Global.JetStream.Durable("Appservice_"+token)+"Pull" {
-				t.Fatalf("Consumer still exists")
-			}
-		}
-
-		// Create the room, this triggers the AS to receive an invite for Bob.
-		if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
-			t.Fatalf("failed to send events: %v", err)
-		}
-
-		select {
-		// Pretty generous timeout duration...
-		case <-time.After(time.Millisecond * 1000): // wait for the AS to process the events
-			t.Errorf("Timed out waiting for join event")
-		case <-evChan:
-		}
-		close(evChan)
-	})
-}
-
-type userDevice struct {
-	accessToken string
-	deviceID    string
-	password    string
-}
-
-func createAccessTokens(t *testing.T, accessTokens map[*test.User]userDevice, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) {
-	t.Helper()
-	for u := range accessTokens {
-		localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
-		userRes := &uapi.PerformAccountCreationResponse{}
-		password := util.RandomString(8)
-		if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
-			AccountType: u.AccountType,
-			Localpart:   localpart,
-			ServerName:  serverName,
-			Password:    password,
-		}, userRes); err != nil {
-			t.Errorf("failed to create account: %s", err)
-		}
-		req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
-			"type": authtypes.LoginTypePassword,
-			"identifier": map[string]interface{}{
-				"type": "m.id.user",
-				"user": u.ID,
-			},
-			"password": password,
-		}))
-		rec := httptest.NewRecorder()
-		routers.Client.ServeHTTP(rec, req)
-		if rec.Code != http.StatusOK {
-			t.Fatalf("failed to login: %s", rec.Body.String())
-		}
-		accessTokens[u] = userDevice{
-			accessToken: gjson.GetBytes(rec.Body.Bytes(), "access_token").String(),
-			deviceID:    gjson.GetBytes(rec.Body.Bytes(), "device_id").String(),
-			password:    password,
-		}
-	}
-}

appservice/consumers/roomserver.go

@@ -26,29 +26,21 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 
 	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/dendrite/syncapi/synctypes"
 
 	log "github.com/sirupsen/logrus"
 )
 
-// ApplicationServiceTransaction is the transaction that is sent off to an
-// application service.
-type ApplicationServiceTransaction struct {
-	Events []synctypes.ClientEvent `json:"events"`
-}
-
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
 	ctx       context.Context
 	cfg       *config.AppServiceAPI
+	client    *http.Client
 	jetstream nats.JetStreamContext
 	topic     string
 	rsAPI     api.AppserviceRoomserverAPI
@@ -64,21 +56,22 @@ type appserviceState struct {
 func NewOutputRoomEventConsumer(
 	process *process.ProcessContext,
 	cfg *config.AppServiceAPI,
+	client *http.Client,
 	js nats.JetStreamContext,
 	rsAPI api.AppserviceRoomserverAPI,
 ) *OutputRoomEventConsumer {
 	return &OutputRoomEventConsumer{
 		ctx:       process.Context(),
 		cfg:       cfg,
+		client:    client,
 		jetstream: js,
-		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputAppserviceEvent),
+		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
 		rsAPI:     rsAPI,
 	}
 }
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	durableNames := make([]string, 0, len(s.cfg.Derived.ApplicationServices))
 	for _, as := range s.cfg.Derived.ApplicationServices {
 		appsvc := as
 		state := &appserviceState{
@@ -96,15 +89,6 @@ func (s *OutputRoomEventConsumer) Start() error {
 		); err != nil {
 			return fmt.Errorf("failed to create %q consumer: %w", token, err)
 		}
-		durableNames = append(durableNames, s.cfg.Matrix.JetStream.Durable("Appservice_"+token))
-	}
-	// Cleanup any consumers still existing on the OutputRoomEvent stream
-	// to avoid messages not being deleted
-	for _, consumerName := range durableNames {
-		err := s.jetstream.DeleteConsumer(s.cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), consumerName+"Pull")
-		if err != nil && err != nats.ErrConsumerNotFound {
-			return err
-		}
 	}
 	return nil
 }
@@ -115,7 +99,7 @@ func (s *OutputRoomEventConsumer) onMessage(
 	ctx context.Context, state *appserviceState, msgs []*nats.Msg,
 ) bool {
 	log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs))
-	events := make([]*types.HeaderedEvent, 0, len(msgs))
+	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs))
 	for _, msg := range msgs {
 		// Only handle events we care about
 		receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
@@ -138,7 +122,6 @@ func (s *OutputRoomEventConsumer) onMessage(
 			if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
 				newEventID := output.NewRoomEvent.Event.EventID()
 				eventsReq := &api.QueryEventsByIDRequest{
-					RoomID:   output.NewRoomEvent.Event.RoomID().String(),
 					EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
 				}
 				eventsRes := &api.QueryEventsByIDResponse{}
@@ -156,6 +139,12 @@ func (s *OutputRoomEventConsumer) onMessage(
 				}
 			}
 
+		case api.OutputTypeNewInviteEvent:
+			if output.NewInviteEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewInviteEvent.Event, state.ApplicationService) {
+				continue
+			}
+			events = append(events, output.NewInviteEvent.Event)
+
 		default:
 			continue
 		}
@@ -185,15 +174,13 @@ func (s *OutputRoomEventConsumer) onMessage(
 // endpoint. It will block for the backoff period if necessary.
 func (s *OutputRoomEventConsumer) sendEvents(
 	ctx context.Context, state *appserviceState,
-	events []*types.HeaderedEvent,
+	events []*gomatrixserverlib.HeaderedEvent,
 	txnID string,
 ) error {
 	// Create the transaction body.
 	transaction, err := json.Marshal(
-		ApplicationServiceTransaction{
-			Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
-				return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
-			}),
+		gomatrixserverlib.ApplicationServiceTransaction{
+			Events: gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll),
 		},
 	)
 	if err != nil {
@@ -202,26 +189,18 @@ func (s *OutputRoomEventConsumer) sendEvents(
 
 	// If txnID is not defined, generate one from the events.
 	if txnID == "" {
-		txnID = fmt.Sprintf("%d_%d", events[0].PDU.OriginServerTS(), len(transaction))
+		txnID = fmt.Sprintf("%d_%d", events[0].Event.OriginServerTS(), len(transaction))
 	}
 
 	// Send the transaction to the appservice.
-	// https://spec.matrix.org/v1.9/application-service-api/#pushing-events
-	path := "_matrix/app/v1/transactions"
-	if s.cfg.LegacyPaths {
-		path = "transactions"
-	}
-	address := fmt.Sprintf("%s/%s/%s", state.RequestUrl(), path, txnID)
-	if s.cfg.LegacyAuth {
-		address += "?access_token=" + url.QueryEscape(state.HSToken)
-	}
+	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
+	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.URL, txnID, url.QueryEscape(state.HSToken))
 	req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
 	if err != nil {
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", state.HSToken))
-	resp, err := state.HTTPClient.Do(req)
+	resp, err := s.client.Do(req)
 	if err != nil {
 		return state.backoffAndPause(err)
 	}
@@ -232,7 +211,7 @@ func (s *OutputRoomEventConsumer) sendEvents(
 	case http.StatusOK:
 		state.backoff = 0
 	default:
-		return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice url %s", resp.StatusCode, address))
+		return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice", resp.StatusCode))
 	}
 	return nil
 }
@@ -252,30 +231,24 @@ func (s *appserviceState) backoffAndPause(err error) error {
 // event falls within one of a given application service's namespaces.
 //
 // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
-func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool {
-	user := ""
-	userID, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
-	if err == nil {
-		user = userID.String()
-	}
-
+func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool {
 	switch {
 	case appservice.URL == "":
 		return false
-	case appservice.IsInterestedInUserID(user):
+	case appservice.IsInterestedInUserID(event.Sender()):
 		return true
-	case appservice.IsInterestedInRoomID(event.RoomID().String()):
+	case appservice.IsInterestedInRoomID(event.RoomID()):
 		return true
 	}
 
-	if event.Type() == spec.MRoomMember && event.StateKey() != nil {
+	if event.Type() == gomatrixserverlib.MRoomMember && event.StateKey() != nil {
 		if appservice.IsInterestedInUserID(*event.StateKey()) {
 			return true
 		}
 	}
 
 	// Check all known room aliases of the room the event came from
-	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID().String()}
+	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
 	var queryRes api.GetAliasesForRoomIDResponse
 	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
 		for _, alias := range queryRes.Aliases {
@@ -286,7 +259,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID().String(),
+			"room_id":    event.RoomID(),
 		}).WithError(err).Errorf("Unable to get aliases for room")
 	}
 
@@ -296,13 +269,13 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 
 // appserviceJoinedAtEvent returns a boolean depending on whether a given
 // appservice has membership at the time a given event was created.
-func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool {
+func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool {
 	// TODO: This is only checking the current room state, not the state at
 	// the event in question. Pretty sure this is what Synapse does too, but
 	// until we have a lighter way of checking the state before the event that
 	// doesn't involve state res, then this is probably OK.
 	membershipReq := &api.QueryMembershipsForRoomRequest{
-		RoomID:     event.RoomID().String(),
+		RoomID:     event.RoomID(),
 		JoinedOnly: true,
 	}
 	membershipRes := &api.QueryMembershipsForRoomResponse{}
@@ -314,7 +287,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 			switch {
 			case ev.StateKey == nil:
 				continue
-			case ev.Type != spec.MRoomMember:
+			case ev.Type != gomatrixserverlib.MRoomMember:
 				continue
 			}
 			var membership gomatrixserverlib.MemberContent
@@ -322,7 +295,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 			switch {
 			case err != nil:
 				continue
-			case membership.Membership == spec.Join:
+			case membership.Membership == gomatrixserverlib.Join:
 				if appservice.IsInterestedInUserID(*ev.StateKey) {
 					return true
 				}
@@ -331,7 +304,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID().String(),
+			"room_id":    event.RoomID(),
 		}).WithError(err).Errorf("Unable to get membership for room")
 	}
 	return false

appservice/inthttp/client.go

@@ -0,0 +1,84 @@
+package inthttp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/internal/httputil"
+)
+
+// HTTP paths for the internal HTTP APIs
+const (
+	AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists"
+	AppServiceUserIDExistsPath    = "/appservice/UserIDExists"
+	AppServiceLocationsPath       = "/appservice/locations"
+	AppServiceUserPath            = "/appservice/users"
+	AppServiceProtocolsPath       = "/appservice/protocols"
+)
+
+// httpAppServiceQueryAPI contains the URL to an appservice query API and a
+// reference to a httpClient used to reach it
+type httpAppServiceQueryAPI struct {
+	appserviceURL string
+	httpClient    *http.Client
+}
+
+// NewAppserviceClient creates a AppServiceQueryAPI implemented by talking
+// to a HTTP POST API.
+// If httpClient is nil an error is returned
+func NewAppserviceClient(
+	appserviceURL string,
+	httpClient *http.Client,
+) (api.AppServiceInternalAPI, error) {
+	if httpClient == nil {
+		return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is <nil>")
+	}
+	return &httpAppServiceQueryAPI{appserviceURL, httpClient}, nil
+}
+
+// RoomAliasExists implements AppServiceQueryAPI
+func (h *httpAppServiceQueryAPI) RoomAliasExists(
+	ctx context.Context,
+	request *api.RoomAliasExistsRequest,
+	response *api.RoomAliasExistsResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"RoomAliasExists", h.appserviceURL+AppServiceRoomAliasExistsPath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
+// UserIDExists implements AppServiceQueryAPI
+func (h *httpAppServiceQueryAPI) UserIDExists(
+	ctx context.Context,
+	request *api.UserIDExistsRequest,
+	response *api.UserIDExistsResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"UserIDExists", h.appserviceURL+AppServiceUserIDExistsPath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
+func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"ASLocation", h.appserviceURL+AppServiceLocationsPath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
+func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"ASUser", h.appserviceURL+AppServiceUserPath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
+func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"ASProtocols", h.appserviceURL+AppServiceProtocolsPath,
+		h.httpClient, ctx, request, response,
+	)
+}

appservice/inthttp/server.go

@@ -0,0 +1,36 @@
+package inthttp
+
+import (
+	"github.com/gorilla/mux"
+
+	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/internal/httputil"
+)
+
+// AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
+func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
+	internalAPIMux.Handle(
+		AppServiceRoomAliasExistsPath,
+		httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", a.RoomAliasExists),
+	)
+
+	internalAPIMux.Handle(
+		AppServiceUserIDExistsPath,
+		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
+	)
+
+	internalAPIMux.Handle(
+		AppServiceProtocolsPath,
+		httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
+	)
+
+	internalAPIMux.Handle(
+		AppServiceLocationsPath,
+		httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
+	)
+
+	internalAPIMux.Handle(
+		AppServiceUserPath,
+		httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
+	)
+}

appservice/query/query.go

@@ -19,21 +19,25 @@ package query
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
 	"sync"
 
+	"github.com/opentracing/opentracing-go"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
+const roomAliasExistsPath = "/rooms/"
+const userIDExistsPath = "/users/"
+
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
+	HTTPClient    *http.Client
 	Cfg           *config.AppServiceAPI
 	ProtocolCache map[string]api.ASProtocolResponse
 	CacheMu       sync.Mutex
@@ -46,29 +50,20 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 	request *api.RoomAliasExistsRequest,
 	response *api.RoomAliasExistsResponse,
 ) error {
-	trace, ctx := internal.StartRegion(ctx, "ApplicationServiceRoomAlias")
-	defer trace.EndRegion()
+	span, ctx := opentracing.StartSpanFromContext(ctx, "ApplicationServiceRoomAlias")
+	defer span.Finish()
 
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
-			path := api.ASRoomAliasExistsPath
-			if a.Cfg.LegacyPaths {
-				path = api.ASRoomAliasExistsLegacyPath
-			}
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.RequestUrl() + path)
+			URL, err := url.Parse(appservice.URL + roomAliasExistsPath)
 			if err != nil {
 				return err
 			}
 
 			URL.Path += request.Alias
-			if a.Cfg.LegacyAuth {
-				q := URL.Query()
-				q.Set("access_token", appservice.HSToken)
-				URL.RawQuery = q.Encode()
-			}
-			apiURL := URL.String()
+			apiURL := URL.String() + "?access_token=" + appservice.HSToken
 
 			// Send a request to each application service. If one responds that it has
 			// created the room, immediately return.
@@ -76,10 +71,9 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			if err != nil {
 				return err
 			}
-			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
 			req = req.WithContext(ctx)
 
-			resp, err := appservice.HTTPClient.Do(req)
+			resp, err := a.HTTPClient.Do(req)
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
@@ -123,28 +117,19 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	request *api.UserIDExistsRequest,
 	response *api.UserIDExistsResponse,
 ) error {
-	trace, ctx := internal.StartRegion(ctx, "ApplicationServiceUserID")
-	defer trace.EndRegion()
+	span, ctx := opentracing.StartSpanFromContext(ctx, "ApplicationServiceUserID")
+	defer span.Finish()
 
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
 			// The full path to the rooms API, includes hs token
-			path := api.ASUserExistsPath
-			if a.Cfg.LegacyPaths {
-				path = api.ASUserExistsLegacyPath
-			}
-			URL, err := url.Parse(appservice.RequestUrl() + path)
+			URL, err := url.Parse(appservice.URL + userIDExistsPath)
 			if err != nil {
 				return err
 			}
 			URL.Path += request.UserID
-			if a.Cfg.LegacyAuth {
-				q := URL.Query()
-				q.Set("access_token", appservice.HSToken)
-				URL.RawQuery = q.Encode()
-			}
-			apiURL := URL.String()
+			apiURL := URL.String() + "?access_token=" + appservice.HSToken
 
 			// Send a request to each application service. If one responds that it has
 			// created the user, immediately return.
@@ -152,8 +137,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
-			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
-			resp, err := appservice.HTTPClient.Do(req.WithContext(ctx))
+			resp, err := a.HTTPClient.Do(req.WithContext(ctx))
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
@@ -193,22 +177,25 @@ type thirdpartyResponses interface {
 	api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
 }
 
-func requestDo[T thirdpartyResponses](as *config.ApplicationService, url string, response *T) error {
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return err
+func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
+	origURL := url
+	// try v1 and unstable appservice endpoints
+	for _, version := range []string{"v1", "unstable"} {
+		var resp *http.Response
+		var body []byte
+		asURL := strings.Replace(origURL, "unstable", version, 1)
+		resp, err = client.Get(asURL)
+		if err != nil {
+			continue
+		}
+		defer resp.Body.Close() // nolint: errcheck
+		body, err = io.ReadAll(resp.Body)
+		if err != nil {
+			continue
+		}
+		return json.Unmarshal(body, &response)
 	}
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", as.HSToken))
-	resp, err := as.HTTPClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close() // nolint: errcheck
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	return json.Unmarshal(body, &response)
+	return err
 }
 
 func (a *AppServiceQueryAPI) Locations(
@@ -221,23 +208,17 @@ func (a *AppServiceQueryAPI) Locations(
 		return err
 	}
 
-	path := api.ASLocationPath
-	if a.Cfg.LegacyPaths {
-		path = api.ASLocationLegacyPath
-	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asLocations []api.ASLocationResponse
-		if a.Cfg.LegacyAuth {
-			params.Set("access_token", as.HSToken)
-		}
+		params.Set("access_token", as.HSToken)
 
-		url := as.RequestUrl() + path
+		url := as.URL + api.ASLocationPath
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASLocationResponse](&as, url+"?"+params.Encode(), &asLocations); err != nil {
-			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service")
+		if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
+			log.WithError(err).Error("unable to get 'locations' from application service")
 			continue
 		}
 
@@ -262,23 +243,17 @@ func (a *AppServiceQueryAPI) User(
 		return err
 	}
 
-	path := api.ASUserPath
-	if a.Cfg.LegacyPaths {
-		path = api.ASUserLegacyPath
-	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asUsers []api.ASUserResponse
-		if a.Cfg.LegacyAuth {
-			params.Set("access_token", as.HSToken)
-		}
+		params.Set("access_token", as.HSToken)
 
-		url := as.RequestUrl() + path
+		url := as.URL + api.ASUserPath
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASUserResponse](&as, url+"?"+params.Encode(), &asUsers); err != nil {
-			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service")
+		if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
+			log.WithError(err).Error("unable to get 'user' from application service")
 			continue
 		}
 
@@ -298,10 +273,6 @@ func (a *AppServiceQueryAPI) Protocols(
 	req *api.ProtocolRequest,
 	resp *api.ProtocolResponse,
 ) error {
-	protocolPath := api.ASProtocolPath
-	if a.Cfg.LegacyPaths {
-		protocolPath = api.ASProtocolLegacyPath
-	}
 
 	// get a single protocol response
 	if req.Protocol != "" {
@@ -319,8 +290,8 @@ func (a *AppServiceQueryAPI) Protocols(
 		response := api.ASProtocolResponse{}
 		for _, as := range a.Cfg.Derived.ApplicationServices {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+req.Protocol, &proto); err != nil {
-				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
+			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil {
+				log.WithError(err).Error("unable to get 'protocol' from application service")
 				continue
 			}
 
@@ -349,8 +320,8 @@ func (a *AppServiceQueryAPI) Protocols(
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		for _, p := range as.Protocols {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+p, &proto); err != nil {
-				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
+			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil {
+				log.WithError(err).Error("unable to get 'protocol' from application service")
 				continue
 			}
 			existing, ok := response[p]

are-we-synapse-yet.list

@@ -936,20 +936,4 @@ fst Room state after a rejected message event is the same as before
 fst Room state after a rejected state event is the same as before
 fpb Federation publicRoom Name/topic keys are correct
 fed New federated private chats get full presence information (SYN-115) (10 subtests)
-dvk Rejects invalid device keys
-rmv User can create and send/receive messages in a room with version 10
-rmv local user can join room with version 10
-rmv User can invite local user to room with version 10
-rmv remote user can join room with version 10
-rmv User can invite remote user to room with version 10
-rmv Remote user can backfill in a room with version 10
-rmv Can reject invites over federation for rooms with version 10
-rmv Can receive redactions from regular users over federation in room version 10
-rmv User can create and send/receive messages in a room with version 11
-rmv local user can join room with version 11
-rmv User can invite local user to room with version 11
-rmv remote user can join room with version 11
-rmv User can invite remote user to room with version 11
-rmv Remote user can backfill in a room with version 11
-rmv Can reject invites over federation for rooms with version 11
-rmv Can receive redactions from regular users over federation in room version 11
+dvk Rejects invalid device keys

build.cmd

@@ -0,0 +1,51 @@
+@echo off
+
+:ENTRY_POINT
+    setlocal EnableDelayedExpansion
+
+    REM script base dir
+    set SCRIPTDIR=%~dp0
+    set PROJDIR=%SCRIPTDIR:~0,-1%
+
+    REM Put installed packages into ./bin
+    set GOBIN=%PROJDIR%\bin
+
+    set FLAGS=
+
+    REM Check if sources are under Git control
+    if not exist ".git" goto :CHECK_BIN
+
+    REM set BUILD=`git rev-parse --short HEAD \\ ""`
+    FOR /F "tokens=*" %%X IN ('git rev-parse --short HEAD') DO (
+        set BUILD=%%X
+    )
+
+    REM set BRANCH=`(git symbolic-ref --short HEAD \ tr -d \/ ) \\ ""`
+    FOR /F "tokens=*" %%X IN ('git symbolic-ref --short HEAD') DO (
+        set BRANCHRAW=%%X
+        set BRANCH=!BRANCHRAW:/=!
+    )
+    if "%BRANCH%" == "main" set BRANCH=
+
+    set FLAGS=-X github.com/matrix-org/dendrite/internal.branch=%BRANCH% -X github.com/matrix-org/dendrite/internal.build=%BUILD%
+
+:CHECK_BIN
+    if exist "bin" goto :ALL_SET
+    mkdir "bin"
+
+:ALL_SET
+    set CGO_ENABLED=1
+    for /D %%P in (cmd\*) do (
+        go build -trimpath -ldflags "%FLAGS%" -v -o ".\bin" ".\%%P"
+    )
+
+    set CGO_ENABLED=0
+    set GOOS=js
+    set GOARCH=wasm
+    go build -trimpath -ldflags "%FLAGS%" -o bin\main.wasm .\cmd\dendritejs-pinecone
+
+    goto :DONE
+
+:DONE
+    echo Done
+    endlocal

build.sh

@@ -0,0 +1,24 @@
+#!/bin/sh -eu
+
+# Put installed packages into ./bin
+export GOBIN=$PWD/`dirname $0`/bin
+
+if [ -d ".git" ] 
+then
+    export BUILD=`git rev-parse --short HEAD || ""`
+    export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""`
+    if [ "$BRANCH" = main ]
+    then
+        export BRANCH=""
+    fi
+
+    export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD"
+else
+    export FLAGS=""
+fi
+
+mkdir -p bin
+
+CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
+
+# CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone

build/dendritejs-pinecone/main.go

@@ -29,16 +29,13 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -161,8 +158,9 @@ func startup() {
 	pManager.AddPeer("wss://pinecone.matrix.org/public")
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: false})
+	cfg.Defaults(true)
 	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
+	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
 	cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
 	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
 	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
@@ -172,37 +170,37 @@ func startup() {
 	cfg.Global.TrustedIDServers = []string{}
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.PrivateKey = sk
-	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.ClientAPI.RegistrationDisabled = false
 	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 
 	if err := cfg.Derive(); err != nil {
 		logrus.Fatalf("Failed to derive values from config: %s", err)
 	}
-	natsInstance := jetstream.NATSInstance{}
-	processCtx := process.NewProcessContext()
-	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
-	routers := httputil.NewRouters()
-	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
-	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
+	base := base.NewBaseDendrite(cfg, "Monolith")
+	defer base.Close() // nolint: errcheck
 
-	federation := conn.CreateFederationClient(cfg, pSessions)
+	federation := conn.CreateFederationClient(base, pSessions)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fedSenderAPI.IsBlacklistedOrBackingOff)
+	rsAPI := roomserver.NewInternalAPI(base)
+
+	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
+	keyAPI.SetUserAPI(userAPI)
 
 	asQuery := appservice.NewInternalAPI(
-		processCtx, cfg, &natsInstance, userAPI, rsAPI,
+		base, userAPI, rsAPI,
 	)
 	rsAPI.SetAppserviceAPI(asQuery)
+	fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, keyRing, true)
 	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
 
 	monolith := setup.Monolith{
-		Config:    cfg,
-		Client:    conn.CreateClient(pSessions),
+		Config:    base.Cfg,
+		Client:    conn.CreateClient(base, pSessions),
 		FedClient: federation,
 		KeyRing:   keyRing,
 
@@ -210,18 +208,20 @@ func startup() {
 		FederationAPI: fedSenderAPI,
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
+		KeyAPI:        keyAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation),
 	}
-	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
+	monolith.AddAllPublicRoutes(base)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
+	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 
 	p2pRouter := pSessions.Protocol("matrix").HTTP().Mux()
-	p2pRouter.Handle(httputil.PublicFederationPathPrefix, routers.Federation)
-	p2pRouter.Handle(httputil.PublicMediaPathPrefix, routers.Media)
+	p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux)
+	p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux)
 
 	// Expose the matrix APIs via fetch - for local traffic
 	go func() {

build/docker/Dockerfile.demo-pinecone

@@ -1,5 +1,4 @@
-# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
-FROM docker.io/golang:1.21-alpine3.18 AS base
+FROM docker.io/golang:1.19-alpine AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,
@@ -18,7 +17,6 @@ RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
-RUN apk --update --no-cache add curl
 LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
 LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
 LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"

build/docker/Dockerfile.demo-yggdrasil

@@ -1,5 +1,4 @@
-# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
-FROM docker.io/golang:1.21-alpine3.18 AS base
+FROM docker.io/golang:1.19-alpine AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/docker/README.md

@@ -5,21 +5,27 @@ These are Docker images for Dendrite!
 They can be found on Docker Hub:
 
 - [matrixdotorg/dendrite-monolith](https://hub.docker.com/r/matrixdotorg/dendrite-monolith) for monolith deployments
+- [matrixdotorg/dendrite-polylith](https://hub.docker.com/r/matrixdotorg/dendrite-polylith) for polylith deployments
 
-## Dockerfile
+## Dockerfiles
 
-The `Dockerfile` is a multistage file which can build Dendrite. From the root of the Dendrite
+The `Dockerfile` is a multistage file which can build all four Dendrite
+images depending on the supplied `--target`. From the root of the Dendrite
 repository, run:
 
 ```
-docker build . -t matrixdotorg/dendrite-monolith
+docker build . --target monolith -t matrixdotorg/dendrite-monolith
+docker build . --target polylith -t matrixdotorg/dendrite-monolith
+docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone
+docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil
 ```
 
-## Compose file
+## Compose files
 
-There is one sample `docker-compose` files:
+There are two sample `docker-compose` files:
 
-- `docker-compose.yml` which runs a Dendrite deployment with Postgres
+- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment
+- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment
 
 ## Configuration
 
@@ -45,14 +51,24 @@ docker run --rm --entrypoint="" \
 
 The key files will now exist in your current working directory, and can be mounted into place.
 
-## Starting Dendrite
+## Starting Dendrite as a monolith deployment
 
-Create your config based on the [`dendrite-sample.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.yaml) sample configuration file.
+Create your config based on the [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) sample configuration file.
 
 Then start the deployment:
 
 ```
-docker-compose -f docker-compose.yml up
+docker-compose -f docker-compose.monolith.yml up
+```
+
+## Starting Dendrite as a polylith deployment
+
+Create your config based on the [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) sample configuration file.
+
+Then start the deployment:
+
+```
+docker-compose -f docker-compose.polylith.yml up
 ```
 
 ## Building the images

build/docker/docker-compose.monolith.yml

@@ -0,0 +1,44 @@
+version: "3.4"
+services:
+  postgres:
+    hostname: postgres
+    image: postgres:14
+    restart: always
+    volumes:
+      - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
+      # To persist your PostgreSQL databases outside of the Docker image, 
+      # to prevent data loss, modify the following ./path_to path:
+      - ./path_to/postgresql:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: itsasecret
+      POSTGRES_USER: dendrite
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U dendrite"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    networks:
+      - internal
+
+  monolith:
+    hostname: monolith
+    image: matrixdotorg/dendrite-monolith:latest
+    command: [
+      "--tls-cert=server.crt",
+      "--tls-key=server.key"
+    ]
+    ports:
+      - 8008:8008
+      - 8448:8448
+    volumes:
+      - ./config:/etc/dendrite
+      - ./media:/var/dendrite/media
+    depends_on:
+      - postgres
+    networks:
+      - internal
+    restart: unless-stopped
+
+networks:
+  internal:
+    attachable: true

build/docker/docker-compose.polylith.yml

@@ -0,0 +1,143 @@
+version: "3.4"
+services:
+  postgres:
+    hostname: postgres
+    image: postgres:14
+    restart: always
+    volumes:
+      - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
+      # To persist your PostgreSQL databases outside of the Docker image, 
+      # to prevent data loss, modify the following ./path_to path:
+      - ./path_to/postgresql:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: itsasecret
+      POSTGRES_USER: dendrite
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U dendrite"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    networks:
+      - internal
+
+  jetstream:
+    hostname: jetstream
+    image: nats:latest
+    command: |
+      --jetstream
+      --store_dir /var/lib/nats
+      --cluster_name Dendrite
+    volumes:
+      # To persist your NATS JetStream streams outside of the Docker image,
+      # prevent data loss, modify the following ./path_to path:
+      - ./path_to/nats:/var/lib/nats
+    networks:
+      - internal
+
+  client_api:
+    hostname: client_api
+    image: matrixdotorg/dendrite-polylith:latest
+    command: clientapi
+    volumes:
+      - ./config:/etc/dendrite
+    depends_on:
+      - jetstream
+      - postgres
+    networks:
+      - internal
+    restart: unless-stopped
+
+  media_api:
+    hostname: media_api
+    image: matrixdotorg/dendrite-polylith:latest
+    command: mediaapi
+    volumes:
+      - ./config:/etc/dendrite
+      - ./media:/var/dendrite/media
+    networks:
+      - internal
+    restart: unless-stopped
+
+  sync_api:
+    hostname: sync_api
+    image: matrixdotorg/dendrite-polylith:latest
+    command: syncapi
+    volumes:
+      - ./config:/etc/dendrite
+    depends_on:
+      - jetstream
+      - postgres
+    networks:
+      - internal
+    restart: unless-stopped
+
+  room_server:
+    hostname: room_server
+    image: matrixdotorg/dendrite-polylith:latest
+    command: roomserver
+    volumes:
+      - ./config:/etc/dendrite
+    depends_on:
+      - jetstream
+      - postgres
+    networks:
+      - internal
+    restart: unless-stopped
+
+  federation_api:
+    hostname: federation_api
+    image: matrixdotorg/dendrite-polylith:latest
+    command: federationapi
+    volumes:
+      - ./config:/etc/dendrite
+    depends_on:
+      - jetstream
+      - postgres
+    networks:
+      - internal
+    restart: unless-stopped
+      
+  key_server:
+    hostname: key_server
+    image: matrixdotorg/dendrite-polylith:latest
+    command: keyserver
+    volumes:
+      - ./config:/etc/dendrite
+    depends_on:
+      - jetstream
+      - postgres
+    networks:
+      - internal
+    restart: unless-stopped
+      
+  user_api:
+    hostname: user_api
+    image: matrixdotorg/dendrite-polylith:latest
+    command: userapi
+    volumes:
+      - ./config:/etc/dendrite
+    depends_on:
+      - jetstream
+      - postgres
+    networks:
+      - internal
+    restart: unless-stopped
+
+  appservice_api:
+    hostname: appservice_api
+    image: matrixdotorg/dendrite-polylith:latest
+    command: appservice
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+    depends_on:
+      - jetstream
+      - postgres
+      - room_server
+      - user_api
+    restart: unless-stopped
+
+networks:
+  internal:
+    attachable: true

build/docker/docker-compose.yml

@@ -1,52 +0,0 @@
-version: "3.4"
-
-services:
-  postgres:
-    hostname: postgres
-    image: postgres:15-alpine
-    restart: always
-    volumes:
-      # This will create a docker volume to persist the database files in.
-      # If you prefer those files to be outside of docker, you'll need to change this.
-      - dendrite_postgres_data:/var/lib/postgresql/data
-    environment:
-      POSTGRES_PASSWORD: itsasecret
-      POSTGRES_USER: dendrite
-      POSTGRES_DATABASE: dendrite
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U dendrite"]
-      interval: 5s
-      timeout: 5s
-      retries: 5
-    networks:
-      - internal
-
-  monolith:
-    hostname: monolith
-    image: matrixdotorg/dendrite-monolith:latest
-    ports:
-      - 8008:8008
-      - 8448:8448
-    volumes:
-      - ./config:/etc/dendrite
-      # The following volumes use docker volumes, change this
-      # if you prefer to have those files outside of docker.
-      - dendrite_media:/var/dendrite/media
-      - dendrite_jetstream:/var/dendrite/jetstream
-      - dendrite_search_index:/var/dendrite/searchindex
-    depends_on:
-      postgres:
-        condition: service_healthy
-    networks:
-      - internal
-    restart: unless-stopped
-
-networks:
-  internal:
-    attachable: true
-
-volumes:
-  dendrite_postgres_data:
-  dendrite_media:
-  dendrite_jetstream:
-  dendrite_search_index:

build/docker/images-build.sh

@@ -7,5 +7,6 @@ TAG=${1:-latest}
 echo "Building tag '${TAG}'"
 
 docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG}
+docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG}
 docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG}
 docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil:${TAG}

build/docker/images-pull.sh

@@ -5,3 +5,4 @@ TAG=${1:-latest}
 echo "Pulling tag '${TAG}'"
 
 docker pull matrixdotorg/dendrite-monolith:${TAG}
+docker pull matrixdotorg/dendrite-polylith:${TAG}

build/docker/images-push.sh

@@ -5,3 +5,4 @@ TAG=${1:-latest}
 echo "Pushing tag '${TAG}'"
 
 docker push matrixdotorg/dendrite-monolith:${TAG}
+docker push matrixdotorg/dendrite-polylith:${TAG}

build/docker/postgres/create_db.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+for db in userapi_accounts mediaapi syncapi roomserver keyserver federationapi appservice mscs; do
+    createdb -U dendrite -O dendrite dendrite_$db
+done

build/gobind-pinecone/build.sh

@@ -7,7 +7,7 @@ do
     case "$option"
     in
     a) gomobile bind -v -target android -trimpath -ldflags="-s -w" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
-    i) gomobile bind -v -target ios -trimpath -ldflags="" -o ~/DendriteBindings/Gobind.xcframework . ;;
+    i) gomobile bind -v -target ios -trimpath -ldflags="" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
     *) echo "No target specified, specify -a or -i"; exit 1 ;;
     esac
 done

build/gobind-pinecone/monolith.go

@@ -18,29 +18,50 @@ import (
 	"context"
 	"crypto/ed25519"
 	"crypto/rand"
+	"crypto/tls"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"net"
+	"net/http"
+	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
+	"go.uber.org/atomic"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conduit"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/monolith"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/relay"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/userapi"
 	userapiAPI "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
-	"github.com/matrix-org/pinecone/types"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
 
+	pineconeConnections "github.com/matrix-org/pinecone/connections"
 	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
 	pineconeRouter "github.com/matrix-org/pinecone/router"
+	pineconeEvents "github.com/matrix-org/pinecone/router/events"
+	pineconeSessions "github.com/matrix-org/pinecone/sessions"
+	"github.com/matrix-org/pinecone/types"
 
 	_ "golang.org/x/mobile/bind"
 )
@@ -50,37 +71,36 @@ const (
 	PeerTypeMulticast = pineconeRouter.PeerTypeMulticast
 	PeerTypeBluetooth = pineconeRouter.PeerTypeBluetooth
 	PeerTypeBonjour   = pineconeRouter.PeerTypeBonjour
-
-	MaxFrameSize = types.MaxFrameSize
 )
 
-// Re-export Conduit in this package for bindings.
-type Conduit struct {
-	conduit.Conduit
-}
-
 type DendriteMonolith struct {
-	logger           logrus.Logger
-	p2pMonolith      monolith.P2PMonolith
-	StorageDirectory string
-	CacheDirectory   string
-	listener         net.Listener
+	logger            logrus.Logger
+	PineconeRouter    *pineconeRouter.Router
+	PineconeMulticast *pineconeMulticast.Multicast
+	PineconeQUIC      *pineconeSessions.Sessions
+	PineconeManager   *pineconeConnections.ConnectionManager
+	StorageDirectory  string
+	CacheDirectory    string
+	listener          net.Listener
+	httpServer        *http.Server
+	processContext    *process.ProcessContext
+	userAPI           userapiAPI.UserInternalAPI
 }
 
 func (m *DendriteMonolith) PublicKey() string {
-	return m.p2pMonolith.Router.PublicKey().String()
+	return m.PineconeRouter.PublicKey().String()
 }
 
 func (m *DendriteMonolith) BaseURL() string {
-	return fmt.Sprintf("http://%s", m.p2pMonolith.Addr())
+	return fmt.Sprintf("http://%s", m.listener.Addr().String())
 }
 
 func (m *DendriteMonolith) PeerCount(peertype int) int {
-	return m.p2pMonolith.Router.PeerCount(peertype)
+	return m.PineconeRouter.PeerCount(peertype)
 }
 
 func (m *DendriteMonolith) SessionCount() int {
-	return len(m.p2pMonolith.Sessions.Protocol(monolith.SessionProtocol).Sessions())
+	return len(m.PineconeQUIC.Protocol("matrix").Sessions())
 }
 
 type InterfaceInfo struct {
@@ -122,156 +142,55 @@ func (m *DendriteMonolith) RegisterNetworkCallback(intfCallback InterfaceRetriev
 		}
 		return intfs
 	}
-	m.p2pMonolith.Multicast.RegisterNetworkCallback(callback)
+	m.PineconeMulticast.RegisterNetworkCallback(callback)
 }
 
 func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
 	if enabled {
-		m.p2pMonolith.Multicast.Start()
+		m.PineconeMulticast.Start()
 	} else {
-		m.p2pMonolith.Multicast.Stop()
+		m.PineconeMulticast.Stop()
 		m.DisconnectType(int(pineconeRouter.PeerTypeMulticast))
 	}
 }
 
 func (m *DendriteMonolith) SetStaticPeer(uri string) {
-	m.p2pMonolith.ConnManager.RemovePeers()
+	m.PineconeManager.RemovePeers()
 	for _, uri := range strings.Split(uri, ",") {
-		m.p2pMonolith.ConnManager.AddPeer(strings.TrimSpace(uri))
+		m.PineconeManager.AddPeer(strings.TrimSpace(uri))
 	}
 }
 
-func getServerKeyFromString(nodeID string) (spec.ServerName, error) {
-	var nodeKey spec.ServerName
-	if userID, err := spec.NewUserID(nodeID, false); err == nil {
-		hexKey, decodeErr := hex.DecodeString(string(userID.Domain()))
-		if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
-			return "", fmt.Errorf("UserID domain is not a valid ed25519 public key: %v", userID.Domain())
-		} else {
-			nodeKey = userID.Domain()
-		}
-	} else {
-		hexKey, decodeErr := hex.DecodeString(nodeID)
-		if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
-			return "", fmt.Errorf("Relay server uri is not a valid ed25519 public key: %v", nodeID)
-		} else {
-			nodeKey = spec.ServerName(nodeID)
-		}
-	}
-
-	return nodeKey, nil
-}
-
-func (m *DendriteMonolith) SetRelayServers(nodeID string, uris string) {
-	relays := []spec.ServerName{}
-	for _, uri := range strings.Split(uris, ",") {
-		uri = strings.TrimSpace(uri)
-		if len(uri) == 0 {
-			continue
-		}
-
-		nodeKey, err := getServerKeyFromString(uri)
-		if err != nil {
-			logrus.Errorf(err.Error())
-			continue
-		}
-		relays = append(relays, nodeKey)
-	}
-
-	nodeKey, err := getServerKeyFromString(nodeID)
-	if err != nil {
-		logrus.Errorf(err.Error())
-		return
-	}
-
-	if string(nodeKey) == m.PublicKey() {
-		logrus.Infof("Setting own relay servers to: %v", relays)
-		m.p2pMonolith.RelayRetriever.SetRelayServers(relays)
-	} else {
-		relay.UpdateNodeRelayServers(
-			spec.ServerName(nodeKey),
-			relays,
-			m.p2pMonolith.ProcessCtx.Context(),
-			m.p2pMonolith.GetFederationAPI(),
-		)
-	}
-}
-
-func (m *DendriteMonolith) GetRelayServers(nodeID string) string {
-	nodeKey, err := getServerKeyFromString(nodeID)
-	if err != nil {
-		logrus.Errorf(err.Error())
-		return ""
-	}
-
-	relaysString := ""
-	if string(nodeKey) == m.PublicKey() {
-		relays := m.p2pMonolith.RelayRetriever.GetRelayServers()
-
-		for i, relay := range relays {
-			if i != 0 {
-				// Append a comma to the previous entry if there is one.
-				relaysString += ","
-			}
-			relaysString += string(relay)
-		}
-	} else {
-		request := api.P2PQueryRelayServersRequest{Server: spec.ServerName(nodeKey)}
-		response := api.P2PQueryRelayServersResponse{}
-		err := m.p2pMonolith.GetFederationAPI().P2PQueryRelayServers(m.p2pMonolith.ProcessCtx.Context(), &request, &response)
-		if err != nil {
-			logrus.Warnf("Failed obtaining list of this node's relay servers: %s", err.Error())
-			return ""
-		}
-
-		for i, relay := range response.RelayServers {
-			if i != 0 {
-				// Append a comma to the previous entry if there is one.
-				relaysString += ","
-			}
-			relaysString += string(relay)
-		}
-	}
-
-	return relaysString
-}
-
-func (m *DendriteMonolith) RelayingEnabled() bool {
-	return m.p2pMonolith.GetRelayAPI().RelayingEnabled()
-}
-
-func (m *DendriteMonolith) SetRelayingEnabled(enabled bool) {
-	m.p2pMonolith.GetRelayAPI().SetRelayingEnabled(enabled)
-}
-
 func (m *DendriteMonolith) DisconnectType(peertype int) {
-	for _, p := range m.p2pMonolith.Router.Peers() {
+	for _, p := range m.PineconeRouter.Peers() {
 		if int(peertype) == p.PeerType {
-			m.p2pMonolith.Router.Disconnect(types.SwitchPortID(p.Port), nil)
+			m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
 		}
 	}
 }
 
 func (m *DendriteMonolith) DisconnectZone(zone string) {
-	for _, p := range m.p2pMonolith.Router.Peers() {
+	for _, p := range m.PineconeRouter.Peers() {
 		if zone == p.Zone {
-			m.p2pMonolith.Router.Disconnect(types.SwitchPortID(p.Port), nil)
+			m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
 		}
 	}
 }
 
 func (m *DendriteMonolith) DisconnectPort(port int) {
-	m.p2pMonolith.Router.Disconnect(types.SwitchPortID(port), nil)
+	m.PineconeRouter.Disconnect(types.SwitchPortID(port), nil)
 }
 
 func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) {
 	l, r := net.Pipe()
-	newConduit := Conduit{conduit.NewConduit(r, 0)}
+	conduit := &Conduit{conn: r, port: 0}
 	go func() {
+		conduit.portMutex.Lock()
+		defer conduit.portMutex.Unlock()
+
 		logrus.Errorf("Attempting authenticated connect")
-		var port types.SwitchPortID
 		var err error
-		if port, err = m.p2pMonolith.Router.Connect(
+		if conduit.port, err = m.PineconeRouter.Connect(
 			l,
 			pineconeRouter.ConnectionZone(zone),
 			pineconeRouter.ConnectionPeerType(peertype),
@@ -279,20 +198,19 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error)
 			logrus.Errorf("Authenticated connect failed: %s", err)
 			_ = l.Close()
 			_ = r.Close()
-			_ = newConduit.Close()
+			_ = conduit.Close()
 			return
 		}
-		newConduit.SetPort(port)
-		logrus.Infof("Authenticated connect succeeded (port %d)", newConduit.Port())
+		logrus.Infof("Authenticated connect succeeded (port %d)", conduit.port)
 	}()
-	return &newConduit, nil
+	return conduit, nil
 }
 
 func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, error) {
-	pubkey := m.p2pMonolith.Router.PublicKey()
+	pubkey := m.PineconeRouter.PublicKey()
 	userID := userutil.MakeUserID(
 		localpart,
-		spec.ServerName(hex.EncodeToString(pubkey[:])),
+		gomatrixserverlib.ServerName(hex.EncodeToString(pubkey[:])),
 	)
 	userReq := &userapiAPI.PerformAccountCreationRequest{
 		AccountType: userapiAPI.AccountTypeUser,
@@ -300,7 +218,7 @@ func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, err
 		Password:    password,
 	}
 	userRes := &userapiAPI.PerformAccountCreationResponse{}
-	if err := m.p2pMonolith.GetUserAPI().PerformAccountCreation(context.Background(), userReq, userRes); err != nil {
+	if err := m.userAPI.PerformAccountCreation(context.Background(), userReq, userRes); err != nil {
 		return userID, fmt.Errorf("userAPI.PerformAccountCreation: %w", err)
 	}
 	return userID, nil
@@ -318,7 +236,7 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
 		AccessToken: hex.EncodeToString(accessTokenBytes[:n]),
 	}
 	loginRes := &userapiAPI.PerformDeviceCreationResponse{}
-	if err := m.p2pMonolith.GetUserAPI().PerformDeviceCreation(context.Background(), loginReq, loginRes); err != nil {
+	if err := m.userAPI.PerformDeviceCreation(context.Background(), loginReq, loginRes); err != nil {
 		return "", fmt.Errorf("userAPI.PerformDeviceCreation: %w", err)
 	}
 	if !loginRes.DeviceCreated {
@@ -327,10 +245,51 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
 	return loginRes.Device.AccessToken, nil
 }
 
+// nolint:gocyclo
 func (m *DendriteMonolith) Start() {
+	var sk ed25519.PrivateKey
+	var pk ed25519.PublicKey
+
 	keyfile := filepath.Join(m.StorageDirectory, "p2p.pem")
-	oldKeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
-	sk, pk := monolith.GetOrCreateKey(keyfile, oldKeyfile)
+	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
+		oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
+		if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
+			if err = test.NewMatrixKey(keyfile); err != nil {
+				panic("failed to generate a new PEM key: " + err.Error())
+			}
+			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+				panic("failed to load PEM key: " + err.Error())
+			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
+		} else {
+			if sk, err = os.ReadFile(oldkeyfile); err != nil {
+				panic("failed to read the old private key: " + err.Error())
+			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
+			if err = test.SaveMatrixKey(keyfile, sk); err != nil {
+				panic("failed to convert the private key to PEM format: " + err.Error())
+			}
+		}
+	} else {
+		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+			panic("failed to load PEM key: " + err.Error())
+		}
+		if len(sk) != ed25519.PrivateKeySize {
+			panic("the private key is not long enough")
+		}
+	}
+
+	pk = sk.Public().(ed25519.PublicKey)
+
+	var err error
+	m.listener, err = net.Listen("tcp", "localhost:65432")
+	if err != nil {
+		panic(err)
+	}
 
 	m.logger = logrus.Logger{
 		Out: BindLogger{},
@@ -338,29 +297,226 @@ func (m *DendriteMonolith) Start() {
 	m.logger.SetOutput(BindLogger{})
 	logrus.SetOutput(BindLogger{})
 
-	m.p2pMonolith = monolith.P2PMonolith{}
-	m.p2pMonolith.SetupPinecone(sk)
+	pineconeEventChannel := make(chan pineconeEvents.Event)
+	m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
+	m.PineconeRouter.EnableHopLimiting()
+	m.PineconeRouter.EnableWakeupBroadcasts()
+	m.PineconeRouter.Subscribe(pineconeEventChannel)
+
+	m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
+	m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
+	m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil)
 
 	prefix := hex.EncodeToString(pk)
-	cfg := monolith.GenerateDefaultConfig(sk, m.StorageDirectory, m.CacheDirectory, prefix)
-	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
+	cfg := &config.Dendrite{}
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.InMemory = false
-	// NOTE : disabled for now since there is a 64 bit alignment panic on 32 bit systems
-	// This isn't actually fixed: https://github.com/blevesearch/zapx/pull/147
-	cfg.SyncAPI.Fulltext.Enabled = false
+	cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(m.CacheDirectory, prefix))
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.MediaAPI.BasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
+	cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
+	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+	cfg.ClientAPI.RegistrationDisabled = false
+	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
+	cfg.SyncAPI.Fulltext.Enabled = true
+	cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(m.CacheDirectory, "search"))
+	if err = cfg.Derive(); err != nil {
+		panic(err)
+	}
 
-	processCtx := process.NewProcessContext()
-	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
-	routers := httputil.NewRouters()
+	base := base.NewBaseDendrite(cfg, "Monolith")
+	base.ConfigureAdminEndpoints()
+	defer base.Close() // nolint: errcheck
 
-	enableRelaying := false
-	enableMetrics := false
-	enableWebsockets := false
-	m.p2pMonolith.SetupDendrite(processCtx, cfg, cm, routers, 65432, enableRelaying, enableMetrics, enableWebsockets)
-	m.p2pMonolith.StartMonolith()
+	federation := conn.CreateFederationClient(base, m.PineconeQUIC)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+
+	rsAPI := roomserver.NewInternalAPI(base)
+
+	fsAPI := federationapi.NewInternalAPI(
+		base, federation, rsAPI, base.Caches, keyRing, true,
+	)
+
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
+	m.userAPI = userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
+	keyAPI.SetUserAPI(m.userAPI)
+
+	asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI)
+
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this dependency
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
+
+	userProvider := users.NewPineconeUserProvider(m.PineconeRouter, m.PineconeQUIC, m.userAPI, federation)
+	roomProvider := rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation)
+
+	monolith := setup.Monolith{
+		Config:    base.Cfg,
+		Client:    conn.CreateClient(base, m.PineconeQUIC),
+		FedClient: federation,
+		KeyRing:   keyRing,
+
+		AppserviceAPI:            asAPI,
+		FederationAPI:            fsAPI,
+		RoomserverAPI:            rsAPI,
+		UserAPI:                  m.userAPI,
+		KeyAPI:                   keyAPI,
+		ExtPublicRoomsProvider:   roomProvider,
+		ExtUserDirectoryProvider: userProvider,
+	}
+	monolith.AddAllPublicRoutes(base)
+
+	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
+	httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
+
+	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
+	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
+	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+
+	pHTTP := m.PineconeQUIC.Protocol("matrix").HTTP()
+	pHTTP.Mux().Handle(users.PublicURL, pMux)
+	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
+	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
+
+	// Build both ends of a HTTP multiplex.
+	h2s := &http2.Server{}
+	m.httpServer = &http.Server{
+		Addr:         ":0",
+		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+		ReadTimeout:  10 * time.Second,
+		WriteTimeout: 10 * time.Second,
+		IdleTimeout:  30 * time.Second,
+		BaseContext: func(_ net.Listener) context.Context {
+			return context.Background()
+		},
+		Handler: h2c.NewHandler(pMux, h2s),
+	}
+
+	m.processContext = base.ProcessContext
+
+	go func() {
+		m.logger.Info("Listening on ", cfg.Global.ServerName)
+
+		switch m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")) {
+		case net.ErrClosed, http.ErrServerClosed:
+			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
+		default:
+			m.logger.Fatal(err)
+		}
+	}()
+	go func() {
+		logrus.Info("Listening on ", m.listener.Addr())
+
+		switch http.Serve(m.listener, httpRouter) {
+		case net.ErrClosed, http.ErrServerClosed:
+			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
+		default:
+			m.logger.Fatal(err)
+		}
+	}()
+
+	go func(ch <-chan pineconeEvents.Event) {
+		eLog := logrus.WithField("pinecone", "events")
+
+		for event := range ch {
+			switch e := event.(type) {
+			case pineconeEvents.PeerAdded:
+			case pineconeEvents.PeerRemoved:
+			case pineconeEvents.TreeParentUpdate:
+			case pineconeEvents.SnakeDescUpdate:
+			case pineconeEvents.TreeRootAnnUpdate:
+			case pineconeEvents.SnakeEntryAdded:
+			case pineconeEvents.SnakeEntryRemoved:
+			case pineconeEvents.BroadcastReceived:
+				eLog.Info("Broadcast received from: ", e.PeerID)
+
+				req := &api.PerformWakeupServersRequest{
+					ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
+				}
+				res := &api.PerformWakeupServersResponse{}
+				if err := fsAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
+					logrus.WithError(err).Error("Failed to wakeup destination", e.PeerID)
+				}
+			case pineconeEvents.BandwidthReport:
+			default:
+			}
+		}
+	}(pineconeEventChannel)
 }
 
 func (m *DendriteMonolith) Stop() {
-	m.p2pMonolith.Stop()
+	m.processContext.ShutdownDendrite()
+	_ = m.listener.Close()
+	m.PineconeMulticast.Stop()
+	_ = m.PineconeQUIC.Close()
+	_ = m.PineconeRouter.Close()
+	m.processContext.WaitForComponentsToFinish()
+}
+
+const MaxFrameSize = types.MaxFrameSize
+
+type Conduit struct {
+	closed    atomic.Bool
+	conn      net.Conn
+	port      types.SwitchPortID
+	portMutex sync.Mutex
+}
+
+func (c *Conduit) Port() int {
+	c.portMutex.Lock()
+	defer c.portMutex.Unlock()
+	return int(c.port)
+}
+
+func (c *Conduit) Read(b []byte) (int, error) {
+	if c.closed.Load() {
+		return 0, io.EOF
+	}
+	return c.conn.Read(b)
+}
+
+func (c *Conduit) ReadCopy() ([]byte, error) {
+	if c.closed.Load() {
+		return nil, io.EOF
+	}
+	var buf [65535 * 2]byte
+	n, err := c.conn.Read(buf[:])
+	if err != nil {
+		return nil, err
+	}
+	return buf[:n], nil
+}
+
+func (c *Conduit) Write(b []byte) (int, error) {
+	if c.closed.Load() {
+		return 0, io.EOF
+	}
+	return c.conn.Write(b)
+}
+
+func (c *Conduit) Close() error {
+	if c.closed.Load() {
+		return io.ErrClosedPipe
+	}
+	c.closed.Store(true)
+	return c.conn.Close()
 }

build/gobind-pinecone/monolith_test.go

@@ -1,158 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gobind
-
-import (
-	"strings"
-	"testing"
-
-	"github.com/matrix-org/gomatrixserverlib/spec"
-)
-
-func TestMonolithStarts(t *testing.T) {
-	monolith := DendriteMonolith{
-		StorageDirectory: t.TempDir(),
-		CacheDirectory:   t.TempDir(),
-	}
-	monolith.Start()
-	monolith.PublicKey()
-	monolith.Stop()
-}
-
-func TestMonolithSetRelayServers(t *testing.T) {
-	testCases := []struct {
-		name           string
-		nodeID         string
-		relays         string
-		expectedRelays string
-		expectSelf     bool
-	}{
-		{
-			name:           "assorted valid, invalid, empty & self keys",
-			nodeID:         "@valid:abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
-			relays:         "@valid:123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd,@invalid:notakey,,",
-			expectedRelays: "123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
-			expectSelf:     true,
-		},
-		{
-			name:           "invalid node key",
-			nodeID:         "@invalid:notakey",
-			relays:         "@valid:123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd,@invalid:notakey,,",
-			expectedRelays: "",
-			expectSelf:     false,
-		},
-		{
-			name:           "node is self",
-			nodeID:         "self",
-			relays:         "@valid:123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd,@invalid:notakey,,",
-			expectedRelays: "123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
-			expectSelf:     false,
-		},
-	}
-
-	for _, tc := range testCases {
-		monolith := DendriteMonolith{
-			StorageDirectory: t.TempDir(),
-			CacheDirectory:   t.TempDir(),
-		}
-		monolith.Start()
-
-		inputRelays := tc.relays
-		expectedRelays := tc.expectedRelays
-		if tc.expectSelf {
-			inputRelays += "," + monolith.PublicKey()
-			expectedRelays += "," + monolith.PublicKey()
-		}
-		nodeID := tc.nodeID
-		if nodeID == "self" {
-			nodeID = monolith.PublicKey()
-		}
-
-		monolith.SetRelayServers(nodeID, inputRelays)
-		relays := monolith.GetRelayServers(nodeID)
-		monolith.Stop()
-
-		if !containSameKeys(strings.Split(relays, ","), strings.Split(expectedRelays, ",")) {
-			t.Fatalf("%s: expected %s got %s", tc.name, expectedRelays, relays)
-		}
-	}
-}
-
-func containSameKeys(expected []string, actual []string) bool {
-	if len(expected) != len(actual) {
-		return false
-	}
-
-	for _, expectedKey := range expected {
-		hasMatch := false
-		for _, actualKey := range actual {
-			if actualKey == expectedKey {
-				hasMatch = true
-			}
-		}
-
-		if !hasMatch {
-			return false
-		}
-	}
-
-	return true
-}
-
-func TestParseServerKey(t *testing.T) {
-	testCases := []struct {
-		name        string
-		serverKey   string
-		expectedErr bool
-		expectedKey spec.ServerName
-	}{
-		{
-			name:        "valid userid as key",
-			serverKey:   "@valid:abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
-			expectedErr: false,
-			expectedKey: "abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
-		},
-		{
-			name:        "valid key",
-			serverKey:   "abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
-			expectedErr: false,
-			expectedKey: "abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
-		},
-		{
-			name:        "invalid userid key",
-			serverKey:   "@invalid:notakey",
-			expectedErr: true,
-			expectedKey: "",
-		},
-		{
-			name:        "invalid key",
-			serverKey:   "@invalid:notakey",
-			expectedErr: true,
-			expectedKey: "",
-		},
-	}
-
-	for _, tc := range testCases {
-		key, err := getServerKeyFromString(tc.serverKey)
-		if tc.expectedErr && err == nil {
-			t.Fatalf("%s: expected an error", tc.name)
-		} else if !tc.expectedErr && err != nil {
-			t.Fatalf("%s: didn't expect an error: %s", tc.name, err.Error())
-		}
-		if tc.expectedKey != key {
-			t.Fatalf("%s: keys not equal. expected: %s got: %s", tc.name, tc.expectedKey, key)
-		}
-	}
-}

build/gobind-yggdrasil/monolith.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/getsentry/sentry-go"
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
@@ -20,20 +19,16 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 
 	_ "golang.org/x/mobile/bind"
@@ -132,10 +127,10 @@ func (m *DendriteMonolith) Start() {
 
 	cfg := &config.Dendrite{}
 	cfg.Defaults(config.DefaultOpts{
-		Generate:       true,
-		SingleDatabase: true,
+		Generate:   true,
+		Monolithic: true,
 	})
-	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory))
@@ -154,71 +149,27 @@ func (m *DendriteMonolith) Start() {
 		panic(err)
 	}
 
-	configErrors := &config.ConfigErrors{}
-	cfg.Verify(configErrors)
-	if len(*configErrors) > 0 {
-		for _, err := range *configErrors {
-			logrus.Errorf("Configuration error: %s", err)
-		}
-		logrus.Fatalf("Failed to start due to configuration errors")
-	}
+	base := base.NewBaseDendrite(cfg, "Monolith")
+	base.ConfigureAdminEndpoints()
+	m.processContext = base.ProcessContext
+	defer base.Close() // nolint: errcheck
 
-	internal.SetupStdLogging()
-	internal.SetupHookLogging(cfg.Logging)
-	internal.SetupPprof()
-
-	logrus.Infof("Dendrite version %s", internal.VersionString())
-
-	if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled {
-		logrus.Warn("Open registration is enabled")
-	}
-
-	closer, err := cfg.SetupTracing()
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to start opentracing")
-	}
-	defer closer.Close()
-
-	if cfg.Global.Sentry.Enabled {
-		logrus.Info("Setting up Sentry for debugging...")
-		err = sentry.Init(sentry.ClientOptions{
-			Dsn:              cfg.Global.Sentry.DSN,
-			Environment:      cfg.Global.Sentry.Environment,
-			Debug:            true,
-			ServerName:       string(cfg.Global.ServerName),
-			Release:          "dendrite@" + internal.VersionString(),
-			AttachStacktrace: true,
-		})
-		if err != nil {
-			logrus.WithError(err).Panic("failed to start Sentry")
-		}
-	}
-	processCtx := process.NewProcessContext()
-	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
-	routers := httputil.NewRouters()
-	basepkg.ConfigureAdminEndpoints(processCtx, routers)
-	m.processContext = processCtx
-	defer func() {
-		processCtx.ShutdownDendrite()
-		processCtx.WaitForShutdown()
-	}() // nolint: errcheck
-
-	federation := ygg.CreateFederationClient(cfg)
+	federation := ygg.CreateFederationClient(base)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
-	natsInstance := jetstream.NATSInstance{}
-	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
+	rsAPI := roomserver.NewInternalAPI(base)
 
 	fsAPI := federationapi.NewInternalAPI(
-		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
+		base, federation, rsAPI, base.Caches, keyRing, true,
 	)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
+	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
+	keyAPI.SetUserAPI(userAPI)
 
-	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
@@ -226,8 +177,8 @@ func (m *DendriteMonolith) Start() {
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{
-		Config:    cfg,
-		Client:    ygg.CreateClient(),
+		Config:    base.Cfg,
+		Client:    ygg.CreateClient(base),
 		FedClient: federation,
 		KeyRing:   keyRing,
 
@@ -235,21 +186,23 @@ func (m *DendriteMonolith) Start() {
 		FederationAPI: fsAPI,
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
+		KeyAPI:        keyAPI,
 		ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider(
 			ygg, fsAPI, federation,
 		),
 	}
-	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
+	monolith.AddAllPublicRoutes(base)
 
 	httpRouter := mux.NewRouter()
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
-	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin)
-	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin)
+	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 
 	yggRouter := mux.NewRouter()
-	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(routers.Federation)
-	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
+	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
+	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 
 	// Build both ends of a HTTP multiplex.
 	m.httpServer = &http.Server{

build/scripts/Complement.Dockerfile

@@ -1,6 +1,6 @@
 #syntax=docker/dockerfile:1.2
 
-FROM golang:1.20-bullseye as build
+FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y sqlite3
 WORKDIR /build
 
@@ -16,16 +16,13 @@ RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
-    CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
-    CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
-    cp build/scripts/complement-cmd.sh /complement-cmd.sh
+    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
 
 ENV SERVER_NAME=localhost
 ENV API=0
-ENV COVER=0
 EXPOSE 8008 8448
 
 # At runtime, generate TLS cert based on the CA now mounted at /ca
@@ -33,4 +30,4 @@ EXPOSE 8008 8448
 CMD ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    exec /complement-cmd.sh
+    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

build/scripts/ComplementLocal.Dockerfile

@@ -12,20 +12,18 @@ FROM golang:1.18-stretch
 RUN apt-get update && apt-get install -y sqlite3
 
 ENV SERVER_NAME=localhost
-ENV COVER=0
 EXPOSE 8008 8448
 
 WORKDIR /runtime
 # This script compiles Dendrite for us.
 RUN echo '\
     #!/bin/bash -eux \n\
-    if test -f "/runtime/dendrite" && test -f "/runtime/dendrite-cover"; then \n\
+    if test -f "/runtime/dendrite-monolith-server"; then \n\
     echo "Skipping compilation; binaries exist" \n\
     exit 0 \n\
     fi \n\
     cd /dendrite \n\
-    go build -v -o /runtime /dendrite/cmd/dendrite \n\
-    go test -c -cover -covermode=atomic -o /runtime/dendrite-cover -coverpkg "github.com/matrix-org/..." /dendrite/cmd/dendrite \n\
+    go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
     ' > compile.sh && chmod +x compile.sh
 
 # This script runs Dendrite for us. Must be run in the /runtime directory.
@@ -35,8 +33,7 @@ RUN echo '\
     ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
-    [ ${COVER} -eq 1 ] && exec ./dendrite-cover --test.coverprofile=integrationcover.log --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
-    exec ./dendrite --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
     ' > run.sh && chmod +x run.sh
 
 

build/scripts/ComplementPostgres.Dockerfile

@@ -1,19 +1,19 @@
 #syntax=docker/dockerfile:1.2
 
-FROM golang:1.20-bullseye as build
+FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 
 # No password when connecting over localhost
-RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/13/main/pg_hba.conf && \
+RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/9.6/main/pg_hba.conf && \
     # Bump up max conns for moar concurrency
-    sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/13/main/postgresql.conf
+    sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
     #!/bin/bash -eu \n\
     pg_lsclusters \n\
-    pg_ctlcluster 13 main start \n\
+    pg_ctlcluster 9.6 main start \n\
     \n\
     until pg_isready \n\
     do \n\
@@ -34,16 +34,13 @@ RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
-    CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
-    CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
-    cp build/scripts/complement-cmd.sh /complement-cmd.sh
+    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
 
 ENV SERVER_NAME=localhost
 ENV API=0
-ENV COVER=0
 EXPOSE 8008 8448
 
 
@@ -54,4 +51,4 @@ CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NA
     # Bump max_open_conns up here in the global database config
     sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    exec /complement-cmd.sh
+    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

build/scripts/complement-cmd.sh

@@ -1,21 +0,0 @@
-#!/bin/bash -e
-
-# This script is intended to be used inside a docker container for Complement
-
-export GOCOVERDIR=/tmp/covdatafiles
-mkdir -p "${GOCOVERDIR}"
-if [[ "${COVER}" -eq 1 ]]; then
-  echo "Running with coverage"
-  exec /dendrite/dendrite-cover \
-    --really-enable-open-registration \
-    --tls-cert server.crt \
-    --tls-key server.key \
-    --config dendrite.yaml
-else
-  echo "Not running with coverage"
-  exec /dendrite/dendrite \
-    --really-enable-open-registration \
-    --tls-cert server.crt \
-    --tls-key server.key \
-    --config dendrite.yaml
-fi

build/scripts/complement.sh

@@ -15,5 +15,5 @@ tar -xzf master.tar.gz
 
 # Run the tests!
 cd complement-master
-COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi
+COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests
 

charts/dendrite/Chart.yaml

@@ -0,0 +1,81 @@
+---
+apiVersion: v2
+appVersion: v0.9.4
+description: Dendrite Matrix Homeserver
+name: dendrite
+version: 7.1.2
+kubeVersion: ">=1.19.0-0"
+keywords:
+  - dendrite
+  - matrix
+  - homeserver
+  - monolith
+  - federation
+  - polylith
+home: https://github.com/samipsolutions/helm-charts/tree/master/charts/stable/dendrite
+maintainers:
+  - name: Skyler Mäntysaari
+    url: https://github.com/samip5
+sources:
+  - https://github.com/matrix-org/dendrite
+  - https://github.com/matrix-org/dendrite/tree/master/build/docker
+dependencies:
+  - name: common
+    repository: https://bjw-s.github.io/helm-charts/
+    version: 0.1.0
+  - name: nats
+    version: 0.17.5
+    repository: https://nats-io.github.io/k8s/helm/charts/
+    condition: nats.enabled
+  # Client API
+  - name: common
+    repository: https://bjw-s.github.io/helm-charts/
+    version: 0.1.0
+    alias: clientapi
+    condition: dendrite.polylithEnabled
+  # Media API
+  - name: common
+    repository: https://bjw-s.github.io/helm-charts/
+    version: 0.1.0
+    alias: mediaapi
+    condition: dendrite.polylithEnabled
+  # Sync API
+  - name: common
+    repository: https://bjw-s.github.io/helm-charts/
+    version: 0.1.0
+    alias: syncapi
+    condition: dendrite.polylithEnabled
+  # Room Server
+  - name: common
+    repository: https://bjw-s.github.io/helm-charts/
+    version: 0.1.0
+    alias: roomserver
+    condition: dendrite.polylithEnabled
+  # Federation API
+  - name: common
+    repository: https://bjw-s.github.io/helm-charts/
+    version: 0.1.0
+    alias: federationapi
+    condition: dendrite.polylithEnabled
+  # Key Server
+  - name: common
+    repository: https://bjw-s.github.io/helm-charts/
+    version: 0.1.0
+    alias: keyserver
+    condition: dendrite.polylithEnabled
+  # User API
+  - name: common
+    repository: https://bjw-s.github.io/helm-charts/
+    version: 0.1.0
+    alias: userapi
+    condition: dendrite.polylithEnabled
+  # App Service API
+  - name: common
+    repository: https://bjw-s.github.io/helm-charts/
+    version: 0.1.0
+    alias: appserviceapi
+    condition: dendrite.polylithEnabled
+annotations:
+  artifacthub.io/changes: |-
+    - kind: changed
+      description: Upgrade nats chart dep.

charts/dendrite/README.md

@@ -0,0 +1,257 @@
+# dendrite
+
+![Version: 7.1.1](https://img.shields.io/badge/Version-7.1.1-informational?style=flat-square) ![AppVersion: v0.9.4](https://img.shields.io/badge/AppVersion-v0.9.4-informational?style=flat-square)
+
+Dendrite Matrix Homeserver
+
+**This chart is not maintained by the upstream project and any issues with the chart should be raised [here](https://github.com/samipsolutions/helm-charts/issues/new/choose)**
+
+## Source Code
+
+* <https://github.com/matrix-org/dendrite>
+* <https://github.com/matrix-org/dendrite/tree/master/build/docker>
+
+## Requirements
+
+Kubernetes: `>=1.19.0-0`
+
+## Dependencies
+
+| Repository | Name | Version |
+|------------|------|---------|
+| https://bjw-s.github.io/helm-charts/ | common | 0.1.0 |
+| https://bjw-s.github.io/helm-charts/ | keyserver(common) | 0.1.0 |
+| https://bjw-s.github.io/helm-charts/ | clientapi(common) | 0.1.0 |
+| https://bjw-s.github.io/helm-charts/ | mediaapi(common) | 0.1.0 |
+| https://bjw-s.github.io/helm-charts/ | syncapi(common) | 0.1.0 |
+| https://bjw-s.github.io/helm-charts/ | roomserver(common) | 0.1.0 |
+| https://bjw-s.github.io/helm-charts/ | federationapi(common) | 0.1.0 |
+| https://bjw-s.github.io/helm-charts/ | userapi(common) | 0.1.0 |
+| https://bjw-s.github.io/helm-charts/ | appserviceapi(common) | 0.1.0 |
+| https://nats-io.github.io/k8s/helm/charts/ | nats | 0.17.1 |
+
+## TL;DR
+
+```console
+helm repo add samipsolutions https://helm.samipsolutions.fi/
+helm repo update
+helm install dendrite samipsolutions/dendrite
+```
+
+## Installing the Chart
+
+To install the chart with the release name `dendrite`
+
+```console
+helm install dendrite samipsolutions/dendrite
+```
+
+## Uninstalling the Chart
+
+To uninstall the `dendrite` deployment
+
+```console
+helm uninstall dendrite
+```
+
+The command removes all the Kubernetes components associated with the chart **including persistent volumes** and deletes the release.
+
+## Configuration
+
+Read through the [values.yaml](./values.yaml) file. It has several commented out suggested values.
+Other values may be used from the [values.yaml](https://github.com/k8s-at-home/library-charts/tree/main/charts/stable/common/values.yaml) from the [common library](https://github.com/k8s-at-home/library-charts/tree/main/charts/stable/common).
+
+Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`.
+
+```console
+helm install dendrite \
+  --set env.TZ="America/New York" \
+    samipsolutions/dendrite
+```
+
+Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart.
+
+```console
+helm install dendrite samipsolutions/dendrite -f values.yaml
+```
+
+## Custom configuration
+
+### Polylith Ingress
+
+Due to the complexity of setting up ingress for each individual component it
+is left up to the individual to add the necessary ingress fields to polylith deployments.
+
+For more information see:
+- https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#nginx-or-other-reverse-proxy
+- and https://github.com/matrix-org/dendrite/blob/master/docs/nginx/polylith-sample.conf
+
+## Values
+
+**Important**: When deploying an application Helm chart you can add more values from our common library chart [here](https://github.com/k8s-at-home/library-charts/tree/main/charts/stable/common)
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| appserviceapi | object | See values.yaml | Configure the app service api. For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| appserviceapi.database | object | See values.yaml | Override general dendrite.database parameters. |
+| appserviceapi.database.conn_max_lifetime | string | dendrite.database.conn_max_lifetime | Maximum connection lifetime |
+| appserviceapi.database.connection_string | string | file or derived from included postgresql deployment | Custom connection string |
+| appserviceapi.database.max_idle_conns | string | dendrite.database.max_idle_conns | Maximum dile connections |
+| appserviceapi.database.max_open_conns | string | dendrite.database.max_open_conns | Maximum open connections |
+| appserviceapi.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| appserviceapi.image.repository | string | `"matrixdotorg/dendrite-polylith"` | image repository |
+| appserviceapi.image.tag | string | chart.appVersion | image tag |
+| clientapi | object | See values.yaml | Configuration for the client api component. For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| clientapi.config.captcha | object | See values.yaml | Configure captcha for registration |
+| clientapi.config.rate_limiting | object | values.yaml | Configure rate limiting. |
+| clientapi.config.registration_disabled | bool | `true` | Enable or disable registration for this homeserver. |
+| clientapi.config.registration_shared_secret | string | `""` | Shared secret that allows registration, despite registration_disabled. |
+| clientapi.config.turn | object | See values.yaml | Configure TURN |
+| clientapi.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| clientapi.image.repository | string | `"matrixdotorg/dendrite-polylith"` | image repository |
+| clientapi.image.tag | string | chart.appVersion | image tag |
+| database.conn_max_lifetime | int | `-1` |  |
+| database.connection_string | string | `"file:dendrite?sslmode=disable"` |  |
+| database.max_idle_conns | int | `2` |  |
+| database.max_open_conns | int | `100` |  |
+| dendrite | object | See values.yaml | Configuration for Dendrite. For more information see [the sample denrite-config.yaml](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| dendrite.global | object | See values.yaml | Configure the global settings for dendrite. |
+| dendrite.global.cache | object | `{"max_age":"1h","max_size_estimated":"1gb"}` | Congigure the in-memory caches |
+| dendrite.global.cache.max_age | string | `"1h"` | The maximum amount of time that a cache entry can live for in memory |
+| dendrite.global.cache.max_size_estimated | string | `"1gb"` | Configure the maximum estimated cache size (not a hard limit) |
+| dendrite.global.disable_federation | bool | `false` | Disables federation |
+| dendrite.global.dns_cache | object | See values.yaml | Configure DNS cache. |
+| dendrite.global.dns_cache.enabled | bool | See values.yaml | If enabled, dns cache will be enabled. |
+| dendrite.global.key_validity_period | string | `"168h0m0s"` | Configure the key_validity period |
+| dendrite.global.metrics | object | See values.yaml | Configure prometheus metrics collection for dendrite. |
+| dendrite.global.metrics.enabled | bool | See values.yaml | If enabled, metrics collection will be enabled |
+| dendrite.global.mscs | list | `[]` | Configure experimental MSC's |
+| dendrite.global.presence | object | `{"enable_inbound":false,"enable_outbound":false}` | Configure handling of presence events |
+| dendrite.global.presence.enable_inbound | bool | `false` | Whether inbound presence events are allowed, e.g. receiving presence events from other servers |
+| dendrite.global.presence.enable_outbound | bool | `false` | Whether outbound presence events are allowed, e.g. sending presence events to other servers |
+| dendrite.global.server_name | string | `"localhost"` | (required) Configure the server name for the dendrite instance. |
+| dendrite.global.server_notices | object | `{"avatar_url":"","display_name":"Server alerts","enabled":false,"local_part":"_server","room_name":"Server Alerts"}` | Server notices allows server admins to send messages to all users. |
+| dendrite.global.server_notices.avatar_url | string | `""` | The mxid of the avatar to use |
+| dendrite.global.server_notices.display_name | string | `"Server alerts"` | The displayname to be used when sending notices |
+| dendrite.global.server_notices.local_part | string | `"_server"` | The server localpart to be used when sending notices, ensure this is not yet taken |
+| dendrite.global.server_notices.room_name | string | `"Server Alerts"` | The roomname to be used when creating messages |
+| dendrite.global.trusted_third_party_id_servers | list | `["matrix.org","vector.im"]` | Configure the list of domains the server will trust as identity servers |
+| dendrite.global.well_known_client_name | string | `""` | Configure the well-known client name and optional port |
+| dendrite.global.well_known_server_name | string | `""` | Configure the well-known server name and optional port |
+| dendrite.logging | list | See values.yaml | Configure logging. |
+| dendrite.matrix_key_secret.create | bool | `false` | Create matrix_key secret using the keyBody below. |
+| dendrite.matrix_key_secret.existingSecret | string | `""` | Use an existing secret |
+| dendrite.matrix_key_secret.keyBody | string | `""` | New Key Body |
+| dendrite.matrix_key_secret.secretPath | string | `"matrix_key.pem"` | Field in the secret to get the key from |
+| dendrite.polylithEnabled | bool | `false` | Enable polylith deployment |
+| dendrite.polylith_ingress | object | See values.yaml | Enable and configure polylith ingress as per https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf |
+| dendrite.polylith_ingress.syncapi_paths | list | See values.yaml | Sync API Paths are a little tricky since they require regular expressions. Therefore the paths will depend on the ingress controller used. See values.yaml for nginx and traefik. |
+| dendrite.report_stats | object | `{"enabled":false,"endpoint":""}` | Usage statistics reporting configuration |
+| dendrite.report_stats.enabled | bool | false | Enable or disable usage reporting |
+| dendrite.report_stats.endpoint | string | `""` | Push endpoint for usage statistics |
+| dendrite.tls_secret | object | See values.yaml | If enabled, use an existing secrets for the TLS certificate and key. Otherwise, to enable TLS a `server.crt` and `server.key` must be mounted at `/etc/dendrite`. |
+| dendrite.tracing | object | See values.yaml | Configure opentracing. |
+| federationapi | object | values.yaml | Configure the Federation API For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| federationapi.database | object | See values.yaml | Override general dendrite.database parameters. |
+| federationapi.database.conn_max_lifetime | string | dendrite.database.conn_max_lifetime | Maximum connection lifetime |
+| federationapi.database.connection_string | string | file or derived from included postgresql deployment | Custom connection string |
+| federationapi.database.max_idle_conns | string | dendrite.database.max_idle_conns | Maximum dile connections |
+| federationapi.database.max_open_conns | string | dendrite.database.max_open_conns | Maximum open connections |
+| federationapi.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| federationapi.image.repository | string | `"matrixdotorg/dendrite-polylith"` | image repository |
+| federationapi.image.tag | string | chart.appVersion | image tag |
+| image | object | `{"pullPolicy":"IfNotPresent","repository":"ghcr.io/matrix-org/dendrite-monolith","tag":null}` |  IMPORTANT NOTE This chart inherits from our common library chart. You can check the default values/options here: https://github.com/k8s-at-home/library-charts/tree/main/charts/stable/common/values.yaml |
+| image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| image.repository | string | `"ghcr.io/matrix-org/dendrite-monolith"` | image repository |
+| image.tag | string | chart.appVersion | image tag |
+| ingress.main | object | See values.yaml | (Monolith Only) Enable and configure ingress settings for the chart under this key. |
+| keyserver | object | See values.yaml | Configure the key server. For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| keyserver.database | object | See values.yaml | Override general dendrite.database parameters. |
+| keyserver.database.conn_max_lifetime | string | dendrite.database.conn_max_lifetime | Maximum connection lifetime |
+| keyserver.database.connection_string | string | file or derived from included postgresql deployment | Custom connection string |
+| keyserver.database.max_idle_conns | string | dendrite.database.max_idle_conns | Maximum dile connections |
+| keyserver.database.max_open_conns | string | dendrite.database.max_open_conns | Maximum open connections |
+| keyserver.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| keyserver.image.repository | string | `"matrixdotorg/dendrite-polylith"` | image repository |
+| keyserver.image.tag | string | chart.appVersion | image tag |
+| mediaapi | object | values.yaml | Configure the Media API For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| mediaapi.database | object | See values.yaml | Override general dendrite.database parameters. |
+| mediaapi.database.conn_max_lifetime | string | dendrite.database.conn_max_lifetime | Maximum connection lifetime |
+| mediaapi.database.connection_string | string | file or derived from included postgresql deployment | Custom connection string |
+| mediaapi.database.max_idle_conns | string | dendrite.database.max_idle_conns | Maximum dile connections |
+| mediaapi.database.max_open_conns | string | dendrite.database.max_open_conns | Maximum open connections |
+| mediaapi.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| mediaapi.image.repository | string | `"matrixdotorg/dendrite-polylith"` | image repository |
+| mediaapi.image.tag | string | chart.appVersion | image tag |
+| mscs | object | values.yaml | Configuration for experimental MSCs For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| mscs.database | object | See values.yaml | Override general dendrite.database parameters. |
+| mscs.database.conn_max_lifetime | string | dendrite.database.conn_max_lifetime | Maximum connection lifetime |
+| mscs.database.connection_string | string | file or derived from included postgresql deployment | Custom connection string |
+| mscs.database.max_idle_conns | string | dendrite.database.max_idle_conns | Maximum dile connections |
+| mscs.database.max_open_conns | string | dendrite.database.max_open_conns | Maximum open connections |
+| nats.enabled | bool | See value.yaml | Enable and configure NATS for dendrite. Can be disabled for monolith deployments - an internal NATS server will be used in its place. |
+| nats.nats.image | string | `"nats:2.7.1-alpine"` |  |
+| nats.nats.jetstream.enabled | bool | `true` |  |
+| persistence | object | See values.yaml | Configure persistence settings for the chart under this key. |
+| persistence.jetstream | object | See values.yaml | Configure Jetsream persistence. This is highly recommended in production. |
+| roomserver | object | values.yaml | Configure the Room Server For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| roomserver.database | object | See values.yaml | Override general dendrite.database parameters. |
+| roomserver.database.conn_max_lifetime | string | dendrite.database.conn_max_lifetime | Maximum connection lifetime |
+| roomserver.database.connection_string | string | file or derived from included postgresql deployment | Custom connection string |
+| roomserver.database.max_idle_conns | string | dendrite.database.max_idle_conns | Maximum dile connections |
+| roomserver.database.max_open_conns | string | dendrite.database.max_open_conns | Maximum open connections |
+| roomserver.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| roomserver.image.repository | string | `"matrixdotorg/dendrite-polylith"` | image repository |
+| roomserver.image.tag | string | chart.appVersion | image tag |
+| service | object | See values.yaml | If added dendrite will start a HTTP and HTTPS listener args:   - "--tls-cert=server.crt"   - "--tls-key=server.key" -- Configures service settings for the chart. |
+| service.main.ports.http | object | See values.yaml | Configures the default HTTP listener for dendrite |
+| service.main.ports.https | object | See values.yaml | Configures the HTTPS listener for dendrite |
+| syncapi | object | values.yaml | Configure the Sync API For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| syncapi.database | object | See values.yaml | Override general dendrite.database parameters. |
+| syncapi.database.conn_max_lifetime | string | dendrite.database.conn_max_lifetime | Maximum connection lifetime |
+| syncapi.database.connection_string | string | file or derived from included postgresql deployment | Custom connection string |
+| syncapi.database.max_idle_conns | string | dendrite.database.max_idle_conns | Maximum dile connections |
+| syncapi.database.max_open_conns | string | dendrite.database.max_open_conns | Maximum open connections |
+| syncapi.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| syncapi.image.repository | string | `"matrixdotorg/dendrite-polylith"` | image repository |
+| syncapi.image.tag | string | chart.appVersion | image tag |
+| userapi | object | values.yaml | Configure the User API For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) |
+| userapi.config.bcrypt_cost | int | 10 | bcrypt cost (2^[cost] = rounds) |
+| userapi.database | object | See values.yaml | Override general dendrite.database parameters. |
+| userapi.database.conn_max_lifetime | string | dendrite.database.conn_max_lifetime | Maximum connection lifetime |
+| userapi.database.connection_string | string | file or derived from included postgresql deployment | Custom connection string |
+| userapi.database.max_idle_conns | string | dendrite.database.max_idle_conns | Maximum dile connections |
+| userapi.database.max_open_conns | string | dendrite.database.max_open_conns | Maximum open connections |
+| userapi.image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
+| userapi.image.repository | string | `"matrixdotorg/dendrite-polylith"` | image repository |
+| userapi.image.tag | string | chart.appVersion | image tag |
+
+## Changelog
+
+### Version 7.1.1
+
+#### Added
+
+N/A
+
+#### Changed
+
+N/A
+
+#### Fixed
+
+* Global database config
+
+### Older versions
+
+A historical overview of changes can be found on [ArtifactHUB](https://artifacthub.io/packages/helm/samipsolutions/dendrite?modal=changelog)
+
+## Support
+
+- See the [Docs](https://docs.k8s-at-home.com/our-helm-charts/getting-started/)
+- Open an [issue](https://github.com/samipsolutions/helm-charts/issues/new/choose)
+- Ask a [question](https://github.com/k8s-at-home/organization/discussions)
+- Join our [Discord](https://discord.gg/sTMX7Vh) community
+
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v0.1.1](https://github.com/k8s-at-home/helm-docs/releases/v0.1.1)

charts/dendrite/README_CONFIG.md.gotmpl

@@ -0,0 +1,17 @@
+{{- define "custom.custom.configuration.header" -}}
+## Custom configuration
+{{- end -}}
+
+{{- define "custom.custom.configuration" -}}
+{{ template "custom.custom.configuration.header" . }}
+
+### Polylith Ingress
+
+Due to the complexity of setting up ingress for each individual component it
+is left up to the individual to add the necessary ingress fields to polylith deployments.
+
+For more information see:
+- https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#nginx-or-other-reverse-proxy
+- and https://github.com/matrix-org/dendrite/blob/master/docs/nginx/polylith-sample.conf
+
+{{- end -}}

charts/dendrite/ci/ct-values.yaml

@@ -0,0 +1,10 @@
+---
+dendrite:
+  matrix_key_secret:
+    create: true
+    keyBody: |
+      -----BEGIN MATRIX PRIVATE KEY-----
+      Key-ID: ed25519:P8gZqV
+
+      qVzy2Cwokt15RjGy8OzFSq6z0JFmI6QX/1Zw1VP73uU=
+      -----END MATRIX PRIVATE KEY-----

charts/dendrite/ci/nats-values.yaml

@@ -0,0 +1,12 @@
+---
+dendrite:
+  matrix_key_secret:
+    create: true
+    keyBody: |
+      -----BEGIN MATRIX PRIVATE KEY-----
+      Key-ID: ed25519:P8gZqV
+
+      qVzy2Cwokt15RjGy8OzFSq6z0JFmI6QX/1Zw1VP73uU=
+      -----END MATRIX PRIVATE KEY-----
+nats:
+  enabled: true

charts/dendrite/ci/polylith-basic-values.yaml

@@ -0,0 +1,13 @@
+---
+dendrite:
+  polylithEnabled: true
+  matrix_key_secret:
+    create: true
+    keyBody: |
+      -----BEGIN MATRIX PRIVATE KEY-----
+      Key-ID: ed25519:P8gZqV
+
+      qVzy2Cwokt15RjGy8OzFSq6z0JFmI6QX/1Zw1VP73uU=
+      -----END MATRIX PRIVATE KEY-----
+nats:
+  enabled: true

charts/dendrite/ci/polylith-full-values.yaml

@@ -0,0 +1,19 @@
+---
+dendrite:
+  polylithEnabled: true
+  matrix_key_secret:
+    create: true
+    keyBody: |
+      -----BEGIN MATRIX PRIVATE KEY-----
+      Key-ID: ed25519:P8gZqV
+
+      qVzy2Cwokt15RjGy8OzFSq6z0JFmI6QX/1Zw1VP73uU=
+      -----END MATRIX PRIVATE KEY-----
+  polylith_ingress:
+    enabled: true
+    host: matrix.k8s-at-home.org
+nats:
+  enabled: true
+persistence:
+  jetstream:
+    enabled: true

charts/dendrite/templates/NOTES.txt

@@ -0,0 +1 @@
+{{- include "common.notes.defaultNotes" . -}}

charts/dendrite/templates/_helper.tpl

@@ -0,0 +1,3 @@
+{{- define "dendrite.names.key" -}}
+    {{- default (printf "%s-key" (include "common.names.fullname" .)) .Values.dendrite.matrix_key_secret.existingSecret -}}
+{{- end -}}

charts/dendrite/templates/common.yaml

@@ -0,0 +1,77 @@
+{{- if .Values.dendrite.polylithEnabled }}
+  {{ $components := list "clientapi" "appserviceapi" "federationapi" "userapi" "keyserver" "mediaapi" "syncapi" "roomserver" }}
+  {{- range $components }}
+    {{- include "common.values.setup" (index $.Subcharts .) }}
+    {{- with (index $.Values .) }}
+      {{- with .image }}
+        {{- $_ := set . "tag" (default $.Chart.AppVersion .tag) -}}
+      {{- end -}}
+      {{- if not .persistence }}
+        {{- $_ := set . "persistence" (dict)}}
+      {{- end }}
+      {{- $_ := set .persistence "dendrite-key" (include "dendrite.keyVolume" $ | fromYaml) -}}
+      {{- $_ := set .persistence "dendrite-config" (include "dendrite.configVolume" $ | fromYaml) -}}
+      {{- $_ := set .persistence "dendrite-tls" (include "dendrite.tlsVolume" $ | fromYaml) -}}
+      {{- $_ := set .persistence "jetstream" $.Values.persistence.jetstream -}}
+    {{- end }}
+    {{- include "common.all" (index $.Subcharts .) }}
+  {{- end }}
+  {{- with (index $.Values "mediaapi") }}
+    {{- $_ := set .persistence "media" $.Values.persistence.media -}}
+  {{- end }}
+{{- else }}
+  {{ include "common.values.setup" . }}
+  {{- $_ := set .Values.persistence "dendrite-key" (include "dendrite.keyVolume" . | fromYaml) -}}
+  {{- $_ := set .Values.persistence "dendrite-config" (include "dendrite.configVolume" . | fromYaml) -}}
+  {{- $_ := set .Values.persistence "dendrite-tls" (include "dendrite.tlsVolume" . | fromYaml) -}}
+  {{ include "common.all" . }}
+{{- end }}
+{{- define "dendrite.hardcodedValues" -}}
+probes:
+  liveness:
+    enabled: true
+    custom: true
+    spec:
+      initialDelaySeconds: 10
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 10
+      httpGet:
+        path: /_dendrite/monitor/health
+        {{- if .Values.dendrite.polylithEnabled }}
+        port: internal
+        {{ else }}
+        port: http
+        {{ end }}
+  readiness:
+    enabled: true
+    custom: true
+    spec:
+      initialDelaySeconds: 10
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 10
+      httpGet:
+        path: /_dendrite/monitor/health
+        {{- if .Values.dendrite.polylithEnabled }}
+        port: internal
+        {{ else }}
+        port: http
+        {{ end }}
+  startup:
+    enabled: true
+    custom: true
+    spec:
+      initialDelaySeconds: 5
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 10
+      httpGet:
+        path: /_dendrite/monitor/up
+        {{- if .Values.dendrite.polylithEnabled }}
+        port: internal
+        {{ else }}
+        port: http
+        {{ end }}
+{{- end -}}
+{{- $_ := mergeOverwrite .Values (include "dendrite.hardcodedValues" . | fromYaml) -}}

charts/dendrite/templates/dendrite-config.yaml

@@ -0,0 +1,208 @@
+{{- $componentSpecificDatabaseConfig := or .Values.dendrite.polylithEnabled -}}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "common.names.fullname" . }}-config
+type: Opaque
+stringData:
+  dendrite.yaml: |
+    version: 2
+    global:
+      server_name: {{ required "A server_name must be provided." .Values.dendrite.global.server_name | quote }}
+      private_key: matrix_key.pem
+      key_validity_period: {{ default "168h0m0s" .Values.dendrite.global.key_validity_period | quote }}
+      cache:
+        max_size_estimated: {{ default "1gb" .Values.dendrite.global.cache.max_size_estimated | quote }}
+        max_age: {{ default "1h" .Values.dendrite.global.cache.max_age | quote }}
+      well_known_server_name: {{ default "" .Values.dendrite.global.well_known_server_name | quote }}
+      well_known_client_name: {{ default "" .Values.dendrite.global.well_known_client_name | quote }}
+      trusted_third_party_id_servers:
+      {{- toYaml .Values.dendrite.global.trusted_third_party_id_servers | nindent 8 }}
+      disable_federation: {{ default false .Values.dendrite.global.disable_federation }}
+      presence:
+        enable_inbound: {{ default false .Values.dendrite.global.presence.enable_inbound}}
+        enable_outbound: {{ default false .Values.dendrite.global.presence.enable_outbound }}
+      report_stats:
+        enabled: {{ default false .Values.dendrite.report_stats.enabled }}
+        endpoint: {{ default "https://matrix.org/report-usage-stats/push" .Values.dendrite.report_stats.endpoint }}
+      server_notices:
+        enabled: {{ default false .Values.dendrite.global.server_notices.enabled }}
+        local_part: {{ default "_server" .Values.dendrite.global.server_notices.local_part | quote }}
+        display_name: {{ default "Server alerts" .Values.dendrite.global.server_notices.display_name | quote }}
+        avatar_url: {{ default "" .Values.dendrite.global.server_notices.avatar_url | quote }}
+        room_name: {{ default "Server Alerts" .Values.dendrite.global.server_notices.room_name | quote }}
+      jetstream:
+        addresses:
+          {{- if .Values.nats.enabled }}
+          - {{ template "common.names.fullname" $.Subcharts.nats }}:4222
+          {{- else }}
+          []
+          {{- end }}
+        in_memory: {{ not .Values.persistence.jetstream.enabled }}
+        storage_path: {{ .Values.persistence.jetstream.mountPath }}
+        topic_prefix: "Dendrite"
+      metrics:
+        enabled: {{ default false .Values.dendrite.global.metrics.enabled }}
+        basic_auth:
+          username: {{ default "metrics" .Values.dendrite.global.metrics.basic_auth.username | quote }}
+          password: {{ default "metrics" .Values.dendrite.global.metrics.basic_auth.password | quote }}
+      dns_cache:
+        enabled: {{ default false .Values.dendrite.global.dns_cache.enabled }}
+        cache_size: {{ default 256 .Values.dendrite.global.dns_cache.cache_size }}
+        cache_lifetime: {{ default "5m" .Values.dendrite.global.dns_cache.cache_lifetime }}
+      {{- if not $componentSpecificDatabaseConfig }}
+      database:
+        connection_string: {{ .Values.database.connection_string }}
+        max_open_conns: {{ default 100 .Values.database.max_open_conns }}
+        max_idle_conns: {{ default 5 .Values.database.max_idle_conns }}
+        conn_max_lifetime: {{default -1 .Values.database.conn_max_lifetime }}
+      {{- end }}
+    app_service_api:
+      {{- if .Values.dendrite.polylithEnabled }}
+      internal_api:
+        listen: http://0.0.0.0:{{ .Values.appserviceapi.service.main.ports.internal.port }}
+        connect: http://{{ include "common.names.fullname" (index $.Subcharts "appserviceapi") }}:{{ .Values.appserviceapi.service.main.ports.internal.port }}
+      {{- end }}
+      {{- if $componentSpecificDatabaseConfig }}
+      database:
+        connection_string: {{ .Values.appserviceapi.database.connection_string }}
+        max_open_conns: {{ default .Values.dendrite.database.max_open_conns .Values.appserviceapi.database.max_open_conns }}
+        max_idle_conns: {{ default .Values.dendrite.database.max_idle_conns .Values.appserviceapi.database.max_idle_conns }}
+        conn_max_lifetime: {{ default .Values.dendrite.database.conn_max_lifetime .Values.appserviceapi.database.conn_max_lifetime }}
+      {{- end }}
+      config_files: {{- toYaml .Values.appserviceapi.config.config_files | nindent 8 }}
+    client_api:
+      {{- if .Values.dendrite.polylithEnabled }}
+      internal_api:
+        listen: http://0.0.0.0:{{ .Values.clientapi.service.main.ports.internal.port }}
+        connect: http://{{ include "common.names.fullname" (index $.Subcharts "clientapi") }}:{{ .Values.clientapi.service.main.ports.internal.port }}
+      external_api:
+        listen: http://0.0.0.0:{{ .Values.clientapi.service.main.ports.external.port }}
+      {{- end }}
+      registration_disabled: {{ .Values.clientapi.config.registration_disabled }}
+      registration_shared_secret: {{ default "" .Values.clientapi.config.registration_shared_secret | quote }}
+      enable_registration_captcha: {{ default false .Values.clientapi.config.captcha.enabled }}
+      recaptcha_public_key: {{ default "" .Values.clientapi.config.captcha.recaptcha_public_key | quote }}
+      recaptcha_private_key: {{ default "" .Values.clientapi.config.captcha.recaptcha_private_key | quote }}
+      recaptcha_bypass_secret: {{ default "" .Values.clientapi.config.captcha.recaptcha_bypass_secret | quote }}
+      recaptcha_siteverify_api: {{ default "" .Values.clientapi.config.captcha.recaptcha_siteverify_api | quote }}
+      turn: {{- toYaml .Values.clientapi.config.turn | nindent 8 }}
+      rate_limiting:
+        enabled: {{ default true .Values.clientapi.config.rate_limiting.enabled }}
+        threshold: {{ default 5 .Values.clientapi.config.rate_limiting.threshold }}
+        cooloff_ms: {{ default 500 .Values.clientapi.config.rate_limiting.cooloff_ms }}
+        exempt_user_ids: {{ .Values.clientapi.config.exempt_user_ids }}
+    federation_api:
+      {{- if .Values.dendrite.polylithEnabled }}
+      internal_api:
+        listen: http://0.0.0.0:7772
+        connect: http://{{ include "common.names.fullname" (index $.Subcharts "federationapi") }}:7772
+      external_api:
+        listen: http://0.0.0.0:8072
+        conn_max_lifetime: {{ default .Values.dendrite.database.conn_max_lifetime .Values.federationapi.database.conn_max_lifetime }}
+      federation_certificates: {{- toYaml .Values.federationapi.config.federation_certificates | nindent 8 }}
+      proxy_outbound:
+        enabled: {{ default false .Values.federationapi.config.proxy_outbound.enabled }}
+        protocol: {{ default "http" .Values.federationapi.config.proxy_outbound.protocol | quote }}
+        host: {{ default "localhost" .Values.federationapi.config.proxy_outbound.host | quote }}
+        port: {{ default 8080 .Values.federationapi.config.proxy_outbound.port }}
+      {{- end }}
+      {{- if $componentSpecificDatabaseConfig }}
+      database:
+        connection_string: {{ .Values.federationapi.database.connection_string }}
+        max_open_conns: {{ default .Values.dendrite.database.max_open_conns .Values.federationapi.database.max_open_conns }}
+        max_idle_conns: {{ default .Values.dendrite.database.max_idle_conns .Values.federationapi.database.max_idle_conns }}
+      {{- end }}
+      send_max_retries: {{ default 16 .Values.federationapi.config.send_max_retries }}
+      disable_tls_validation: {{ default false .Values.federationapi.config.disable_tls_validation }}
+      key_perspectives: {{- toYaml .Values.federationapi.config.key_perspectives | nindent 8 }}
+      prefer_direct_fetch: {{ default false .Values.federationapi.config.prefer_direct_fetch }}
+    key_server:
+      {{- if .Values.dendrite.polylithEnabled }}
+      internal_api:
+        listen: http://0.0.0.0:7779
+        connect: http://{{ include "common.names.fullname" (index $.Subcharts "keyserver") }}:7779
+      {{- end }}
+      {{- if $componentSpecificDatabaseConfig }}
+      database:
+        connection_string: {{ .Values.keyserver.database.connection_string }}
+        max_open_conns: {{ default .Values.dendrite.database.max_open_conns .Values.keyserver.database.max_open_conns }}
+        max_idle_conns: {{ default .Values.dendrite.database.max_idle_conns .Values.keyserver.database.max_idle_conns }}
+        conn_max_lifetime: {{ default .Values.dendrite.database.conn_max_lifetime .Values.keyserver.database.conn_max_lifetime }}
+      {{- end }}
+    media_api:
+      {{- if .Values.dendrite.polylithEnabled }}
+      internal_api:
+        listen: http://0.0.0.0:7774
+        connect: http://{{ include "common.names.fullname" (index $.Subcharts "mediaapi") }}:7774
+      external_api:
+        listen: http://0.0.0.0:8074
+      {{- end }}
+      {{- if $componentSpecificDatabaseConfig }}
+      database:
+        connection_string: {{ .Values.mediaapi.database.connection_string }}
+        max_open_conns: {{ default .Values.dendrite.database.max_open_conns .Values.mediaapi.database.max_open_conns }}
+        max_idle_conns: {{ default .Values.dendrite.database.max_idle_conns .Values.mediaapi.database.max_idle_conns }}
+        conn_max_lifetime: {{ default .Values.dendrite.database.conn_max_lifetime .Values.mediaapi.database.conn_max_lifetime }}
+      {{- end }}
+      base_path: {{ default "/var/dendrite/media" .Values.mediaapi.config.base_path | quote }}
+      max_file_size_bytes: {{ int ( default 10485760 .Values.mediaapi.config.max_file_size_bytes ) }}
+      dynamic_thumbnails: {{ default false .Values.mediaapi.config.dynamic_thumbnails }}
+      max_thumbnail_generators: {{ default 10 .Values.mediaapi.config.max_thumbnail_generators }}
+      thumbnail_sizes: {{- toYaml .Values.mediaapi.config.thumbnail_sizes | nindent 8 }}
+    mscs:
+      mscs: {{ .Values.dendrite.global.mscs | toYaml | nindent 8 }}
+      {{- if $componentSpecificDatabaseConfig }}
+      database:
+        connection_string: {{ .Values.mscs.database.connection_string }}
+        max_open_conns: {{ default .Values.dendrite.database.max_open_conns .Values.mscs.database.max_open_conns }}
+        max_idle_conns: {{ default .Values.dendrite.database.max_idle_conns .Values.mscs.database.max_idle_conns }}
+        conn_max_lifetime: {{ default .Values.dendrite.database.conn_max_lifetime .Values.mscs.database.conn_max_lifetime }}
+      {{- end }}
+    room_server:
+      {{- if .Values.dendrite.polylithEnabled }}
+      internal_api:
+        listen: http://0.0.0.0:7770
+        connect: http://{{ include "common.names.fullname" (index $.Subcharts "roomserver") }}:7770
+      {{- end }}
+      {{- if $componentSpecificDatabaseConfig }}
+      database:
+        connection_string: {{ .Values.roomserver.database.connection_string }}
+        max_open_conns: {{ default .Values.dendrite.database.max_open_conns .Values.roomserver.database.max_open_conns }}
+        max_idle_conns: {{ default .Values.dendrite.database.max_idle_conns .Values.roomserver.database.max_idle_conns }}
+        conn_max_lifetime: {{ default .Values.dendrite.database.conn_max_lifetime .Values.roomserver.database.conn_max_lifetime }}
+      {{- end }}
+    sync_api:
+      {{- if .Values.dendrite.polylithEnabled }}
+      internal_api:
+        listen: http://0.0.0.0:7773
+        connect: http://{{ include "common.names.fullname" (index $.Subcharts "syncapi") }}:7773
+      external_api:
+          listen: http://0.0.0.0:8073
+      {{- end }}
+      {{- if $componentSpecificDatabaseConfig }}
+      database:
+        connection_string: {{ .Values.syncapi.database.connection_string }}
+        max_open_conns: {{ default .Values.dendrite.database.max_open_conns .Values.syncapi.database.max_open_conns }}
+        max_idle_conns: {{ default .Values.dendrite.database.max_idle_conns .Values.syncapi.database.max_idle_conns }}
+        conn_max_lifetime: {{ default .Values.dendrite.database.conn_max_lifetime .Values.syncapi.database.conn_max_lifetime }}
+      {{- end }}
+    user_api:
+      {{- if .Values.dendrite.polylithEnabled }}
+      internal_api:
+        listen: http://0.0.0.0:7781
+        connect: http://{{ include "common.names.fullname" (index $.Subcharts "userapi") }}:7781
+      {{- end }}
+      {{- if $componentSpecificDatabaseConfig }}
+      account_database:
+        connection_string: {{ .Values.userapi.database.connection_string }}
+        max_open_conns: {{ default .Values.dendrite.database.max_open_conns .Values.userapi.database.max_open_conns }}
+        max_idle_conns: {{ default .Values.dendrite.database.max_idle_conns .Values.userapi.database.max_idle_conns }}
+        conn_max_lifetime: {{ default .Values.dendrite.database.conn_max_lifetime .Values.userapi.database.conn_max_lifetime }}
+      {{- end }}
+      bcrypt_cost: {{ default 10 .Values.userapi.config.bcrypt_cost }}
+    tracing:
+      enabled: {{ .Values.dendrite.tracing.enabled }}
+      jaeger: {{- toYaml .Values.dendrite.tracing.jaeger | nindent 8 }}
+    logging: {{- toYaml .Values.dendrite.logging | nindent 6 }}

charts/dendrite/templates/ingress.yaml

@@ -0,0 +1,57 @@
+{{- if .Values.dendrite.polylith_ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "common.names.fullname" . }}
+  {{- if .Values.dendrite.polylith_ingress.annotations }}
+  annotations: {{ toYaml .Values.dendrite.polylith_ingress.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.dendrite.tls_secret.enabled }}
+  tls:
+  - hosts:
+      - {{ .Values.dendrite.polylith_ingress.host | quote }}
+    secretName: {{ .Values.dendrite.tls_secret.existingSecret }}
+  {{- end }}
+  rules:
+    - host: {{ .Values.dendrite.polylith_ingress.host | quote }}
+      http:
+        paths:
+        {{- range .Values.dendrite.polylith_ingress.syncapi_paths }}
+        - path: {{ . | quote }}
+          pathType: Exact
+          backend:
+            service:
+              name: {{ include "common.names.fullname" (index $.Subcharts  "syncapi") }}
+              port:
+                number: {{ $.Values.syncapi.service.main.ports.external.port }}
+        {{- end }}
+        - path: /_matrix/client
+          pathType: Prefix
+          backend:
+            service:
+              name: {{ include "common.names.fullname" (index $.Subcharts  "clientapi") }}
+              port:
+                number: {{ .Values.clientapi.service.main.ports.external.port }}
+        - path: /_matrix/federation
+          pathType: Prefix
+          backend:
+            service:
+              name: {{ include "common.names.fullname" (index $.Subcharts  "federationapi") }}
+              port:
+                number: {{ .Values.federationapi.service.main.ports.external.port }}
+        - path: /_matrix/key
+          pathType: Prefix
+          backend:
+            service:
+              name: {{ include "common.names.fullname" (index $.Subcharts  "federationapi") }}
+              port:
+                number: {{ .Values.federationapi.service.main.ports.external.port }}
+        - path: /_matrix/media
+          pathType: Prefix
+          backend:
+            service:
+              name: {{ include "common.names.fullname" (index $.Subcharts  "mediaapi") }}
+              port:
+                number: {{ .Values.mediaapi.service.main.ports.external.port }}
+{{- end -}}

charts/dendrite/templates/matrix-key-secret.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.dendrite.matrix_key_secret.create }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "dendrite.names.key" . }}
+stringData:
+  {{ .Values.dendrite.matrix_key_secret.secretPath }}: | {{ .Values.dendrite.matrix_key_secret.keyBody | nindent 4 }}
+{{- end }}

charts/dendrite/templates/volumes.yaml

@@ -0,0 +1,35 @@
+{{- define "dendrite.keyVolume" -}}
+enabled: {{ .Values.dendrite.matrix_key_secret.enabled }}
+type: "custom"
+volumeSpec:
+  secret:
+    defaultMode: 0600
+    secretName: {{ include "dendrite.names.key" . }}
+subPath:
+  - path: {{ .Values.dendrite.matrix_key_secret.secretPath }}
+    mountPath: "/etc/dendrite/matrix_key.pem"
+{{- end -}}
+{{- define "dendrite.tlsVolume" -}}
+enabled: {{ .Values.dendrite.tls_secret.enabled }}
+type: "custom"
+volumeSpec:
+  secret:
+    defaultMode: 0600
+    secretName: {{ .Values.dendrite.tls_secret.existingSecret }}
+subPath:
+  - path: {{ .Values.dendrite.tls_secret.crtPath }}
+    mountPath: "/etc/dendrite/server.crt"
+  - path: {{ .Values.dendrite.tls_secret.keyPath }}
+    mountPath: "/etc/dendrite/server.key"
+{{- end -}}
+{{- define "dendrite.configVolume" -}}
+enabled: true
+type: "custom"
+volumeSpec:
+  secret:
+    defaultMode: 0600
+    secretName: {{ include "common.names.fullname" . }}-config
+subPath:
+  - path: dendrite.yaml
+    mountPath: "/etc/dendrite/dendrite.yaml"
+{{- end -}}

charts/dendrite/values.yaml

@@ -0,0 +1,599 @@
+#
+# IMPORTANT NOTE
+#
+# This chart inherits from our common library chart. You can check the default
+# values/options here:
+# https://github.com/k8s-at-home/library-charts/tree/main/charts/stable/common/values.yaml
+#
+---
+image:
+  # -- image repository
+  repository: ghcr.io/matrix-org/dendrite-monolith
+  # -- image tag
+  # @default -- chart.appVersion
+  tag:
+  # -- image pull policy
+  pullPolicy: IfNotPresent
+
+# -- If added dendrite will start a HTTP and HTTPS listener
+# args:
+#   - "--tls-cert=server.crt"
+#   - "--tls-key=server.key"
+
+# -- Configures service settings for the chart.
+# @default -- See values.yaml
+service:
+  main:
+    ports:
+      # -- Configures the default HTTP listener for dendrite
+      # @default -- See values.yaml
+      http:
+        port: 8008
+      # -- Configures the HTTPS listener for dendrite
+      # @default -- See values.yaml
+      https:
+        enabled: true
+        port: 8448
+        protocol: HTTPS
+
+ingress:
+  # -- (Monolith Only) Enable and configure ingress settings for the chart under
+  # this key.
+  # @default -- See values.yaml
+  main:
+    enabled: false
+
+# -- Configure persistence settings for the chart under this key.
+# @default -- See values.yaml
+persistence:
+  media:
+    enabled: false
+    mountPath: &mediaPath /var/dendrite/media
+    accessMode: ReadWriteOnce
+    size: 5Gi
+  # -- Configure Jetsream persistence. This is highly recommended in production.
+  # @default -- See values.yaml
+  jetstream:
+    enabled: false
+    mountPath: /var/dendrite/jetstream
+    accessMode: ReadWriteOnce
+    size: 1Gi
+
+# Configure global database settings
+# @default -- see values.yaml
+database:
+  connection_string: file:dendrite?sslmode=disable
+  max_open_conns: 100
+  max_idle_conns: 2
+  conn_max_lifetime: -1
+
+# -- Configure the key server.
+# For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- See values.yaml
+keyserver:
+  image:
+    # -- image repository
+    repository: matrixdotorg/dendrite-polylith
+    # -- image tag
+    # @default -- chart.appVersion
+    tag:
+    # -- image pull policy
+    pullPolicy: IfNotPresent
+  service:
+    main:
+      ports:
+        http:
+          enabled: false
+        internal:
+          enabled: true
+          port: 7779
+  args: "keyserver"
+  # -- Override general dendrite.database parameters.
+  # @default -- See values.yaml
+  database:
+    # -- Custom connection string
+    # @default -- file or derived from included postgresql deployment
+    connection_string: null
+    # -- Maximum open connections
+    # @default -- dendrite.database.max_open_conns
+    max_open_conns: null
+    # -- Maximum dile connections
+    # @default -- dendrite.database.max_idle_conns
+    max_idle_conns: null
+    # -- Maximum connection lifetime
+    # @default -- dendrite.database.conn_max_lifetime
+    conn_max_lifetime: null
+
+# -- Configure the app service api.
+# For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- See values.yaml
+appserviceapi:
+  image:
+    # -- image repository
+    repository: matrixdotorg/dendrite-polylith
+    # -- image tag
+    # @default -- chart.appVersion
+    tag:
+    # -- image pull policy
+    pullPolicy: IfNotPresent
+  service:
+    main:
+      ports:
+        http:
+          enabled: false
+        internal:
+          enabled: true
+          port: 7777
+  ingress:
+
+  args: "appservice"
+  # -- Override general dendrite.database parameters.
+  # @default -- See values.yaml
+  database:
+    # -- Custom connection string
+    # @default -- file or derived from included postgresql deployment
+    connection_string: null
+    # -- Maximum open connections
+    # @default -- dendrite.database.max_open_conns
+    max_open_conns: null
+    # -- Maximum dile connections
+    # @default -- dendrite.database.max_idle_conns
+    max_idle_conns: null
+    # -- Maximum connection lifetime
+    # @default -- dendrite.database.conn_max_lifetime
+    conn_max_lifetime: null
+  config:
+    config_files: []
+
+# -- Configuration for the client api component.
+# For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- See values.yaml
+clientapi:
+  image:
+    # -- image repository
+    repository: matrixdotorg/dendrite-polylith
+    # -- image tag
+    # @default -- chart.appVersion
+    tag:
+    # -- image pull policy
+    pullPolicy: IfNotPresent
+  service:
+    main:
+      ports:
+        http:
+          enabled: false
+        internal:
+          enabled: true
+          port: 7771
+        external:
+          enabled: true
+          port: 8071
+  args: "clientapi"
+  config:
+    # -- Enable or disable registration for this homeserver.
+    registration_disabled: true
+    # -- Shared secret that allows registration, despite registration_disabled.
+    registration_shared_secret: ""
+    # -- Configure captcha for registration
+    # @default -- See values.yaml
+    captcha:
+      enabled: false
+      recaptcha_public_key: ""
+      recaptcha_private_key: ""
+      recaptcha_bypass_secret: ""
+      recaptcha_siteverify_api: ""
+    # -- Configure TURN
+    # @default -- See values.yaml
+    turn:
+      turn_user_lifetime: ""
+      turn_uris: []
+      turn_shared_secret: ""
+      turn_username: ""
+      turn_password: ""
+    # -- Configure rate limiting.
+    # @default -- values.yaml
+    rate_limiting:
+      enabled: true
+      threshold: 5
+      cooloff_ms: 500
+      exempt_user_ids: []
+
+# -- Configure the Federation API
+# For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- values.yaml
+federationapi:
+  image:
+    # -- image repository
+    repository: matrixdotorg/dendrite-polylith
+    # -- image tag
+    # @default -- chart.appVersion
+    tag:
+    # -- image pull policy
+    pullPolicy: IfNotPresent
+  service:
+    main:
+      ports:
+        http:
+          enabled: false
+        internal:
+          enabled: true
+          port: 7772
+        external:
+          enabled: true
+          port: 8072
+  args: "federationapi"
+  # -- Override general dendrite.database parameters.
+  # @default -- See values.yaml
+  database:
+    # -- Custom connection string
+    # @default -- file or derived from included postgresql deployment
+    connection_string: null
+    # -- Maximum open connections
+    # @default -- dendrite.database.max_open_conns
+    max_open_conns: null
+    # -- Maximum dile connections
+    # @default -- dendrite.database.max_idle_conns
+    max_idle_conns: null
+    # -- Maximum connection lifetime
+    # @default -- dendrite.database.conn_max_lifetime
+    conn_max_lifetime: null
+  config:
+    federation-certificates: []
+    send-max_retires: 16
+    disable_tls_validation: false
+    proxy_outbound:
+      enabled: false
+      protocol: http
+      host: localhost
+      port: 8080
+    key_perspectives:
+      - server_name: matrix.org
+        keys:
+          - key_id: ed25519:auto
+            public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
+          - key_id: ed25519:a_RXGa
+            public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
+    prefer_direct_fetch: false
+
+# -- Configure the User API
+# For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- values.yaml
+userapi:
+  image:
+    # -- image repository
+    repository: matrixdotorg/dendrite-polylith
+    # -- image tag
+    # @default -- chart.appVersion
+    tag:
+    # -- image pull policy
+    pullPolicy: IfNotPresent
+  service:
+    main:
+      ports:
+        http:
+          enabled: false
+        internal:
+          enabled: true
+          port: 7781
+  args: "userapi"
+  # -- Override general dendrite.database parameters.
+  # @default -- See values.yaml
+  database:
+    # -- Custom connection string
+    # @default -- file or derived from included postgresql deployment
+    connection_string: null
+    # -- Maximum open connections
+    # @default -- dendrite.database.max_open_conns
+    max_open_conns: null
+    # -- Maximum dile connections
+    # @default -- dendrite.database.max_idle_conns
+    max_idle_conns: null
+    # -- Maximum connection lifetime
+    # @default -- dendrite.database.conn_max_lifetime
+    conn_max_lifetime: null
+  config:
+    # -- bcrypt cost (2^[cost] = rounds)
+    # @default -- 10
+    bcrypt_cost: 10
+
+# -- Configure the Sync API
+# For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- values.yaml
+syncapi:
+  image:
+    # -- image repository
+    repository: matrixdotorg/dendrite-polylith
+    # -- image tag
+    # @default -- chart.appVersion
+    tag:
+    # -- image pull policy
+    pullPolicy: IfNotPresent
+  service:
+    main:
+      ports:
+        http:
+          enabled: false
+        internal:
+          enabled: true
+          port: 7773
+        external:
+          enabled: true
+          port: 8073
+  args: "syncapi"
+  # -- Override general dendrite.database parameters.
+  # @default -- See values.yaml
+  database:
+    # -- Custom connection string
+    # @default -- file or derived from included postgresql deployment
+    connection_string: null
+    # -- Maximum open connections
+    # @default -- dendrite.database.max_open_conns
+    max_open_conns: null
+    # -- Maximum dile connections
+    # @default -- dendrite.database.max_idle_conns
+    max_idle_conns: null
+    # -- Maximum connection lifetime
+    # @default -- dendrite.database.conn_max_lifetime
+    conn_max_lifetime: null
+
+# -- Configure the Room Server
+# For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- values.yaml
+roomserver:
+  image:
+    # -- image repository
+    repository: matrixdotorg/dendrite-polylith
+    # -- image tag
+    # @default -- chart.appVersion
+    tag:
+    # -- image pull policy
+    pullPolicy: IfNotPresent
+  service:
+    main:
+      ports:
+        http:
+          enabled: false
+        internal:
+          enabled: true
+          port: 7770
+  args: "roomserver"
+  # -- Override general dendrite.database parameters.
+  # @default -- See values.yaml
+  database:
+    # -- Custom connection string
+    # @default -- file or derived from included postgresql deployment
+    connection_string: null
+    # -- Maximum open connections
+    # @default -- dendrite.database.max_open_conns
+    max_open_conns: null
+    # -- Maximum dile connections
+    # @default -- dendrite.database.max_idle_conns
+    max_idle_conns: null
+    # -- Maximum connection lifetime
+    # @default -- dendrite.database.conn_max_lifetime
+    conn_max_lifetime: null
+
+# -- Configure the Media API
+# For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- values.yaml
+mediaapi:
+  image:
+    # -- image repository
+    repository: matrixdotorg/dendrite-polylith
+    # -- image tag
+    # @default -- chart.appVersion
+    tag:
+    # -- image pull policy
+    pullPolicy: IfNotPresent
+  service:
+    main:
+      ports:
+        http:
+          enabled: false
+        internal:
+          enabled: true
+          port: 7774
+        external:
+          enabled: true
+          port: 8074
+  args: "mediaapi"
+  # -- Override general dendrite.database parameters.
+  # @default -- See values.yaml
+  database:
+    # -- Custom connection string
+    # @default -- file or derived from included postgresql deployment
+    connection_string: null
+    # -- Maximum open connections
+    # @default -- dendrite.database.max_open_conns
+    max_open_conns: null
+    # -- Maximum dile connections
+    # @default -- dendrite.database.max_idle_conns
+    max_idle_conns: null
+    # -- Maximum connection lifetime
+    # @default -- dendrite.database.conn_max_lifetime
+    conn_max_lifetime: null
+  config:
+    base_path: *mediaPath
+    max_file_size_bytes: 10485760
+    dynamic_thumbnails: false
+    max_thumbnail_generators: 10
+    thumbnail_sizes:
+      - width: 32
+        height: 32
+        method: crop
+      - width: 96
+        height: 96
+        method: crop
+      - width: 640
+        height: 480
+        method: scale
+
+# -- Configuration for experimental MSCs
+# For more information see [the sample dendrite configuration](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- values.yaml
+mscs:
+  # -- Override general dendrite.database parameters.
+  # @default -- See values.yaml
+  database:
+    # -- Custom connection string
+    # @default -- file or derived from included postgresql deployment
+    connection_string: null
+    # -- Maximum open connections
+    # @default -- dendrite.database.max_open_conns
+    max_open_conns: null
+    # -- Maximum dile connections
+    # @default -- dendrite.database.max_idle_conns
+    max_idle_conns: null
+    # -- Maximum connection lifetime
+    # @default -- dendrite.database.conn_max_lifetime
+    conn_max_lifetime: null
+
+# -- Configuration for Dendrite.
+# For more information see [the sample
+# denrite-config.yaml](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+# @default -- See values.yaml
+dendrite:
+  # -- Enable polylith deployment
+  polylithEnabled: false
+  # -- Usage statistics reporting configuration
+  report_stats:
+    # -- Enable or disable usage reporting
+    # @default -- false
+    enabled: false
+    # -- Push endpoint for usage statistics
+    endpoint: ""
+  # -- If enabled, use an existing secrets for the TLS certificate and key.
+  # Otherwise, to enable TLS a `server.crt` and `server.key` must be mounted at
+  # `/etc/dendrite`.
+  # @default -- See values.yaml
+  tls_secret:
+    enabled: false
+    existingSecret: ""
+    crtPath: tls.crt
+    keyPath: tls.key
+
+  matrix_key_secret:
+    # -- Create matrix_key secret using the keyBody below.
+    create: false
+    # -- New Key Body
+    keyBody: ""
+    # -- Use an existing secret
+    existingSecret: ""
+    # -- Field in the secret to get the key from
+    secretPath: matrix_key.pem
+
+  # -- Enable and configure polylith ingress as per
+  # https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf
+  # @default -- See values.yaml
+  polylith_ingress:
+    enabled: false
+    host: ""
+    annotations: {}
+    # -- Sync API Paths are a little tricky since they require regular expressions. Therefore
+    # the paths will depend on the ingress controller used. See values.yaml for nginx and traefik.
+    # @default -- See values.yaml
+    syncapi_paths: []
+      # For Traefik uncomment these lines
+      # - /_matrix/client/{version:.*?}/rooms/{roomid:.*?}/messages
+      # - /_matrix/client/{version:.*?}/keys/changes
+      # - /_matrix/client/{version:.*?}/user/{userid:.*?}/filter/{filterid:.*?}
+      # - /_matrix/client/{version:.*?}/user/{userid:.*?}/filter
+      # - /_matrix/client/{version:.*?}/sync
+      #
+      # For nginx uncomment these lines and add the annotations here:
+      # https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#use-regex
+      # - /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$
+
+  # -- Configure the global settings for dendrite.
+  # @default -- See values.yaml
+  global:
+    # -- (required) Configure the server name for the dendrite instance.
+    server_name: localhost
+    # -- Configure the key_validity period
+    key_validity_period: 168h0m0s
+    # -- Congigure the in-memory caches
+    cache:
+      # -- Configure the maximum estimated cache size (not a hard limit)
+      max_size_estimated: "1gb"
+      # -- The maximum amount of time that a cache entry can live for in memory
+      max_age: "1h"
+    # -- Configure the well-known server name and optional port
+    well_known_server_name: ""
+    # -- Configure the well-known client name and optional port
+    well_known_client_name: ""
+    # -- Configure the list of domains the server will trust as identity servers
+    trusted_third_party_id_servers:
+      - matrix.org
+      - vector.im
+    # -- Disables federation
+    disable_federation: false
+    # -- Configure handling of presence events
+    presence:
+      # -- Whether inbound presence events are allowed, e.g. receiving presence events from other servers
+      enable_inbound: false
+      # -- Whether outbound presence events are allowed, e.g. sending presence events to other servers
+      enable_outbound: false
+
+    # -- Server notices allows server admins to send messages to all users.
+    server_notices:
+      enabled: false
+      # -- The server localpart to be used when sending notices, ensure this is not yet taken
+      local_part: "_server"
+      # -- The displayname to be used when sending notices
+      display_name: "Server alerts"
+      # -- The mxid of the avatar to use
+      avatar_url: ""
+      # -- The roomname to be used when creating messages
+      room_name: "Server Alerts"
+    # -- Configure prometheus metrics collection for dendrite.
+    # @default -- See values.yaml
+    metrics:
+      # -- If enabled, metrics collection will be enabled
+      # @default -- See values.yaml
+      enabled: false
+      basic_auth:
+        username: metrics
+        password: metrics
+    # -- Configure DNS cache.
+    # @default -- See values.yaml
+    dns_cache:
+      # -- If enabled, dns cache will be enabled.
+      # @default -- See values.yaml
+      enabled: false
+      cache_size: 256
+      cache_lifetime: "5m"
+    # -- Configure experimental MSC's
+    mscs: []
+
+  # -- Configure opentracing.
+  # @default -- See values.yaml
+  tracing:
+    enabled: false
+    jaeger:
+      serviceName: ""
+      disabled: false
+      rpc_metrics: false
+      tags: []
+      sampler: null
+      reporter: null
+      headers: null
+      baggage_restrictions: null
+      throttler: null
+
+  # -- Configure logging.
+  # @default -- See values.yaml
+  logging:
+    - type: file
+      level: info
+      params:
+        path: /var/log/dendrite
+
+nats:
+  # -- Enable and configure NATS for dendrite. Can be disabled for monolith
+  # deployments - an internal NATS server will be used in its place.
+  # @default -- See value.yaml
+  enabled: false
+  nats:
+    image: nats:2.7.1-alpine
+    jetstream:
+      enabled: true

clientapi/api/api.go

@@ -14,18 +14,10 @@
 
 package api
 
-import "github.com/matrix-org/gomatrixserverlib/fclient"
+import "github.com/matrix-org/gomatrixserverlib"
 
 // ExtraPublicRoomsProvider provides a way to inject extra published rooms into /publicRooms requests.
 type ExtraPublicRoomsProvider interface {
 	// Rooms returns the extra rooms. This is called on-demand by clients, so cache appropriately.
-	Rooms() []fclient.PublicRoom
-}
-
-type RegistrationToken struct {
-	Token       *string `json:"token"`
-	UsesAllowed *int32  `json:"uses_allowed"`
-	Pending     *int32  `json:"pending"`
-	Completed   *int32  `json:"completed"`
-	ExpiryTime  *int64  `json:"expiry_time"`
+	Rooms() []gomatrixserverlib.PublicRoom
 }

clientapi/auth/auth.go

@@ -23,8 +23,8 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -58,7 +58,7 @@ func VerifyUserFromRequest(
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: spec.MissingToken(err.Error()),
+			JSON: jsonerror.MissingToken(err.Error()),
 		}
 	}
 	var res api.QueryAccessTokenResponse
@@ -68,23 +68,21 @@ func VerifyUserFromRequest(
 	}, &res)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccessToken failed")
-		return nil, &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		jsonErr := jsonerror.InternalServerError()
+		return nil, &jsonErr
 	}
 	if res.Err != "" {
 		if strings.HasPrefix(strings.ToLower(res.Err), "forbidden:") { // TODO: use actual error and no string comparison
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: spec.Forbidden(res.Err),
+				JSON: jsonerror.Forbidden(res.Err),
 			}
 		}
 	}
 	if res.Device == nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: spec.UnknownToken("Unknown token"),
+			JSON: jsonerror.UnknownToken("Unknown token"),
 		}
 	}
 	return res.Device, nil

clientapi/auth/authtypes/threepid.go

@@ -16,8 +16,6 @@ package authtypes
 
 // ThreePID represents a third-party identifier
 type ThreePID struct {
-	Address     string `json:"address"`
-	Medium      string `json:"medium"`
-	AddedAt     int64  `json:"added_at"`
-	ValidatedAt int64  `json:"validated_at"`
+	Address string `json:"address"`
+	Medium  string `json:"medium"`
 }

clientapi/auth/login.go

@@ -15,14 +15,15 @@
 package auth
 
 import (
+	"context"
 	"encoding/json"
 	"io"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -31,17 +32,12 @@ import (
 // called after authorization has completed, with the result of the authorization.
 // If the final return value is non-nil, an error occurred and the cleanup function
 // is nil.
-func LoginFromJSONReader(
-	req *http.Request,
-	useraccountAPI uapi.UserLoginAPI,
-	userAPI UserInternalAPIForLogin,
-	cfg *config.ClientAPI,
-) (*Login, LoginCleanupFunc, *util.JSONResponse) {
-	reqBytes, err := io.ReadAll(req.Body)
+func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	reqBytes, err := io.ReadAll(r)
 	if err != nil {
 		err := &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Reading request body failed: " + err.Error()),
+			JSON: jsonerror.BadJSON("Reading request body failed: " + err.Error()),
 		}
 		return nil, nil, err
 	}
@@ -52,7 +48,7 @@ func LoginFromJSONReader(
 	if err := json.Unmarshal(reqBytes, &header); err != nil {
 		err := &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Reading request body failed: " + err.Error()),
+			JSON: jsonerror.BadJSON("Reading request body failed: " + err.Error()),
 		}
 		return nil, nil, err
 	}
@@ -61,37 +57,23 @@ func LoginFromJSONReader(
 	switch header.Type {
 	case authtypes.LoginTypePassword:
 		typ = &LoginTypePassword{
-			UserAPI: useraccountAPI,
-			Config:  cfg,
+			GetAccountByPassword: useraccountAPI.QueryAccountByPassword,
+			Config:               cfg,
 		}
 	case authtypes.LoginTypeToken:
 		typ = &LoginTypeToken{
 			UserAPI: userAPI,
 			Config:  cfg,
 		}
-	case authtypes.LoginTypeApplicationService:
-		token, err := ExtractAccessToken(req)
-		if err != nil {
-			err := &util.JSONResponse{
-				Code: http.StatusForbidden,
-				JSON: spec.MissingToken(err.Error()),
-			}
-			return nil, nil, err
-		}
-
-		typ = &LoginTypeApplicationService{
-			Config: cfg,
-			Token:  token,
-		}
 	default:
 		err := util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam("unhandled login type: " + header.Type),
+			JSON: jsonerror.InvalidArgumentValue("unhandled login type: " + header.Type),
 		}
 		return nil, nil, &err
 	}
 
-	return typ.LoginFromJSON(req.Context(), reqBytes)
+	return typ.LoginFromJSON(ctx, reqBytes)
 }
 
 // UserInternalAPIForLogin contains the aspects of UserAPI required for logging in.

clientapi/auth/login_application_service.go

@@ -1,55 +0,0 @@
-// Copyright 2023 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
-	"context"
-
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/util"
-)
-
-// LoginTypeApplicationService describes how to authenticate as an
-// application service
-type LoginTypeApplicationService struct {
-	Config *config.ClientAPI
-	Token  string
-}
-
-// Name implements Type
-func (t *LoginTypeApplicationService) Name() string {
-	return authtypes.LoginTypeApplicationService
-}
-
-// LoginFromJSON implements Type
-func (t *LoginTypeApplicationService) LoginFromJSON(
-	ctx context.Context, reqBytes []byte,
-) (*Login, LoginCleanupFunc, *util.JSONResponse) {
-	var r Login
-	if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil {
-		return nil, nil, err
-	}
-
-	_, err := internal.ValidateApplicationServiceRequest(t.Config, r.Identifier.User, t.Token)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cleanup := func(ctx context.Context, j *util.JSONResponse) {}
-	return &r, cleanup, nil
-}

clientapi/auth/login_test.go

@@ -17,17 +17,14 @@ package auth
 import (
 	"context"
 	"net/http"
-	"net/http/httptest"
 	"reflect"
-	"regexp"
 	"strings"
 	"testing"
 
-	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/fclient"
-	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
@@ -35,9 +32,8 @@ func TestLoginFromJSONReader(t *testing.T) {
 	ctx := context.Background()
 
 	tsts := []struct {
-		Name  string
-		Body  string
-		Token string
+		Name string
+		Body string
 
 		WantUsername      string
 		WantDeviceID      string
@@ -51,7 +47,7 @@ func TestLoginFromJSONReader(t *testing.T) {
 				"password": "herpassword",
 				"device_id": "adevice"
             }`,
-			WantUsername: "@alice:example.com",
+			WantUsername: "alice",
 			WantDeviceID: "adevice",
 		},
 		{
@@ -65,69 +61,21 @@ func TestLoginFromJSONReader(t *testing.T) {
 			WantDeviceID:      "adevice",
 			WantDeletedTokens: []string{"atoken"},
 		},
-		{
-			Name: "appServiceWorksUserID",
-			Body: `{
-				"type": "m.login.application_service",
-				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
-				"device_id": "adevice"
-			}`,
-			Token: "astoken",
-
-			WantUsername: "@alice:example.com",
-			WantDeviceID: "adevice",
-		},
-		{
-			Name: "appServiceWorksLocalpart",
-			Body: `{
-				"type": "m.login.application_service",
-				"identifier": { "type": "m.id.user", "user": "alice" },
-				"device_id": "adevice"
-			}`,
-			Token: "astoken",
-
-			WantUsername: "alice",
-			WantDeviceID: "adevice",
-		},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
 			var userAPI fakeUserInternalAPI
 			cfg := &config.ClientAPI{
 				Matrix: &config.Global{
-					SigningIdentity: fclient.SigningIdentity{
+					SigningIdentity: gomatrixserverlib.SigningIdentity{
 						ServerName: serverName,
 					},
 				},
-				Derived: &config.Derived{
-					ApplicationServices: []config.ApplicationService{
-						{
-							ID:      "anapplicationservice",
-							ASToken: "astoken",
-							NamespaceMap: map[string][]config.ApplicationServiceNamespace{
-								"users": {
-									{
-										Exclusive:    true,
-										Regex:        "@alice:example.com",
-										RegexpObject: regexp.MustCompile("@alice:example.com"),
-									},
-								},
-							},
-						},
-					},
-				},
 			}
-
-			req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body))
-			if tst.Token != "" {
-				req.Header.Add("Authorization", "Bearer "+tst.Token)
+			login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
+			if err != nil {
+				t.Fatalf("LoginFromJSONReader failed: %+v", err)
 			}
-
-			login, cleanup, jsonErr := LoginFromJSONReader(req, &userAPI, &userAPI, cfg)
-			if jsonErr != nil {
-				t.Fatalf("LoginFromJSONReader failed: %+v", jsonErr)
-			}
-
 			cleanup(ctx, &util.JSONResponse{Code: http.StatusOK})
 
 			if login.Username() != tst.WantUsername {
@@ -155,17 +103,16 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 	ctx := context.Background()
 
 	tsts := []struct {
-		Name  string
-		Body  string
-		Token string
+		Name string
+		Body string
 
-		WantErrCode spec.MatrixErrorCode
+		WantErrCode string
 	}{
-		{Name: "empty", WantErrCode: spec.ErrorBadJSON},
+		{Name: "empty", WantErrCode: "M_BAD_JSON"},
 		{
 			Name:        "badUnmarshal",
 			Body:        `badsyntaxJSON`,
-			WantErrCode: spec.ErrorBadJSON,
+			WantErrCode: "M_BAD_JSON",
 		},
 		{
 			Name: "badPassword",
@@ -175,7 +122,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 				"password": "invalidpassword",
 				"device_id": "adevice"
             }`,
-			WantErrCode: spec.ErrorForbidden,
+			WantErrCode: "M_FORBIDDEN",
 		},
 		{
 			Name: "badToken",
@@ -184,7 +131,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 				"token": "invalidtoken",
 				"device_id": "adevice"
             }`,
-			WantErrCode: spec.ErrorForbidden,
+			WantErrCode: "M_FORBIDDEN",
 		},
 		{
 			Name: "badType",
@@ -192,46 +139,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 				"type": "m.login.invalid",
 				"device_id": "adevice"
             }`,
-			WantErrCode: spec.ErrorInvalidParam,
-		},
-		{
-			Name: "noASToken",
-			Body: `{
-				"type": "m.login.application_service",
-				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
-				"device_id": "adevice"
-			}`,
-			WantErrCode: "M_MISSING_TOKEN",
-		},
-		{
-			Name:  "badASToken",
-			Token: "badastoken",
-			Body: `{
-				"type": "m.login.application_service",
-				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
-				"device_id": "adevice"
-			}`,
-			WantErrCode: "M_UNKNOWN_TOKEN",
-		},
-		{
-			Name:  "badASNamespace",
-			Token: "astoken",
-			Body: `{
-				"type": "m.login.application_service",
-				"identifier": { "type": "m.id.user", "user": "@bob:example.com" },
-				"device_id": "adevice"
-			}`,
-			WantErrCode: "M_EXCLUSIVE",
-		},
-		{
-			Name:  "badASUserID",
-			Token: "astoken",
-			Body: `{
-				"type": "m.login.application_service",
-				"identifier": { "type": "m.id.user", "user": "@alice:wrong.example.com" },
-				"device_id": "adevice"
-			}`,
-			WantErrCode: "M_INVALID_USERNAME",
+			WantErrCode: "M_INVALID_ARGUMENT_VALUE",
 		},
 	}
 	for _, tst := range tsts {
@@ -239,38 +147,16 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 			var userAPI fakeUserInternalAPI
 			cfg := &config.ClientAPI{
 				Matrix: &config.Global{
-					SigningIdentity: fclient.SigningIdentity{
+					SigningIdentity: gomatrixserverlib.SigningIdentity{
 						ServerName: serverName,
 					},
 				},
-				Derived: &config.Derived{
-					ApplicationServices: []config.ApplicationService{
-						{
-							ID:      "anapplicationservice",
-							ASToken: "astoken",
-							NamespaceMap: map[string][]config.ApplicationServiceNamespace{
-								"users": {
-									{
-										Exclusive:    true,
-										Regex:        "@alice:example.com",
-										RegexpObject: regexp.MustCompile("@alice:example.com"),
-									},
-								},
-							},
-						},
-					},
-				},
 			}
-			req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body))
-			if tst.Token != "" {
-				req.Header.Add("Authorization", "Bearer "+tst.Token)
-			}
-
-			_, cleanup, errRes := LoginFromJSONReader(req, &userAPI, &userAPI, cfg)
+			_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
 			if errRes == nil {
 				cleanup(ctx, nil)
 				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
-			} else if merr, ok := errRes.JSON.(spec.MatrixError); ok && merr.ErrCode != tst.WantErrCode {
+			} else if merr, ok := errRes.JSON.(*jsonerror.MatrixError); ok && merr.ErrCode != tst.WantErrCode {
 				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
 			}
 		})
@@ -288,15 +174,7 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *
 		return nil
 	}
 	res.Exists = true
-	res.Account = &uapi.Account{UserID: userutil.MakeUserID(req.Localpart, req.ServerName)}
-	return nil
-}
-
-func (ua *fakeUserInternalAPI) QueryAccountByLocalpart(ctx context.Context, req *uapi.QueryAccountByLocalpartRequest, res *uapi.QueryAccountByLocalpartResponse) error {
-	return nil
-}
-
-func (ua *fakeUserInternalAPI) PerformAccountCreation(ctx context.Context, req *uapi.PerformAccountCreationRequest, res *uapi.PerformAccountCreationResponse) error {
+	res.Account = &uapi.Account{}
 	return nil
 }
 

clientapi/auth/login_token.go

@@ -20,9 +20,9 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -48,15 +48,13 @@ func (t *LoginTypeToken) LoginFromJSON(ctx context.Context, reqBytes []byte) (*L
 	var res uapi.QueryLoginTokenResponse
 	if err := t.UserAPI.QueryLoginToken(ctx, &uapi.QueryLoginTokenRequest{Token: r.Token}, &res); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("UserAPI.QueryLoginToken failed")
-		return nil, nil, &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		jsonErr := jsonerror.InternalServerError()
+		return nil, nil, &jsonErr
 	}
 	if res.Data == nil {
 		return nil, nil, &util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("invalid login token"),
+			JSON: jsonerror.Forbidden("invalid login token"),
 		}
 	}
 

clientapi/auth/password.go

@@ -16,21 +16,20 @@ package auth
 
 import (
 	"context"
-	"database/sql"
-	"github.com/go-ldap/ldap/v3"
-	"github.com/google/uuid"
 	"net/http"
 	"strings"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
+type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error
+
 type PasswordRequest struct {
 	Login
 	Password string `json:"password"`
@@ -38,8 +37,8 @@ type PasswordRequest struct {
 
 // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based
 type LoginTypePassword struct {
-	Config  *config.ClientAPI
-	UserAPI api.UserLoginAPI
+	GetAccountByPassword GetAccountByPassword
+	Config               *config.ClientAPI
 }
 
 func (t *LoginTypePassword) Name() string {
@@ -60,227 +59,68 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
 	return login, func(context.Context, *util.JSONResponse) {}, nil
 }
 
-func (t *LoginTypePassword) Login(ctx context.Context, request *PasswordRequest) (*Login, *util.JSONResponse) {
-	fullUsername := request.Username()
-	if fullUsername == "" {
+func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
+	r := req.(*PasswordRequest)
+	username := r.Username()
+	if username == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: spec.BadJSON("A username must be supplied."),
+			JSON: jsonerror.BadJSON("A username must be supplied."),
 		}
 	}
-	if len(request.Password) == 0 {
+	if len(r.Password) == 0 {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: spec.BadJSON("A password must be supplied."),
+			JSON: jsonerror.BadJSON("A password must be supplied."),
 		}
 	}
-	username, domain, err := userutil.ParseUsernameParam(fullUsername, t.Config.Matrix)
+	localpart, domain, err := userutil.ParseUsernameParam(username, t.Config.Matrix)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: spec.InvalidUsername(err.Error()),
+			JSON: jsonerror.InvalidUsername(err.Error()),
 		}
 	}
 	if !t.Config.Matrix.IsLocalServerName(domain) {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: spec.InvalidUsername("The server name is not known."),
+			JSON: jsonerror.InvalidUsername("The server name is not known."),
 		}
 	}
-
-	var account *api.Account
-	if t.Config.Ldap.Enabled {
-		isAdmin, err := t.authenticateLdap(username, request.Password)
-		if err != nil {
-			return nil, err
-		}
-		acc, err := t.getOrCreateAccount(ctx, username, domain, isAdmin)
-		if err != nil {
-			return nil, err
-		}
-		account = acc
-	} else {
-		acc, err := t.authenticateDb(ctx, username, domain, request.Password)
-		if err != nil {
-			return nil, err
-		}
-		account = acc
-	}
-
-	// Set the user, so login.Username() can do the right thing
-	request.Identifier.User = account.UserID
-	request.User = account.UserID
-	return &request.Login, nil
-}
-
-func (t *LoginTypePassword) authenticateDb(ctx context.Context, username string, domain spec.ServerName, password string) (*api.Account, *util.JSONResponse) {
+	// Squash username to all lowercase letters
 	res := &api.QueryAccountByPasswordResponse{}
-	err := t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-		Localpart:         strings.ToLower(username),
+	err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+		Localpart:         strings.ToLower(localpart),
 		ServerName:        domain,
-		PlaintextPassword: password,
+		PlaintextPassword: r.Password,
 	}, res)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("Unable to fetch account by password."),
+			JSON: jsonerror.Unknown("Unable to fetch account by password."),
 		}
 	}
 
 	if !res.Exists {
-		err = t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-			Localpart:         username,
+		err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+			Localpart:         localpart,
 			ServerName:        domain,
-			PlaintextPassword: password,
+			PlaintextPassword: r.Password,
 		}, res)
 		if err != nil {
 			return nil, &util.JSONResponse{
 				Code: http.StatusInternalServerError,
-				JSON: spec.Unknown("Unable to fetch account by password."),
+				JSON: jsonerror.Unknown("Unable to fetch account by password."),
 			}
 		}
+		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
+		// but that would leak the existence of the user.
 		if !res.Exists {
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: spec.Forbidden("The username or password was incorrect or the account does not exist."),
+				JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."),
 			}
 		}
 	}
-	return res.Account, nil
-}
-func (t *LoginTypePassword) authenticateLdap(username, password string) (bool, *util.JSONResponse) {
-	var conn *ldap.Conn
-	conn, err := ldap.DialURL(t.Config.Ldap.Uri)
-	if err != nil {
-		return false, &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("unable to connect to ldap: " + err.Error()),
-		}
-	}
-	defer conn.Close()
-
-	if t.Config.Ldap.AdminBindEnabled {
-		err = conn.Bind(t.Config.Ldap.AdminBindDn, t.Config.Ldap.AdminBindPassword)
-		if err != nil {
-			return false, &util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.Unknown("unable to bind to ldap: " + err.Error()),
-			}
-		}
-		filter := strings.ReplaceAll(t.Config.Ldap.SearchFilter, "{username}", username)
-		searchRequest := ldap.NewSearchRequest(
-			t.Config.Ldap.BaseDn, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases,
-			0, 0, false, filter, []string{t.Config.Ldap.SearchAttribute}, nil,
-		)
-		result, err := conn.Search(searchRequest)
-		if err != nil {
-			return false, &util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.Unknown("unable to bind to search ldap: " + err.Error()),
-			}
-		}
-		if len(result.Entries) > 1 {
-			return false, &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: spec.BadJSON("'user' must be duplicated."),
-			}
-		}
-		if len(result.Entries) < 1 {
-			return false, &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: spec.BadJSON("'user' not found."),
-			}
-		}
-
-		userDN := result.Entries[0].DN
-		err = conn.Bind(userDN, password)
-		if err != nil {
-			return false, &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: spec.InvalidUsername(err.Error()),
-			}
-		}
-	} else {
-		bindDn := strings.ReplaceAll(t.Config.Ldap.UserBindDn, "{username}", username)
-		err = conn.Bind(bindDn, password)
-		if err != nil {
-			return false, &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: spec.InvalidUsername(err.Error()),
-			}
-		}
-	}
-
-	isAdmin, err := t.isLdapAdmin(conn, username)
-	if err != nil {
-		return false, &util.JSONResponse{
-			Code: http.StatusUnauthorized,
-			JSON: spec.InvalidUsername(err.Error()),
-		}
-	}
-	return isAdmin, nil
-}
-
-func (t *LoginTypePassword) isLdapAdmin(conn *ldap.Conn, username string) (bool, error) {
-	searchRequest := ldap.NewSearchRequest(
-		t.Config.Ldap.AdminGroupDn,
-		ldap.ScopeWholeSubtree, ldap.DerefAlways, 0, 0, false,
-		strings.ReplaceAll(t.Config.Ldap.AdminGroupFilter, "{username}", username),
-		[]string{t.Config.Ldap.AdminGroupAttribute},
-		nil)
-
-	sr, err := conn.Search(searchRequest)
-	if err != nil {
-		return false, err
-	}
-
-	if len(sr.Entries) < 1 {
-		return false, nil
-	}
-	return true, nil
-}
-
-func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, username string, domain spec.ServerName, admin bool) (*api.Account, *util.JSONResponse) {
-	var existing api.QueryAccountByLocalpartResponse
-	err := t.UserAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{
-		Localpart:  username,
-		ServerName: domain,
-	}, &existing)
-
-	if err == nil {
-		return existing.Account, nil
-	}
-	if err != sql.ErrNoRows {
-		return nil, &util.JSONResponse{
-			Code: http.StatusUnauthorized,
-			JSON: spec.InvalidUsername(err.Error()),
-		}
-	}
-
-	accountType := api.AccountTypeUser
-	if admin {
-		accountType = api.AccountTypeAdmin
-	}
-	var created api.PerformAccountCreationResponse
-	err = t.UserAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{
-		AppServiceID: "ldap",
-		Localpart:    username,
-		Password:     uuid.New().String(),
-		AccountType:  accountType,
-		OnConflict:   api.ConflictAbort,
-	}, &created)
-
-	if err != nil {
-		if _, ok := err.(*api.ErrorConflict); ok {
-			return nil, &util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.UserInUse("Desired user ID is already taken."),
-			}
-		}
-		return nil, &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("failed to create account: " + err.Error()),
-		}
-	}
-	return created.Account, nil
+	return &r.Login, nil
 }

clientapi/auth/user_interactive.go

@@ -20,9 +20,9 @@ import (
 	"net/http"
 	"sync"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
@@ -55,7 +55,7 @@ type LoginCleanupFunc func(context.Context, *util.JSONResponse)
 // https://matrix.org/docs/spec/client_server/r0.6.1#identifier-types
 type LoginIdentifier struct {
 	Type string `json:"type"`
-	// when type = m.id.user or m.id.application_service
+	// when type = m.id.user
 	User string `json:"user"`
 	// when type = m.id.thirdparty
 	Medium  string `json:"medium"`
@@ -113,8 +113,8 @@ type UserInteractive struct {
 
 func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
-		UserAPI: userAccountAPI,
-		Config:  cfg,
+		GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
+		Config:               cfg,
 	}
 	return &UserInteractive{
 		Flows: []userInteractiveFlow{
@@ -178,10 +178,8 @@ func (u *UserInteractive) NewSession() *util.JSONResponse {
 	sessionID, err := GenerateAccessToken()
 	if err != nil {
 		logrus.WithError(err).Error("failed to generate session ID")
-		return &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		res := jsonerror.InternalServerError()
+		return &res
 	}
 	u.Lock()
 	u.Sessions[sessionID] = []string{}
@@ -195,19 +193,15 @@ func (u *UserInteractive) ResponseWithChallenge(sessionID string, response inter
 	mixedObjects := make(map[string]interface{})
 	b, err := json.Marshal(response)
 	if err != nil {
-		return &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		ise := jsonerror.InternalServerError()
+		return &ise
 	}
 	_ = json.Unmarshal(b, &mixedObjects)
 	challenge := u.challenge(sessionID)
 	b, err = json.Marshal(challenge.JSON)
 	if err != nil {
-		return &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		ise := jsonerror.InternalServerError()
+		return &ise
 	}
 	_ = json.Unmarshal(b, &mixedObjects)
 
@@ -240,7 +234,7 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 	if !ok {
 		return nil, &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Unknown auth.type: " + authType),
+			JSON: jsonerror.BadJSON("Unknown auth.type: " + authType),
 		}
 	}
 
@@ -256,7 +250,7 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 		if !u.IsSingleStageFlow(authType) {
 			return nil, &util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: spec.Unknown("The auth.session is missing or unknown."),
+				JSON: jsonerror.Unknown("The auth.session is missing or unknown."),
 			}
 		}
 	}

clientapi/auth/user_interactive_test.go

@@ -8,14 +8,13 @@ import (
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/fclient"
-	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 var (
 	ctx        = context.Background()
-	serverName = spec.ServerName("example.com")
+	serverName = gomatrixserverlib.ServerName("example.com")
 	// space separated localpart+password -> account
 	lookup = make(map[string]*api.Account)
 	device = &api.Device{
@@ -45,18 +44,10 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a
 	return nil
 }
 
-func (d *fakeAccountDatabase) QueryAccountByLocalpart(ctx context.Context, req *api.QueryAccountByLocalpartRequest, res *api.QueryAccountByLocalpartResponse) error {
-	return nil
-}
-
-func (d *fakeAccountDatabase) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error {
-	return nil
-}
-
 func setup() *UserInteractive {
 	cfg := &config.ClientAPI{
 		Matrix: &config.Global{
-			SigningIdentity: fclient.SigningIdentity{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
 				ServerName: serverName,
 			},
 		},

clientapi/clientapi.go

@@ -15,54 +15,55 @@
 package clientapi
 
 import (
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/process"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/fclient"
-
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/transactions"
+	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
 func AddPublicRoutes(
-	processContext *process.ProcessContext,
-	routers httputil.Routers,
-	cfg *config.Dendrite,
-	natsInstance *jetstream.NATSInstance,
-	federation fclient.FederationClient,
+	base *base.BaseDendrite,
+	federation *gomatrixserverlib.FederationClient,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 	transactionsCache *transactions.Cache,
 	fsAPI federationAPI.ClientFederationAPI,
 	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.QuerySearchProfilesAPI,
-	extRoomsProvider api.ExtraPublicRoomsProvider, enableMetrics bool,
+	keyAPI keyserverAPI.ClientKeyAPI,
+	extRoomsProvider api.ExtraPublicRoomsProvider,
 ) {
-	js, natsClient := natsInstance.Prepare(processContext, &cfg.Global.JetStream)
+	cfg := &base.Cfg.ClientAPI
+	mscCfg := &base.Cfg.MSCs
+	js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	syncProducer := &producers.SyncAPIProducer{
 		JetStream:              js,
-		TopicReceiptEvent:      cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent),
-		TopicSendToDeviceEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
-		TopicTypingEvent:       cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent),
-		TopicPresenceEvent:     cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
 		UserAPI:                userAPI,
-		ServerName:             cfg.Global.ServerName,
+		ServerName:             cfg.Matrix.ServerName,
 	}
 
 	routing.Setup(
-		routers,
+		base.PublicClientAPIMux,
+		base.PublicWellKnownAPIMux,
+		base.SynapseAdminMux,
+		base.DendriteAdminMux,
 		cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
-		syncProducer, transactionsCache, fsAPI,
-		extRoomsProvider, natsClient, enableMetrics,
+		syncProducer, transactionsCache, fsAPI, keyAPI,
+		extRoomsProvider, mscCfg, natsClient,
 	)
 }

clientapi/httputil/httputil.go

@@ -20,7 +20,7 @@ import (
 	"net/http"
 	"unicode/utf8"
 
-	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/util"
 )
 
@@ -32,10 +32,8 @@ func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONRespon
 	body, err := io.ReadAll(req.Body)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
-		return &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		resp := jsonerror.InternalServerError()
+		return &resp
 	}
 
 	return UnmarshalJSON(body, iface)
@@ -45,7 +43,7 @@ func UnmarshalJSON(body []byte, iface interface{}) *util.JSONResponse {
 	if !utf8.Valid(body) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.NotJSON("Body contains invalid UTF-8"),
+			JSON: jsonerror.NotJSON("Body contains invalid UTF-8"),
 		}
 	}
 
@@ -55,7 +53,7 @@ func UnmarshalJSON(body []byte, iface interface{}) *util.JSONResponse {
 		// valid JSON with incorrect types for values.
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
+			JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
 		}
 	}
 	return nil

clientapi/jsonerror/jsonerror.go

@@ -0,0 +1,229 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jsonerror
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+)
+
+// MatrixError represents the "standard error response" in Matrix.
+// http://matrix.org/docs/spec/client_server/r0.2.0.html#api-standards
+type MatrixError struct {
+	ErrCode string `json:"errcode"`
+	Err     string `json:"error"`
+}
+
+func (e MatrixError) Error() string {
+	return fmt.Sprintf("%s: %s", e.ErrCode, e.Err)
+}
+
+// InternalServerError returns a 500 Internal Server Error in a matrix-compliant
+// format.
+func InternalServerError() util.JSONResponse {
+	return util.JSONResponse{
+		Code: http.StatusInternalServerError,
+		JSON: Unknown("Internal Server Error"),
+	}
+}
+
+// Unknown is an unexpected error
+func Unknown(msg string) *MatrixError {
+	return &MatrixError{"M_UNKNOWN", msg}
+}
+
+// Forbidden is an error when the client tries to access a resource
+// they are not allowed to access.
+func Forbidden(msg string) *MatrixError {
+	return &MatrixError{"M_FORBIDDEN", msg}
+}
+
+// BadJSON is an error when the client supplies malformed JSON.
+func BadJSON(msg string) *MatrixError {
+	return &MatrixError{"M_BAD_JSON", msg}
+}
+
+// BadAlias is an error when the client supplies a bad alias.
+func BadAlias(msg string) *MatrixError {
+	return &MatrixError{"M_BAD_ALIAS", msg}
+}
+
+// NotJSON is an error when the client supplies something that is not JSON
+// to a JSON endpoint.
+func NotJSON(msg string) *MatrixError {
+	return &MatrixError{"M_NOT_JSON", msg}
+}
+
+// NotFound is an error when the client tries to access an unknown resource.
+func NotFound(msg string) *MatrixError {
+	return &MatrixError{"M_NOT_FOUND", msg}
+}
+
+// MissingArgument is an error when the client tries to access a resource
+// without providing an argument that is required.
+func MissingArgument(msg string) *MatrixError {
+	return &MatrixError{"M_MISSING_ARGUMENT", msg}
+}
+
+// InvalidArgumentValue is an error when the client tries to provide an
+// invalid value for a valid argument
+func InvalidArgumentValue(msg string) *MatrixError {
+	return &MatrixError{"M_INVALID_ARGUMENT_VALUE", msg}
+}
+
+// MissingToken is an error when the client tries to access a resource which
+// requires authentication without supplying credentials.
+func MissingToken(msg string) *MatrixError {
+	return &MatrixError{"M_MISSING_TOKEN", msg}
+}
+
+// UnknownToken is an error when the client tries to access a resource which
+// requires authentication and supplies an unrecognised token
+func UnknownToken(msg string) *MatrixError {
+	return &MatrixError{"M_UNKNOWN_TOKEN", msg}
+}
+
+// WeakPassword is an error which is returned when the client tries to register
+// using a weak password. http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based
+func WeakPassword(msg string) *MatrixError {
+	return &MatrixError{"M_WEAK_PASSWORD", msg}
+}
+
+// InvalidUsername is an error returned when the client tries to register an
+// invalid username
+func InvalidUsername(msg string) *MatrixError {
+	return &MatrixError{"M_INVALID_USERNAME", msg}
+}
+
+// UserInUse is an error returned when the client tries to register an
+// username that already exists
+func UserInUse(msg string) *MatrixError {
+	return &MatrixError{"M_USER_IN_USE", msg}
+}
+
+// RoomInUse is an error returned when the client tries to make a room
+// that already exists
+func RoomInUse(msg string) *MatrixError {
+	return &MatrixError{"M_ROOM_IN_USE", msg}
+}
+
+// ASExclusive is an error returned when an application service tries to
+// register an username that is outside of its registered namespace, or if a
+// user attempts to register a username or room alias within an exclusive
+// namespace.
+func ASExclusive(msg string) *MatrixError {
+	return &MatrixError{"M_EXCLUSIVE", msg}
+}
+
+// GuestAccessForbidden is an error which is returned when the client is
+// forbidden from accessing a resource as a guest.
+func GuestAccessForbidden(msg string) *MatrixError {
+	return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg}
+}
+
+// InvalidSignature is an error which is returned when the client tries
+// to upload invalid signatures.
+func InvalidSignature(msg string) *MatrixError {
+	return &MatrixError{"M_INVALID_SIGNATURE", msg}
+}
+
+// InvalidParam is an error that is returned when a parameter was invalid,
+// traditionally with cross-signing.
+func InvalidParam(msg string) *MatrixError {
+	return &MatrixError{"M_INVALID_PARAM", msg}
+}
+
+// MissingParam is an error that is returned when a parameter was incorrect,
+// traditionally with cross-signing.
+func MissingParam(msg string) *MatrixError {
+	return &MatrixError{"M_MISSING_PARAM", msg}
+}
+
+// UnableToAuthoriseJoin is an error that is returned when a server can't
+// determine whether to allow a restricted join or not.
+func UnableToAuthoriseJoin(msg string) *MatrixError {
+	return &MatrixError{"M_UNABLE_TO_AUTHORISE_JOIN", msg}
+}
+
+// LeaveServerNoticeError is an error returned when trying to reject an invite
+// for a server notice room.
+func LeaveServerNoticeError() *MatrixError {
+	return &MatrixError{
+		ErrCode: "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM",
+		Err:     "You cannot reject this invite",
+	}
+}
+
+type IncompatibleRoomVersionError struct {
+	RoomVersion string `json:"room_version"`
+	Error       string `json:"error"`
+	Code        string `json:"errcode"`
+}
+
+// IncompatibleRoomVersion is an error which is returned when the client
+// requests a room with a version that is unsupported.
+func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *IncompatibleRoomVersionError {
+	return &IncompatibleRoomVersionError{
+		Code:        "M_INCOMPATIBLE_ROOM_VERSION",
+		RoomVersion: string(roomVersion),
+		Error:       "Your homeserver does not support the features required to join this room",
+	}
+}
+
+// UnsupportedRoomVersion is an error which is returned when the client
+// requests a room with a version that is unsupported.
+func UnsupportedRoomVersion(msg string) *MatrixError {
+	return &MatrixError{"M_UNSUPPORTED_ROOM_VERSION", msg}
+}
+
+// LimitExceededError is a rate-limiting error.
+type LimitExceededError struct {
+	MatrixError
+	RetryAfterMS int64 `json:"retry_after_ms,omitempty"`
+}
+
+// LimitExceeded is an error when the client tries to send events too quickly.
+func LimitExceeded(msg string, retryAfterMS int64) *LimitExceededError {
+	return &LimitExceededError{
+		MatrixError:  MatrixError{"M_LIMIT_EXCEEDED", msg},
+		RetryAfterMS: retryAfterMS,
+	}
+}
+
+// NotTrusted is an error which is returned when the client asks the server to
+// proxy a request (e.g. 3PID association) to a server that isn't trusted
+func NotTrusted(serverName string) *MatrixError {
+	return &MatrixError{
+		ErrCode: "M_SERVER_NOT_TRUSTED",
+		Err:     fmt.Sprintf("Untrusted server '%s'", serverName),
+	}
+}
+
+// InternalAPIError is returned when Dendrite failed to reach an internal API.
+func InternalAPIError(ctx context.Context, err error) util.JSONResponse {
+	logrus.WithContext(ctx).WithError(err).Error("Error reaching an internal API")
+	return util.JSONResponse{
+		Code: http.StatusInternalServerError,
+		JSON: &MatrixError{
+			ErrCode: "M_INTERNAL_SERVER_ERROR",
+			Err:     "Dendrite encountered an error reaching an internal API.",
+		},
+	}
+}

clientapi/jsonerror/jsonerror_test.go

@@ -0,0 +1,44 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jsonerror
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestLimitExceeded(t *testing.T) {
+	e := LimitExceeded("too fast", 5000)
+	jsonBytes, err := json.Marshal(&e)
+	if err != nil {
+		t.Fatalf("TestLimitExceeded: Failed to marshal LimitExceeded error. %s", err.Error())
+	}
+	want := `{"errcode":"M_LIMIT_EXCEEDED","error":"too fast","retry_after_ms":5000}`
+	if string(jsonBytes) != want {
+		t.Errorf("TestLimitExceeded: want %s, got %s", want, string(jsonBytes))
+	}
+}
+
+func TestForbidden(t *testing.T) {
+	e := Forbidden("you shall not pass")
+	jsonBytes, err := json.Marshal(&e)
+	if err != nil {
+		t.Fatalf("TestForbidden: Failed to marshal Forbidden error. %s", err.Error())
+	}
+	want := `{"errcode":"M_FORBIDDEN","error":"you shall not pass"}`
+	if string(jsonBytes) != want {
+		t.Errorf("TestForbidden: want %s, got %s", want, string(jsonBytes))
+	}
+}

clientapi/producers/syncapi.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 
@@ -38,13 +37,13 @@ type SyncAPIProducer struct {
 	TopicTypingEvent       string
 	TopicPresenceEvent     string
 	JetStream              nats.JetStreamContext
-	ServerName             spec.ServerName
+	ServerName             gomatrixserverlib.ServerName
 	UserAPI                userapi.ClientUserAPI
 }
 
 func (p *SyncAPIProducer) SendReceipt(
 	ctx context.Context,
-	userID, roomID, eventID, receiptType string, timestamp spec.Timestamp,
+	userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp,
 ) error {
 	m := &nats.Msg{
 		Subject: p.TopicReceiptEvent,
@@ -155,7 +154,7 @@ func (p *SyncAPIProducer) SendPresence(
 		m.Header.Set("status_msg", *statusMsg)
 	}
 
-	m.Header.Set("last_active_ts", strconv.Itoa(int(spec.AsTimestamp(time.Now()))))
+	m.Header.Set("last_active_ts", strconv.Itoa(int(gomatrixserverlib.AsTimestamp(time.Now()))))
 
 	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
 	return err

clientapi/routing/account_data.go

@@ -21,11 +21,11 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/util"
 )
@@ -38,7 +38,7 @@ func GetAccountData(
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("userID does not match the current user"),
+			JSON: jsonerror.Forbidden("userID does not match the current user"),
 		}
 	}
 
@@ -69,7 +69,7 @@ func GetAccountData(
 
 	return util.JSONResponse{
 		Code: http.StatusNotFound,
-		JSON: spec.NotFound("data not found"),
+		JSON: jsonerror.NotFound("data not found"),
 	}
 }
 
@@ -81,7 +81,7 @@ func SaveAccountData(
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("userID does not match the current user"),
+			JSON: jsonerror.Forbidden("userID does not match the current user"),
 		}
 	}
 
@@ -90,30 +90,27 @@ func SaveAccountData(
 	if req.Body == http.NoBody {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.NotJSON("Content not JSON"),
+			JSON: jsonerror.NotJSON("Content not JSON"),
 		}
 	}
 
 	if dataType == "m.fully_read" || dataType == "m.push_rules" {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden(fmt.Sprintf("Unable to modify %q using this API", dataType)),
+			JSON: jsonerror.Forbidden(fmt.Sprintf("Unable to modify %q using this API", dataType)),
 		}
 	}
 
 	body, err := io.ReadAll(req.Body)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	if !json.Valid(body) {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Bad JSON content"),
+			JSON: jsonerror.BadJSON("Bad JSON content"),
 		}
 	}
 
@@ -145,16 +142,8 @@ func SaveReadMarker(
 	userAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
 ) util.JSONResponse {
-	deviceUserID, err := spec.NewUserID(device.UserID, true)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("userID for this device is invalid"),
-		}
-	}
-
 	// Verify that the user is a member of this room
-	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
+	resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}
@@ -168,10 +157,7 @@ func SaveReadMarker(
 	if r.FullyRead != "" {
 		data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead})
 		if err != nil {
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{},
-			}
+			return jsonerror.InternalServerError()
 		}
 
 		dataReq := api.InputAccountDataRequest{

clientapi/routing/admin.go

@@ -1,401 +1,144 @@
 package routing
 
 import (
-	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net/http"
-	"regexp"
-	"strconv"
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/exp/constraints"
 
-	clientapi "github.com/matrix-org/dendrite/clientapi/api"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/userapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
-var validRegistrationTokenRegex = regexp.MustCompile("^[[:ascii:][:digit:]_]*$")
-
-func AdminCreateNewRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	if !cfg.RegistrationRequiresToken {
+func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	roomID, ok := vars["roomID"]
+	if !ok {
 		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("Registration via tokens is not enabled on this homeserver"),
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting room ID."),
 		}
 	}
-	request := struct {
-		Token       string `json:"token"`
-		UsesAllowed *int32 `json:"uses_allowed,omitempty"`
-		ExpiryTime  *int64 `json:"expiry_time,omitempty"`
-		Length      int32  `json:"length"`
-	}{}
+	res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
+	if err := rsAPI.PerformAdminEvacuateRoom(
+		req.Context(),
+		&roomserverAPI.PerformAdminEvacuateRoomRequest{
+			RoomID: roomID,
+		},
+		res,
+	); err != nil {
+		return util.ErrorResponse(err)
+	}
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{
+			"affected": res.Affected,
+		},
+	}
+}
 
+func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	userID, ok := vars["userID"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting user ID."),
+		}
+	}
+	_, domain, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return util.MessageResponse(http.StatusBadRequest, err.Error())
+	}
+	if !cfg.Matrix.IsLocalServerName(domain) {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("User ID must belong to this server."),
+		}
+	}
+	res := &roomserverAPI.PerformAdminEvacuateUserResponse{}
+	if err := rsAPI.PerformAdminEvacuateUser(
+		req.Context(),
+		&roomserverAPI.PerformAdminEvacuateUserRequest{
+			UserID: userID,
+		},
+		res,
+	); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{
+			"affected": res.Affected,
+		},
+	}
+}
+
+func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	serverName := cfg.Matrix.ServerName
+	localpart, ok := vars["localpart"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting user localpart."),
+		}
+	}
+	if l, s, err := cfg.Matrix.SplitLocalID('@', localpart); err == nil {
+		localpart, serverName = l, s
+	}
+	request := struct {
+		Password string `json:"password"`
+	}{}
 	if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON(fmt.Sprintf("Failed to decode request body: %s", err)),
-		}
-	}
-
-	token := request.Token
-	usesAllowed := request.UsesAllowed
-	expiryTime := request.ExpiryTime
-	length := request.Length
-
-	if len(token) == 0 {
-		if length == 0 {
-			// length not provided in request. Assign default value of 16.
-			length = 16
-		}
-		// token not present in request body. Hence, generate a random token.
-		if length <= 0 || length > 64 {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.BadJSON("length must be greater than zero and not greater than 64"),
-			}
-		}
-		token = util.RandomString(int(length))
-	}
-
-	if len(token) > 64 {
-		//Token present in request body, but is too long.
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("token must not be longer than 64"),
-		}
-	}
-
-	isTokenValid := validRegistrationTokenRegex.Match([]byte(token))
-	if !isTokenValid {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("token must consist only of characters matched by the regex [A-Za-z0-9-_]"),
-		}
-	}
-	// At this point, we have a valid token, either through request body or through random generation.
-	if usesAllowed != nil && *usesAllowed < 0 {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("uses_allowed must be a non-negative integer or null"),
-		}
-	}
-	if expiryTime != nil && spec.Timestamp(*expiryTime).Time().Before(time.Now()) {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("expiry_time must not be in the past"),
-		}
-	}
-	pending := int32(0)
-	completed := int32(0)
-	// If usesAllowed or expiryTime is 0, it means they are not present in the request. NULL (indicating unlimited uses / no expiration will be persisted in DB)
-	registrationToken := &clientapi.RegistrationToken{
-		Token:       &token,
-		UsesAllowed: usesAllowed,
-		Pending:     &pending,
-		Completed:   &completed,
-		ExpiryTime:  expiryTime,
-	}
-	created, err := userAPI.PerformAdminCreateRegistrationToken(req.Context(), registrationToken)
-	if !created {
-		return util.JSONResponse{
-			Code: http.StatusConflict,
-			JSON: map[string]string{
-				"error": fmt.Sprintf("token: %s already exists", token),
-			},
-		}
-	}
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: err,
-		}
-	}
-	return util.JSONResponse{
-		Code: 200,
-		JSON: map[string]interface{}{
-			"token":        token,
-			"uses_allowed": getReturnValue(usesAllowed),
-			"pending":      pending,
-			"completed":    completed,
-			"expiry_time":  getReturnValue(expiryTime),
-		},
-	}
-}
-
-func getReturnValue[t constraints.Integer](in *t) any {
-	if in == nil {
-		return nil
-	}
-	return *in
-}
-
-func AdminListRegistrationTokens(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	queryParams := req.URL.Query()
-	returnAll := true
-	valid := true
-	validQuery, ok := queryParams["valid"]
-	if ok {
-		returnAll = false
-		validValue, err := strconv.ParseBool(validQuery[0])
-		if err != nil {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.BadJSON("invalid 'valid' query parameter"),
-			}
-		}
-		valid = validValue
-	}
-	tokens, err := userAPI.PerformAdminListRegistrationTokens(req.Context(), returnAll, valid)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.ErrorUnknown,
-		}
-	}
-	return util.JSONResponse{
-		Code: 200,
-		JSON: map[string]interface{}{
-			"registration_tokens": tokens,
-		},
-	}
-}
-
-func AdminGetRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-	tokenText := vars["token"]
-	token, err := userAPI.PerformAdminGetRegistrationToken(req.Context(), tokenText)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: spec.NotFound(fmt.Sprintf("token: %s not found", tokenText)),
-		}
-	}
-	return util.JSONResponse{
-		Code: 200,
-		JSON: token,
-	}
-}
-
-func AdminDeleteRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-	tokenText := vars["token"]
-	err = userAPI.PerformAdminDeleteRegistrationToken(req.Context(), tokenText)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: err,
-		}
-	}
-	return util.JSONResponse{
-		Code: 200,
-		JSON: map[string]interface{}{},
-	}
-}
-
-func AdminUpdateRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-	tokenText := vars["token"]
-	request := make(map[string]*int64)
-	if err = json.NewDecoder(req.Body).Decode(&request); err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON(fmt.Sprintf("Failed to decode request body: %s", err)),
-		}
-	}
-	newAttributes := make(map[string]interface{})
-	usesAllowed, ok := request["uses_allowed"]
-	if ok {
-		// Only add usesAllowed to newAtrributes if it is present and valid
-		if usesAllowed != nil && *usesAllowed < 0 {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.BadJSON("uses_allowed must be a non-negative integer or null"),
-			}
-		}
-		newAttributes["usesAllowed"] = usesAllowed
-	}
-	expiryTime, ok := request["expiry_time"]
-	if ok {
-		// Only add expiryTime to newAtrributes if it is present and valid
-		if expiryTime != nil && spec.Timestamp(*expiryTime).Time().Before(time.Now()) {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.BadJSON("expiry_time must not be in the past"),
-			}
-		}
-		newAttributes["expiryTime"] = expiryTime
-	}
-	if len(newAttributes) == 0 {
-		// No attributes to update. Return existing token
-		return AdminGetRegistrationToken(req, cfg, userAPI)
-	}
-	updatedToken, err := userAPI.PerformAdminUpdateRegistrationToken(req.Context(), tokenText, newAttributes)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: spec.NotFound(fmt.Sprintf("token: %s not found", tokenText)),
-		}
-	}
-	return util.JSONResponse{
-		Code: 200,
-		JSON: *updatedToken,
-	}
-}
-
-func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-
-	affected, err := rsAPI.PerformAdminEvacuateRoom(req.Context(), vars["roomID"])
-	switch err.(type) {
-	case nil:
-	case eventutil.ErrRoomNoExists:
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: spec.NotFound(err.Error()),
-		}
-	default:
-		logrus.WithError(err).WithField("roomID", vars["roomID"]).Error("Failed to evacuate room")
-		return util.ErrorResponse(err)
-	}
-	return util.JSONResponse{
-		Code: 200,
-		JSON: map[string]interface{}{
-			"affected": affected,
-		},
-	}
-}
-
-func AdminEvacuateUser(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-
-	affected, err := rsAPI.PerformAdminEvacuateUser(req.Context(), vars["userID"])
-	if err != nil {
-		logrus.WithError(err).WithField("userID", vars["userID"]).Error("Failed to evacuate user")
-		return util.MessageResponse(http.StatusBadRequest, err.Error())
-	}
-
-	return util.JSONResponse{
-		Code: 200,
-		JSON: map[string]interface{}{
-			"affected": affected,
-		},
-	}
-}
-
-func AdminPurgeRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-
-	if err = rsAPI.PerformAdminPurgeRoom(context.Background(), vars["roomID"]); err != nil {
-		return util.ErrorResponse(err)
-	}
-
-	return util.JSONResponse{
-		Code: 200,
-		JSON: struct{}{},
-	}
-}
-
-func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.Device, userAPI api.ClientUserAPI) util.JSONResponse {
-	if req.Body == nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.Unknown("Missing request body"),
-		}
-	}
-	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-	var localpart string
-	userID := vars["userID"]
-	localpart, serverName, err := cfg.Matrix.SplitLocalID('@', userID)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam(err.Error()),
-		}
-	}
-	accAvailableResp := &api.QueryAccountAvailabilityResponse{}
-	if err = userAPI.QueryAccountAvailability(req.Context(), &api.QueryAccountAvailabilityRequest{
-		Localpart:  localpart,
-		ServerName: serverName,
-	}, accAvailableResp); err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
-	}
-	if accAvailableResp.Available {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: spec.Unknown("User does not exist"),
-		}
-	}
-	request := struct {
-		Password      string `json:"password"`
-		LogoutDevices bool   `json:"logout_devices"`
-	}{}
-	if err = json.NewDecoder(req.Body).Decode(&request); err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.Unknown("Failed to decode request body: " + err.Error()),
+			JSON: jsonerror.Unknown("Failed to decode request body: " + err.Error()),
 		}
 	}
 	if request.Password == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.MissingParam("Expecting non-empty password."),
+			JSON: jsonerror.MissingArgument("Expecting non-empty password."),
 		}
 	}
-
-	if err = internal.ValidatePassword(request.Password); err != nil {
-		return *internal.PasswordResponse(err)
-	}
-
-	updateReq := &api.PerformPasswordUpdateRequest{
+	updateReq := &userapi.PerformPasswordUpdateRequest{
 		Localpart:     localpart,
 		ServerName:    serverName,
 		Password:      request.Password,
-		LogoutDevices: request.LogoutDevices,
+		LogoutDevices: true,
 	}
-	updateRes := &api.PerformPasswordUpdateResponse{}
+	updateRes := &userapi.PerformPasswordUpdateResponse{}
 	if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.Unknown("Failed to perform password update: " + err.Error()),
+			JSON: jsonerror.Unknown("Failed to perform password update: " + err.Error()),
 		}
 	}
 	return util.JSONResponse{
@@ -408,14 +151,11 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.De
 	}
 }
 
-func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *api.Device, natsClient *nats.Conn) util.JSONResponse {
+func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, natsClient *nats.Conn) util.JSONResponse {
 	_, err := natsClient.RequestMsg(nats.NewMsg(cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex)), time.Second*10)
 	if err != nil {
 		logrus.WithError(err).Error("failed to publish nats message")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -437,7 +177,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien
 	if cfg.Matrix.IsLocalServerName(domain) {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam("Can not mark local device list as stale"),
+			JSON: jsonerror.InvalidParam("Can not mark local device list as stale"),
 		}
 	}
 
@@ -448,7 +188,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown(fmt.Sprintf("Failed to mark device list as stale: %s", err)),
+			JSON: jsonerror.Unknown(fmt.Sprintf("Failed to mark device list as stale: %s", err)),
 		}
 	}
 	return util.JSONResponse{
@@ -457,7 +197,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien
 	}
 }
 
-func AdminDownloadState(req *http.Request, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -466,122 +206,33 @@ func AdminDownloadState(req *http.Request, device *api.Device, rsAPI roomserverA
 	if !ok {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.MissingParam("Expecting room ID."),
+			JSON: jsonerror.MissingArgument("Expecting room ID."),
 		}
 	}
 	serverName, ok := vars["serverName"]
 	if !ok {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.MissingParam("Expecting remote server name."),
+			JSON: jsonerror.MissingArgument("Expecting remote server name."),
 		}
 	}
-	if err = rsAPI.PerformAdminDownloadState(req.Context(), roomID, device.UserID, spec.ServerName(serverName)); err != nil {
-		if errors.Is(err, eventutil.ErrRoomNoExists{}) {
-			return util.JSONResponse{
-				Code: 200,
-				JSON: spec.NotFound(err.Error()),
-			}
-		}
-		logrus.WithError(err).WithFields(logrus.Fields{
-			"userID":     device.UserID,
-			"serverName": serverName,
-			"roomID":     roomID,
-		}).Error("failed to download state")
-		return util.ErrorResponse(err)
+	res := &roomserverAPI.PerformAdminDownloadStateResponse{}
+	if err := rsAPI.PerformAdminDownloadState(
+		req.Context(),
+		&roomserverAPI.PerformAdminDownloadStateRequest{
+			UserID:     device.UserID,
+			RoomID:     roomID,
+			ServerName: gomatrixserverlib.ServerName(serverName),
+		},
+		res,
+	); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
 	}
 	return util.JSONResponse{
 		Code: 200,
-		JSON: struct{}{},
+		JSON: map[string]interface{}{},
 	}
 }
-
-// GetEventReports returns reported events for a given user/room.
-func GetEventReports(
-	req *http.Request,
-	rsAPI roomserverAPI.ClientRoomserverAPI,
-	from, limit uint64,
-	backwards bool,
-	userID, roomID string,
-) util.JSONResponse {
-
-	eventReports, count, err := rsAPI.QueryAdminEventReports(req.Context(), from, limit, backwards, userID, roomID)
-	if err != nil {
-		logrus.WithError(err).Error("failed to query event reports")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
-	}
-
-	resp := map[string]any{
-		"event_reports": eventReports,
-		"total":         count,
-	}
-
-	// Add a next_token if there are still reports
-	if int64(from+limit) < count {
-		resp["next_token"] = int(from) + len(eventReports)
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: resp,
-	}
-}
-
-func GetEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
-	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			// Given this is an admin endpoint, let them know what didn't work.
-			JSON: spec.InvalidParam(err.Error()),
-		}
-	}
-
-	report, err := rsAPI.QueryAdminEventReport(req.Context(), parsedReportID)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown(err.Error()),
-		}
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: report,
-	}
-}
-
-func DeleteEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
-	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			// Given this is an admin endpoint, let them know what didn't work.
-			JSON: spec.InvalidParam(err.Error()),
-		}
-	}
-
-	err = rsAPI.PerformAdminDeleteEventReport(req.Context(), parsedReportID)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown(err.Error()),
-		}
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: struct{}{},
-	}
-}
-
-func parseUint64OrDefault(input string, defaultValue uint64) uint64 {
-	v, err := strconv.ParseUint(input, 10, 64)
-	if err != nil {
-		return defaultValue
-	}
-	return v
-}

clientapi/routing/admin_whois.go

@@ -17,8 +17,8 @@ package routing
 import (
 	"net/http"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/util"
 )
@@ -51,7 +51,7 @@ func GetAdminWhois(
 	if !allowed {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("userID does not match the current user"),
+			JSON: jsonerror.Forbidden("userID does not match the current user"),
 		}
 	}
 
@@ -61,10 +61,7 @@ func GetAdminWhois(
 	}, &queryRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("GetAdminWhois failed to query user devices")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	devices := make(map[string]deviceInfo)

clientapi/routing/aliases.go

@@ -15,14 +15,14 @@
 package routing
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
+
 	"github.com/matrix-org/util"
 )
 
@@ -31,7 +31,7 @@ func GetAliases(
 	req *http.Request, rsAPI api.ClientRoomserverAPI, device *userapi.Device, roomID string,
 ) util.JSONResponse {
 	stateTuple := gomatrixserverlib.StateKeyTuple{
-		EventType: spec.MRoomHistoryVisibility,
+		EventType: gomatrixserverlib.MRoomHistoryVisibility,
 		StateKey:  "",
 	}
 	stateReq := &api.QueryCurrentStateRequest{
@@ -47,37 +47,26 @@ func GetAliases(
 	visibility := gomatrixserverlib.HistoryVisibilityInvited
 	if historyVisEvent, ok := stateRes.StateEvents[stateTuple]; ok {
 		var err error
-		var content gomatrixserverlib.HistoryVisibilityContent
-		if err = json.Unmarshal(historyVisEvent.Content(), &content); err != nil {
+		visibility, err = historyVisEvent.HistoryVisibility()
+		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("historyVisEvent.HistoryVisibility failed")
 			return util.ErrorResponse(fmt.Errorf("historyVisEvent.HistoryVisibility: %w", err))
 		}
-		visibility = content.HistoryVisibility
 	}
-	if visibility != spec.WorldReadable {
-		deviceUserID, err := spec.NewUserID(device.UserID, true)
-		if err != nil {
-			return util.JSONResponse{
-				Code: http.StatusForbidden,
-				JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
-			}
-		}
+	if visibility != gomatrixserverlib.WorldReadable {
 		queryReq := api.QueryMembershipForUserRequest{
 			RoomID: roomID,
-			UserID: *deviceUserID,
+			UserID: device.UserID,
 		}
 		var queryRes api.QueryMembershipForUserResponse
 		if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed")
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{},
-			}
+			return jsonerror.InternalServerError()
 		}
 		if !queryRes.IsInRoom {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: spec.Forbidden("You aren't a member of this room."),
+				JSON: jsonerror.Forbidden("You aren't a member of this room."),
 			}
 		}
 	}

clientapi/routing/auth_fallback.go

@@ -15,11 +15,11 @@
 package routing
 
 import (
-	"fmt"
 	"html/template"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/util"
 )
@@ -101,28 +101,14 @@ func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]s
 func AuthFallback(
 	w http.ResponseWriter, req *http.Request, authType string,
 	cfg *config.ClientAPI,
-) {
-	// We currently only support "m.login.recaptcha", so fail early if that's not requested
-	if authType == authtypes.LoginTypeRecaptcha {
-		if !cfg.RecaptchaEnabled {
-			writeHTTPMessage(w, req,
-				"Recaptcha login is disabled on this Homeserver",
-				http.StatusBadRequest,
-			)
-			return
-		}
-	} else {
-		writeHTTPMessage(w, req, fmt.Sprintf("Unknown authtype %q", authType), http.StatusNotImplemented)
-		return
-	}
-
+) *util.JSONResponse {
 	sessionID := req.URL.Query().Get("session")
+
 	if sessionID == "" {
-		writeHTTPMessage(w, req,
+		return writeHTTPMessage(w, req,
 			"Session ID not provided",
 			http.StatusBadRequest,
 		)
-		return
 	}
 
 	serveRecaptcha := func() {
@@ -144,44 +130,70 @@ func AuthFallback(
 
 	if req.Method == http.MethodGet {
 		// Handle Recaptcha
-		serveRecaptcha()
-		return
+		if authType == authtypes.LoginTypeRecaptcha {
+			if err := checkRecaptchaEnabled(cfg, w, req); err != nil {
+				return err
+			}
+
+			serveRecaptcha()
+			return nil
+		}
+		return &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Unknown auth stage type"),
+		}
 	} else if req.Method == http.MethodPost {
 		// Handle Recaptcha
-		clientIP := req.RemoteAddr
-		err := req.ParseForm()
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("req.ParseForm failed")
-			w.WriteHeader(http.StatusBadRequest)
-			serveRecaptcha()
-			return
+		if authType == authtypes.LoginTypeRecaptcha {
+			if err := checkRecaptchaEnabled(cfg, w, req); err != nil {
+				return err
+			}
+
+			clientIP := req.RemoteAddr
+			err := req.ParseForm()
+			if err != nil {
+				util.GetLogger(req.Context()).WithError(err).Error("req.ParseForm failed")
+				res := jsonerror.InternalServerError()
+				return &res
+			}
+
+			response := req.Form.Get(cfg.RecaptchaFormField)
+			if err := validateRecaptcha(cfg, response, clientIP); err != nil {
+				util.GetLogger(req.Context()).Error(err)
+				return err
+			}
+
+			// Success. Add recaptcha as a completed login flow
+			sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
+
+			serveSuccess()
+			return nil
 		}
 
-		response := req.Form.Get(cfg.RecaptchaFormField)
-		err = validateRecaptcha(cfg, response, clientIP)
-		switch err {
-		case ErrMissingResponse:
-			w.WriteHeader(http.StatusBadRequest)
-			serveRecaptcha() // serve the initial page again, instead of nothing
-			return
-		case ErrInvalidCaptcha:
-			w.WriteHeader(http.StatusUnauthorized)
-			serveRecaptcha()
-			return
-		case nil:
-		default: // something else failed
-			util.GetLogger(req.Context()).WithError(err).Error("failed to validate recaptcha")
-			serveRecaptcha()
-			return
+		return &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Unknown auth stage type"),
 		}
-
-		// Success. Add recaptcha as a completed login flow
-		sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
-
-		serveSuccess()
-		return
 	}
-	writeHTTPMessage(w, req, "Bad method", http.StatusMethodNotAllowed)
+	return &util.JSONResponse{
+		Code: http.StatusMethodNotAllowed,
+		JSON: jsonerror.NotFound("Bad method"),
+	}
+}
+
+// checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver.
+func checkRecaptchaEnabled(
+	cfg *config.ClientAPI,
+	w http.ResponseWriter,
+	req *http.Request,
+) *util.JSONResponse {
+	if !cfg.RecaptchaEnabled {
+		return writeHTTPMessage(w, req,
+			"Recaptcha login is disabled on this Homeserver",
+			http.StatusBadRequest,
+		)
+	}
+	return nil
 }
 
 // writeHTTPMessage writes the given header and message to the HTTP response writer.
@@ -189,10 +201,13 @@ func AuthFallback(
 func writeHTTPMessage(
 	w http.ResponseWriter, req *http.Request,
 	message string, header int,
-) {
+) *util.JSONResponse {
 	w.WriteHeader(header)
 	_, err := w.Write([]byte(message))
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("w.Write failed")
+		res := jsonerror.InternalServerError()
+		return &res
 	}
+	return nil
 }

clientapi/routing/auth_fallback_test.go

@@ -1,147 +0,0 @@
-package routing
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"strings"
-	"testing"
-
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func Test_AuthFallback(t *testing.T) {
-	cfg := config.Dendrite{}
-	cfg.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
-	for _, useHCaptcha := range []bool{false, true} {
-		for _, recaptchaEnabled := range []bool{false, true} {
-			for _, wantErr := range []bool{false, true} {
-				t.Run(fmt.Sprintf("useHCaptcha(%v) - recaptchaEnabled(%v) - wantErr(%v)", useHCaptcha, recaptchaEnabled, wantErr), func(t *testing.T) {
-					// Set the defaults for each test
-					cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
-					cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled
-					cfg.ClientAPI.RecaptchaPublicKey = "pub"
-					cfg.ClientAPI.RecaptchaPrivateKey = "priv"
-					if useHCaptcha {
-						cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify"
-						cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js"
-						cfg.ClientAPI.RecaptchaFormField = "h-captcha-response"
-						cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha"
-					}
-					cfgErrs := &config.ConfigErrors{}
-					cfg.ClientAPI.Verify(cfgErrs)
-					if len(*cfgErrs) > 0 {
-						t.Fatalf("(hCaptcha=%v) unexpected config errors: %s", useHCaptcha, cfgErrs.Error())
-					}
-
-					req := httptest.NewRequest(http.MethodGet, "/?session=1337", nil)
-					rec := httptest.NewRecorder()
-
-					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
-					if !recaptchaEnabled {
-						if rec.Code != http.StatusBadRequest {
-							t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
-						}
-						if rec.Body.String() != "Recaptcha login is disabled on this Homeserver" {
-							t.Fatalf("unexpected response body: %s", rec.Body.String())
-						}
-					} else {
-						if !strings.Contains(rec.Body.String(), cfg.ClientAPI.RecaptchaSitekeyClass) {
-							t.Fatalf("body does not contain %s: %s", cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String())
-						}
-					}
-
-					srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-						if wantErr {
-							_, _ = w.Write([]byte(`{"success":false}`))
-							return
-						}
-						_, _ = w.Write([]byte(`{"success":true}`))
-					}))
-					defer srv.Close() // nolint: errcheck
-
-					cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
-
-					// check the result after sending the captcha
-					req = httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
-					req.Form = url.Values{}
-					req.Form.Add(cfg.ClientAPI.RecaptchaFormField, "someRandomValue")
-					rec = httptest.NewRecorder()
-					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
-					if recaptchaEnabled {
-						if !wantErr {
-							if rec.Code != http.StatusOK {
-								t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusOK)
-							}
-							if rec.Body.String() != successTemplate {
-								t.Fatalf("unexpected response: %s, want %s", rec.Body.String(), successTemplate)
-							}
-						} else {
-							if rec.Code != http.StatusUnauthorized {
-								t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusUnauthorized)
-							}
-							wantString := "Authentication"
-							if !strings.Contains(rec.Body.String(), wantString) {
-								t.Fatalf("expected response to contain '%s', but didn't: %s", wantString, rec.Body.String())
-							}
-						}
-					} else {
-						if rec.Code != http.StatusBadRequest {
-							t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
-						}
-						if rec.Body.String() != "Recaptcha login is disabled on this Homeserver" {
-							t.Fatalf("unexpected response: %s, want %s", rec.Body.String(), "successTemplate")
-						}
-					}
-				})
-			}
-		}
-	}
-
-	t.Run("unknown fallbacks are handled correctly", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
-		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, "DoesNotExist", &cfg.ClientAPI)
-		if rec.Code != http.StatusNotImplemented {
-			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusNotImplemented)
-		}
-	})
-
-	t.Run("unknown methods are handled correctly", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodDelete, "/?session=1337", nil)
-		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
-		if rec.Code != http.StatusMethodNotAllowed {
-			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusMethodNotAllowed)
-		}
-	})
-
-	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodGet, "/", nil)
-		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
-		if rec.Code != http.StatusBadRequest {
-			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
-		}
-	})
-
-	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodGet, "/", nil)
-		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
-		if rec.Code != http.StatusBadRequest {
-			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
-		}
-	})
-
-	t.Run("missing 'response' is handled correctly", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
-		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
-		if rec.Code != http.StatusBadRequest {
-			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
-		}
-	})
-}

clientapi/routing/capabilities.go

@@ -17,22 +17,26 @@ package routing
 import (
 	"net/http"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/version"
-	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/util"
 )
 
 // GetCapabilities returns information about the server's supported feature set
 // and other relevant capabilities to an authenticated user.
-func GetCapabilities(rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	versionsMap := map[gomatrixserverlib.RoomVersion]string{}
-	for v, desc := range version.SupportedRoomVersions() {
-		if desc.Stable() {
-			versionsMap[v] = "stable"
-		} else {
-			versionsMap[v] = "unstable"
-		}
+func GetCapabilities(
+	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
+) util.JSONResponse {
+	roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{}
+	roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{}
+	if err := rsAPI.QueryRoomVersionCapabilities(
+		req.Context(),
+		&roomVersionsQueryReq,
+		&roomVersionsQueryRes,
+	); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryRoomVersionCapabilities failed")
+		return jsonerror.InternalServerError()
 	}
 
 	response := map[string]interface{}{
@@ -40,10 +44,7 @@ func GetCapabilities(rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse
 			"m.change_password": map[string]bool{
 				"enabled": true,
 			},
-			"m.room_versions": map[string]interface{}{
-				"default":   rsAPI.DefaultRoomVersion(),
-				"available": versionsMap,
-			},
+			"m.room_versions": roomVersionsQueryRes,
 		},
 	}
 

clientapi/routing/createroom.go

@@ -26,9 +26,10 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -37,19 +38,33 @@ import (
 
 // https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom
 type createRoomRequest struct {
-	Invite                    []string                           `json:"invite"`
-	Name                      string                             `json:"name"`
-	Visibility                string                             `json:"visibility"`
-	Topic                     string                             `json:"topic"`
-	Preset                    string                             `json:"preset"`
-	CreationContent           json.RawMessage                    `json:"creation_content"`
-	InitialState              []gomatrixserverlib.FledglingEvent `json:"initial_state"`
-	RoomAliasName             string                             `json:"room_alias_name"`
-	RoomVersion               gomatrixserverlib.RoomVersion      `json:"room_version"`
-	PowerLevelContentOverride json.RawMessage                    `json:"power_level_content_override"`
-	IsDirect                  bool                               `json:"is_direct"`
+	Invite                    []string                      `json:"invite"`
+	Name                      string                        `json:"name"`
+	Visibility                string                        `json:"visibility"`
+	Topic                     string                        `json:"topic"`
+	Preset                    string                        `json:"preset"`
+	CreationContent           json.RawMessage               `json:"creation_content"`
+	InitialState              []fledglingEvent              `json:"initial_state"`
+	RoomAliasName             string                        `json:"room_alias_name"`
+	GuestCanJoin              bool                          `json:"guest_can_join"`
+	RoomVersion               gomatrixserverlib.RoomVersion `json:"room_version"`
+	PowerLevelContentOverride json.RawMessage               `json:"power_level_content_override"`
+	IsDirect                  bool                          `json:"is_direct"`
 }
 
+const (
+	presetPrivateChat        = "private_chat"
+	presetTrustedPrivateChat = "trusted_private_chat"
+	presetPublicChat         = "public_chat"
+)
+
+const (
+	historyVisibilityShared = "shared"
+	// TODO: These should be implemented once history visibility is implemented
+	// historyVisibilityWorldReadable = "world_readable"
+	// historyVisibilityInvited       = "invited"
+)
+
 func (r createRoomRequest) Validate() *util.JSONResponse {
 	whitespace := "\t\n\x0b\x0c\r " // https://docs.python.org/2/library/string.html#string.whitespace
 	// https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/handlers/room.py#L81
@@ -57,23 +72,28 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 	if strings.ContainsAny(r.RoomAliasName, whitespace+":") {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("room_alias_name cannot contain whitespace or ':'"),
+			JSON: jsonerror.BadJSON("room_alias_name cannot contain whitespace or ':'"),
 		}
 	}
 	for _, userID := range r.Invite {
-		if _, err := spec.NewUserID(userID, true); err != nil {
+		// TODO: We should put user ID parsing code into gomatrixserverlib and use that instead
+		//       (see https://github.com/matrix-org/gomatrixserverlib/blob/3394e7c7003312043208aa73727d2256eea3d1f6/eventcontent.go#L347 )
+		//       It should be a struct (with pointers into a single string to avoid copying) and
+		//       we should update all refs to use UserID types rather than strings.
+		// https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/types.py#L92
+		if _, _, err := gomatrixserverlib.SplitID('@', userID); err != nil {
 			return &util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: spec.BadJSON("user id must be in the form @localpart:domain"),
+				JSON: jsonerror.BadJSON("user id must be in the form @localpart:domain"),
 			}
 		}
 	}
 	switch r.Preset {
-	case spec.PresetPrivateChat, spec.PresetTrustedPrivateChat, spec.PresetPublicChat, "":
+	case presetPrivateChat, presetTrustedPrivateChat, presetPublicChat, "":
 	default:
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("preset must be any of 'private_chat', 'trusted_private_chat', 'public_chat'"),
+			JSON: jsonerror.BadJSON("preset must be any of 'private_chat', 'trusted_private_chat', 'public_chat'"),
 		}
 	}
 
@@ -85,7 +105,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 	if err != nil {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("malformed creation_content"),
+			JSON: jsonerror.BadJSON("malformed creation_content"),
 		}
 	}
 
@@ -94,7 +114,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 	if err != nil {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("malformed creation_content"),
+			JSON: jsonerror.BadJSON("malformed creation_content"),
 		}
 	}
 
@@ -107,6 +127,13 @@ type createRoomResponse struct {
 	RoomAlias string `json:"room_alias,omitempty"` // in synapse not spec
 }
 
+// fledglingEvent is a helper representation of an event used when creating many events in succession.
+type fledglingEvent struct {
+	Type     string      `json:"type"`
+	StateKey string      `json:"state_key"`
+	Content  interface{} `json:"content"`
+}
+
 // CreateRoom implements /createRoom
 func CreateRoom(
 	req *http.Request, device *api.Device,
@@ -114,124 +141,456 @@ func CreateRoom(
 	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
-	var createRequest createRoomRequest
-	resErr := httputil.UnmarshalJSONRequest(req, &createRequest)
+	var r createRoomRequest
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
 		return *resErr
 	}
-	if resErr = createRequest.Validate(); resErr != nil {
+	if resErr = r.Validate(); resErr != nil {
 		return *resErr
 	}
 	evTime, err := httputil.ParseTSParam(req)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam(err.Error()),
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
 		}
 	}
-	return createRoom(req.Context(), createRequest, device, cfg, profileAPI, rsAPI, asAPI, evTime)
+	return createRoom(req.Context(), r, device, cfg, profileAPI, rsAPI, asAPI, evTime)
 }
 
 // createRoom implements /createRoom
+// nolint: gocyclo
 func createRoom(
 	ctx context.Context,
-	createRequest createRoomRequest, device *api.Device,
+	r createRoomRequest, device *api.Device,
 	cfg *config.ClientAPI,
 	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 	evTime time.Time,
 ) util.JSONResponse {
-	userID, err := spec.NewUserID(device.UserID, true)
+	_, userDomain, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("invalid userID")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed")
+		return jsonerror.InternalServerError()
 	}
-	if !cfg.Matrix.IsLocalServerName(userID.Domain()) {
+	if !cfg.Matrix.IsLocalServerName(userDomain) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden(fmt.Sprintf("User domain %q not configured locally", userID.Domain())),
+			JSON: jsonerror.Forbidden(fmt.Sprintf("User domain %q not configured locally", userDomain)),
 		}
 	}
 
+	// TODO (#267): Check room ID doesn't clash with an existing one, and we
+	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
+	roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), userDomain)
+
 	logger := util.GetLogger(ctx)
-
-	// TODO: Check room ID doesn't clash with an existing one, and we
-	//       probably shouldn't be using pseudo-random strings, maybe GUIDs?
-	roomID, err := spec.NewRoomID(fmt.Sprintf("!%s:%s", util.RandomString(16), userID.Domain()))
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("invalid roomID")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
-	}
+	userID := device.UserID
 
 	// Clobber keys: creator, room_version
 
-	roomVersion := rsAPI.DefaultRoomVersion()
-	if createRequest.RoomVersion != "" {
-		candidateVersion := gomatrixserverlib.RoomVersion(createRequest.RoomVersion)
+	roomVersion := roomserverVersion.DefaultRoomVersion()
+	if r.RoomVersion != "" {
+		candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion)
 		_, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion)
 		if roomVersionError != nil {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: spec.UnsupportedRoomVersion(roomVersionError.Error()),
+				JSON: jsonerror.UnsupportedRoomVersion(roomVersionError.Error()),
 			}
 		}
 		roomVersion = candidateVersion
 	}
 
+	// TODO: visibility/presets/raw initial state
+	// TODO: Create room alias association
+	// Make sure this doesn't fall into an application service's namespace though!
+
 	logger.WithFields(log.Fields{
-		"userID":      userID.String(),
-		"roomID":      roomID.String(),
+		"userID":      userID,
+		"roomID":      roomID,
 		"roomVersion": roomVersion,
 	}).Info("Creating new room")
 
-	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID.String(), asAPI, profileAPI)
+	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
+		return jsonerror.InternalServerError()
+	}
+
+	createContent := map[string]interface{}{}
+	if len(r.CreationContent) > 0 {
+		if err = json.Unmarshal(r.CreationContent, &createContent); err != nil {
+			util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for creation_content failed")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("invalid create content"),
+			}
+		}
+	}
+	createContent["creator"] = userID
+	createContent["room_version"] = roomVersion
+	powerLevelContent := eventutil.InitialPowerLevelsContent(userID)
+	joinRuleContent := gomatrixserverlib.JoinRuleContent{
+		JoinRule: gomatrixserverlib.Invite,
+	}
+	historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{
+		HistoryVisibility: historyVisibilityShared,
+	}
+
+	if r.PowerLevelContentOverride != nil {
+		// Merge powerLevelContentOverride fields by unmarshalling it atop the defaults
+		err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for power_level_content_override failed")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("malformed power_level_content_override"),
+			}
 		}
 	}
 
-	userDisplayName := profile.DisplayName
-	userAvatarURL := profile.AvatarURL
-
-	keyID := cfg.Matrix.KeyID
-	privateKey := cfg.Matrix.PrivateKey
-
-	req := roomserverAPI.PerformCreateRoomRequest{
-		InvitedUsers:              createRequest.Invite,
-		RoomName:                  createRequest.Name,
-		Visibility:                createRequest.Visibility,
-		Topic:                     createRequest.Topic,
-		StatePreset:               createRequest.Preset,
-		CreationContent:           createRequest.CreationContent,
-		InitialState:              createRequest.InitialState,
-		RoomAliasName:             createRequest.RoomAliasName,
-		RoomVersion:               roomVersion,
-		PowerLevelContentOverride: createRequest.PowerLevelContentOverride,
-		IsDirect:                  createRequest.IsDirect,
-
-		UserDisplayName: userDisplayName,
-		UserAvatarURL:   userAvatarURL,
-		KeyID:           keyID,
-		PrivateKey:      privateKey,
-		EventTime:       evTime,
+	switch r.Preset {
+	case presetPrivateChat:
+		joinRuleContent.JoinRule = gomatrixserverlib.Invite
+		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
+	case presetTrustedPrivateChat:
+		joinRuleContent.JoinRule = gomatrixserverlib.Invite
+		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
+		for _, invitee := range r.Invite {
+			powerLevelContent.Users[invitee] = 100
+		}
+	case presetPublicChat:
+		joinRuleContent.JoinRule = gomatrixserverlib.Public
+		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
 	}
 
-	roomAlias, createRes := rsAPI.PerformCreateRoom(ctx, *userID, *roomID, &req)
-	if createRes != nil {
-		return *createRes
+	createEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomCreate,
+		Content: createContent,
+	}
+	powerLevelEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomPowerLevels,
+		Content: powerLevelContent,
+	}
+	joinRuleEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomJoinRules,
+		Content: joinRuleContent,
+	}
+	historyVisibilityEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomHistoryVisibility,
+		Content: historyVisibilityContent,
+	}
+	membershipEvent := fledglingEvent{
+		Type:     gomatrixserverlib.MRoomMember,
+		StateKey: userID,
+		Content: gomatrixserverlib.MemberContent{
+			Membership:  gomatrixserverlib.Join,
+			DisplayName: profile.DisplayName,
+			AvatarURL:   profile.AvatarURL,
+		},
+	}
+
+	var nameEvent *fledglingEvent
+	var topicEvent *fledglingEvent
+	var guestAccessEvent *fledglingEvent
+	var aliasEvent *fledglingEvent
+
+	if r.Name != "" {
+		nameEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomName,
+			Content: eventutil.NameContent{
+				Name: r.Name,
+			},
+		}
+	}
+
+	if r.Topic != "" {
+		topicEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomTopic,
+			Content: eventutil.TopicContent{
+				Topic: r.Topic,
+			},
+		}
+	}
+
+	if r.GuestCanJoin {
+		guestAccessEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomGuestAccess,
+			Content: eventutil.GuestAccessContent{
+				GuestAccess: "can_join",
+			},
+		}
+	}
+
+	var roomAlias string
+	if r.RoomAliasName != "" {
+		roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, userDomain)
+		// check it's free TODO: This races but is better than nothing
+		hasAliasReq := roomserverAPI.GetRoomIDForAliasRequest{
+			Alias:              roomAlias,
+			IncludeAppservices: false,
+		}
+
+		var aliasResp roomserverAPI.GetRoomIDForAliasResponse
+		err = rsAPI.GetRoomIDForAlias(ctx, &hasAliasReq, &aliasResp)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed")
+			return jsonerror.InternalServerError()
+		}
+		if aliasResp.RoomID != "" {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.RoomInUse("Room ID already exists."),
+			}
+		}
+
+		aliasEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomCanonicalAlias,
+			Content: eventutil.CanonicalAlias{
+				Alias: roomAlias,
+			},
+		}
+	}
+
+	var initialStateEvents []fledglingEvent
+	for i := range r.InitialState {
+		if r.InitialState[i].StateKey != "" {
+			initialStateEvents = append(initialStateEvents, r.InitialState[i])
+			continue
+		}
+
+		switch r.InitialState[i].Type {
+		case gomatrixserverlib.MRoomCreate:
+			continue
+
+		case gomatrixserverlib.MRoomPowerLevels:
+			powerLevelEvent = r.InitialState[i]
+
+		case gomatrixserverlib.MRoomJoinRules:
+			joinRuleEvent = r.InitialState[i]
+
+		case gomatrixserverlib.MRoomHistoryVisibility:
+			historyVisibilityEvent = r.InitialState[i]
+
+		case gomatrixserverlib.MRoomGuestAccess:
+			guestAccessEvent = &r.InitialState[i]
+
+		case gomatrixserverlib.MRoomName:
+			nameEvent = &r.InitialState[i]
+
+		case gomatrixserverlib.MRoomTopic:
+			topicEvent = &r.InitialState[i]
+
+		default:
+			initialStateEvents = append(initialStateEvents, r.InitialState[i])
+		}
+	}
+
+	// send events into the room in order of:
+	//  1- m.room.create
+	//  2- room creator join member
+	//  3- m.room.power_levels
+	//  4- m.room.join_rules
+	//  5- m.room.history_visibility
+	//  6- m.room.canonical_alias (opt)
+	//  7- m.room.guest_access (opt)
+	//  8- other initial state items
+	//  9- m.room.name (opt)
+	//  10- m.room.topic (opt)
+	//  11- invite events (opt) - with is_direct flag if applicable TODO
+	//  12- 3pid invite events (opt) TODO
+	// This differs from Synapse slightly. Synapse would vary the ordering of 3-7
+	// depending on if those events were in "initial_state" or not. This made it
+	// harder to reason about, hence sticking to a strict static ordering.
+	// TODO: Synapse has txn/token ID on each event. Do we need to do this here?
+	eventsToMake := []fledglingEvent{
+		createEvent, membershipEvent, powerLevelEvent, joinRuleEvent, historyVisibilityEvent,
+	}
+	if guestAccessEvent != nil {
+		eventsToMake = append(eventsToMake, *guestAccessEvent)
+	}
+	eventsToMake = append(eventsToMake, initialStateEvents...)
+	if nameEvent != nil {
+		eventsToMake = append(eventsToMake, *nameEvent)
+	}
+	if topicEvent != nil {
+		eventsToMake = append(eventsToMake, *topicEvent)
+	}
+	if aliasEvent != nil {
+		// TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room.
+		// This means we might fail creating the alias but say the canonical alias is something that doesn't exist.
+		eventsToMake = append(eventsToMake, *aliasEvent)
+	}
+
+	// TODO: invite events
+	// TODO: 3pid invite events
+
+	var builtEvents []*gomatrixserverlib.HeaderedEvent
+	authEvents := gomatrixserverlib.NewAuthEvents(nil)
+	for i, e := range eventsToMake {
+		depth := i + 1 // depth starts at 1
+
+		builder := gomatrixserverlib.EventBuilder{
+			Sender:   userID,
+			RoomID:   roomID,
+			Type:     e.Type,
+			StateKey: &e.StateKey,
+			Depth:    int64(depth),
+		}
+		err = builder.SetContent(e.Content)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("builder.SetContent failed")
+			return jsonerror.InternalServerError()
+		}
+		if i > 0 {
+			builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
+		}
+		var ev *gomatrixserverlib.Event
+		ev, err = buildEvent(&builder, userDomain, &authEvents, cfg, evTime, roomVersion)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("buildEvent failed")
+			return jsonerror.InternalServerError()
+		}
+
+		if err = gomatrixserverlib.Allowed(ev, &authEvents); err != nil {
+			util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.Allowed failed")
+			return jsonerror.InternalServerError()
+		}
+
+		// Add the event to the list of auth events
+		builtEvents = append(builtEvents, ev.Headered(roomVersion))
+		err = authEvents.AddEvent(ev)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("authEvents.AddEvent failed")
+			return jsonerror.InternalServerError()
+		}
+	}
+
+	inputs := make([]roomserverAPI.InputRoomEvent, 0, len(builtEvents))
+	for _, event := range builtEvents {
+		inputs = append(inputs, roomserverAPI.InputRoomEvent{
+			Kind:         roomserverAPI.KindNew,
+			Event:        event,
+			Origin:       userDomain,
+			SendAsServer: roomserverAPI.DoNotSendToOtherServers,
+		})
+	}
+	if err = roomserverAPI.SendInputRoomEvents(ctx, rsAPI, device.UserDomain(), inputs, false); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed")
+		return jsonerror.InternalServerError()
+	}
+
+	// TODO(#269): Reserve room alias while we create the room. This stops us
+	// from creating the room but still failing due to the alias having already
+	// been taken.
+	if roomAlias != "" {
+		aliasReq := roomserverAPI.SetRoomAliasRequest{
+			Alias:  roomAlias,
+			RoomID: roomID,
+			UserID: userID,
+		}
+
+		var aliasResp roomserverAPI.SetRoomAliasResponse
+		err = rsAPI.SetRoomAlias(ctx, &aliasReq, &aliasResp)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("aliasAPI.SetRoomAlias failed")
+			return jsonerror.InternalServerError()
+		}
+
+		if aliasResp.AliasExists {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.RoomInUse("Room alias already exists."),
+			}
+		}
+	}
+
+	// If this is a direct message then we should invite the participants.
+	if len(r.Invite) > 0 {
+		// Build some stripped state for the invite.
+		var globalStrippedState []gomatrixserverlib.InviteV2StrippedState
+		for _, event := range builtEvents {
+			// Chosen events from the spec:
+			// https://spec.matrix.org/v1.3/client-server-api/#stripped-state
+			switch event.Type() {
+			case gomatrixserverlib.MRoomCreate:
+				fallthrough
+			case gomatrixserverlib.MRoomName:
+				fallthrough
+			case gomatrixserverlib.MRoomAvatar:
+				fallthrough
+			case gomatrixserverlib.MRoomTopic:
+				fallthrough
+			case gomatrixserverlib.MRoomCanonicalAlias:
+				fallthrough
+			case gomatrixserverlib.MRoomEncryption:
+				fallthrough
+			case gomatrixserverlib.MRoomMember:
+				fallthrough
+			case gomatrixserverlib.MRoomJoinRules:
+				ev := event.Event
+				globalStrippedState = append(
+					globalStrippedState,
+					gomatrixserverlib.NewInviteV2StrippedState(ev),
+				)
+			}
+		}
+
+		// Process the invites.
+		for _, invitee := range r.Invite {
+			// Build the invite event.
+			inviteEvent, err := buildMembershipEvent(
+				ctx, invitee, "", profileAPI, device, gomatrixserverlib.Invite,
+				roomID, r.IsDirect, cfg, evTime, rsAPI, asAPI,
+			)
+			if err != nil {
+				util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
+				continue
+			}
+			inviteStrippedState := append(
+				globalStrippedState,
+				gomatrixserverlib.NewInviteV2StrippedState(inviteEvent.Event),
+			)
+			// Send the invite event to the roomserver.
+			var inviteRes roomserverAPI.PerformInviteResponse
+			event := inviteEvent.Headered(roomVersion)
+			if err := rsAPI.PerformInvite(ctx, &roomserverAPI.PerformInviteRequest{
+				Event:           event,
+				InviteRoomState: inviteStrippedState,
+				RoomVersion:     event.RoomVersion,
+				SendAsServer:    string(userDomain),
+			}, &inviteRes); err != nil {
+				util.GetLogger(ctx).WithError(err).Error("PerformInvite failed")
+				return util.JSONResponse{
+					Code: http.StatusInternalServerError,
+					JSON: jsonerror.InternalServerError(),
+				}
+			}
+			if inviteRes.Error != nil {
+				return inviteRes.Error.JSONResponse()
+			}
+		}
+	}
+
+	if r.Visibility == "public" {
+		// expose this room in the published room list
+		var pubRes roomserverAPI.PerformPublishResponse
+		if err := rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{
+			RoomID:     roomID,
+			Visibility: "public",
+		}, &pubRes); err != nil {
+			return jsonerror.InternalAPIError(ctx, err)
+		}
+		if pubRes.Error != nil {
+			// treat as non-fatal since the room is already made by this point
+			util.GetLogger(ctx).WithError(pubRes.Error).Error("failed to visibility:public")
+		}
 	}
 
 	response := createRoomResponse{
-		RoomID:    roomID.String(),
+		RoomID:    roomID,
 		RoomAlias: roomAlias,
 	}
 
@@ -240,3 +599,31 @@ func createRoom(
 		JSON: response,
 	}
 }
+
+// buildEvent fills out auth_events for the builder then builds the event
+func buildEvent(
+	builder *gomatrixserverlib.EventBuilder,
+	serverName gomatrixserverlib.ServerName,
+	provider gomatrixserverlib.AuthEventProvider,
+	cfg *config.ClientAPI,
+	evTime time.Time,
+	roomVersion gomatrixserverlib.RoomVersion,
+) (*gomatrixserverlib.Event, error) {
+	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
+	if err != nil {
+		return nil, err
+	}
+	refs, err := eventsNeeded.AuthEventReferences(provider)
+	if err != nil {
+		return nil, err
+	}
+	builder.AuthEvents = refs
+	event, err := builder.Build(
+		evTime, serverName, cfg.Matrix.KeyID,
+		cfg.Matrix.PrivateKey, roomVersion,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err)
+	}
+	return event, nil
+}

clientapi/routing/deactivate.go

@@ -5,9 +5,9 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -24,7 +24,7 @@ func Deactivate(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("The request body could not be read: " + err.Error()),
+			JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
 		}
 	}
 
@@ -33,26 +33,19 @@ func Deactivate(
 		return *errRes
 	}
 
-	localpart, serverName, err := gomatrixserverlib.SplitID('@', login.Username())
+	localpart, _, err := gomatrixserverlib.SplitID('@', login.Username())
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	var res api.PerformAccountDeactivationResponse
 	err = accountAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{
-		Localpart:  localpart,
-		ServerName: serverName,
+		Localpart: localpart,
 	}, &res)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformAccountDeactivation failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	return util.JSONResponse{

clientapi/routing/device.go

@@ -15,16 +15,15 @@
 package routing
 
 import (
-	"encoding/json"
 	"io"
 	"net"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/tidwall/gjson"
 )
@@ -60,10 +59,7 @@ func GetDeviceByID(
 	}, &queryRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryDevices failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 	var targetDevice *api.Device
 	for _, device := range queryRes.Devices {
@@ -75,7 +71,7 @@ func GetDeviceByID(
 	if targetDevice == nil {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: spec.NotFound("Unknown device"),
+			JSON: jsonerror.NotFound("Unknown device"),
 		}
 	}
 
@@ -100,10 +96,7 @@ func GetDevicesByLocalpart(
 	}, &queryRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryDevices failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	res := devicesJSON{}
@@ -145,15 +138,18 @@ func UpdateDeviceByID(
 	}, &performRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("PerformDeviceUpdate failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 	if !performRes.DeviceExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: spec.Forbidden("device does not exist"),
+			JSON: jsonerror.Forbidden("device does not exist"),
+		}
+	}
+	if performRes.Forbidden {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("device not owned by current user"),
 		}
 	}
 
@@ -183,7 +179,7 @@ func DeleteDeviceById(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("The request body could not be read: " + err.Error()),
+			JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
 		}
 	}
 
@@ -193,7 +189,7 @@ func DeleteDeviceById(
 		if dev != deviceID {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: spec.Forbidden("session and device mismatch"),
+				JSON: jsonerror.Forbidden("session & device mismatch"),
 			}
 		}
 	}
@@ -215,10 +211,7 @@ func DeleteDeviceById(
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	// make sure that the access token being used matches the login creds used for user interactive auth, else
@@ -226,7 +219,7 @@ func DeleteDeviceById(
 	if login.Username() != localpart && login.Username() != device.UserID {
 		return util.JSONResponse{
 			Code: 403,
-			JSON: spec.Forbidden("Cannot delete another user's device"),
+			JSON: jsonerror.Forbidden("Cannot delete another user's device"),
 		}
 	}
 
@@ -236,10 +229,7 @@ func DeleteDeviceById(
 		DeviceIDs: []string{deviceID},
 	}, &res); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformDeviceDeletion failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	deleteOK = true
@@ -252,51 +242,24 @@ func DeleteDeviceById(
 
 // DeleteDevices handles POST requests to /delete_devices
 func DeleteDevices(
-	req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.ClientUserAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 ) util.JSONResponse {
 	ctx := req.Context()
-
-	bodyBytes, err := io.ReadAll(req.Body)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("The request body could not be read: " + err.Error()),
-		}
-	}
-	defer req.Body.Close() // nolint:errcheck
-
-	// initiate UIA
-	login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, device)
-	if errRes != nil {
-		return *errRes
-	}
-
-	if login.Username() != device.UserID {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("unable to delete devices for other user"),
-		}
-	}
-
 	payload := devicesDeleteJSON{}
-	if err = json.Unmarshal(bodyBytes, &payload); err != nil {
-		util.GetLogger(ctx).WithError(err).Error("unable to unmarshal device deletion request")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+
+	if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil {
+		return *resErr
 	}
 
+	defer req.Body.Close() // nolint: errcheck
+
 	var res api.PerformDeviceDeletionResponse
 	if err := userAPI.PerformDeviceDeletion(ctx, &api.PerformDeviceDeletionRequest{
 		UserID:    device.UserID,
 		DeviceIDs: payload.Devices,
 	}, &res); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformDeviceDeletion failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	return util.JSONResponse{

clientapi/routing/directory.go

@@ -19,11 +19,10 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/fclient"
-	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -35,7 +34,7 @@ type roomDirectoryResponse struct {
 	Servers []string `json:"servers"`
 }
 
-func (r *roomDirectoryResponse) fillServers(servers []spec.ServerName) {
+func (r *roomDirectoryResponse) fillServers(servers []gomatrixserverlib.ServerName) {
 	r.Servers = make([]string, len(servers))
 	for i, s := range servers {
 		r.Servers[i] = string(s)
@@ -46,7 +45,7 @@ func (r *roomDirectoryResponse) fillServers(servers []spec.ServerName) {
 func DirectoryRoom(
 	req *http.Request,
 	roomAlias string,
-	federation fclient.FederationClient,
+	federation *gomatrixserverlib.FederationClient,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	fedSenderAPI federationAPI.ClientFederationAPI,
@@ -55,7 +54,7 @@ func DirectoryRoom(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
+			JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 
@@ -69,10 +68,7 @@ func DirectoryRoom(
 	queryRes := &roomserverAPI.GetRoomIDForAliasResponse{}
 	if err = rsAPI.GetRoomIDForAlias(req.Context(), queryReq, queryRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.GetRoomIDForAlias failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	res.RoomID = queryRes.RoomID
@@ -86,10 +82,7 @@ func DirectoryRoom(
 				// TODO: Return 502 if the remote server errored.
 				// TODO: Return 504 if the remote server timed out.
 				util.GetLogger(req.Context()).WithError(fedErr).Error("federation.LookupRoomAlias failed")
-				return util.JSONResponse{
-					Code: http.StatusInternalServerError,
-					JSON: spec.InternalServerError{},
-				}
+				return jsonerror.InternalServerError()
 			}
 			res.RoomID = fedRes.RoomID
 			res.fillServers(fedRes.Servers)
@@ -98,7 +91,7 @@ func DirectoryRoom(
 		if res.RoomID == "" {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
-				JSON: spec.NotFound(
+				JSON: jsonerror.NotFound(
 					fmt.Sprintf("Room alias %s not found", roomAlias),
 				),
 			}
@@ -108,10 +101,7 @@ func DirectoryRoom(
 		var joinedHostsRes federationAPI.QueryJoinedHostServerNamesInRoomResponse
 		if err = fedSenderAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &joinedHostsReq, &joinedHostsRes); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("fedSenderAPI.QueryJoinedHostServerNamesInRoom failed")
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{},
-			}
+			return jsonerror.InternalServerError()
 		}
 		res.fillServers(joinedHostsRes.ServerNames)
 	}
@@ -134,14 +124,14 @@ func SetLocalAlias(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
+			JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 
 	if !cfg.Matrix.IsLocalServerName(domain) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("Alias must be on local homeserver"),
+			JSON: jsonerror.Forbidden("Alias must be on local homeserver"),
 		}
 	}
 
@@ -154,7 +144,7 @@ func SetLocalAlias(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("User ID must be in the form '@localpart:domain'"),
+			JSON: jsonerror.BadJSON("User ID must be in the form '@localpart:domain'"),
 		}
 	}
 	for _, appservice := range cfg.Derived.ApplicationServices {
@@ -166,7 +156,7 @@ func SetLocalAlias(
 					if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) {
 						return util.JSONResponse{
 							Code: http.StatusBadRequest,
-							JSON: spec.ASExclusive("Alias is reserved by an application service"),
+							JSON: jsonerror.ASExclusive("Alias is reserved by an application service"),
 						}
 					}
 				}
@@ -181,50 +171,21 @@ func SetLocalAlias(
 		return *resErr
 	}
 
-	roomID, err := spec.NewRoomID(r.RoomID)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam("invalid room ID"),
-		}
+	queryReq := roomserverAPI.SetRoomAliasRequest{
+		UserID: device.UserID,
+		RoomID: r.RoomID,
+		Alias:  alias,
 	}
-
-	userID, err := spec.NewUserID(device.UserID, true)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
-		}
-	}
-
-	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *roomID, *userID)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("QuerySenderIDForUser failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
-		}
-	} else if senderID == nil {
-		util.GetLogger(req.Context()).WithField("roomID", *roomID).WithField("userID", *userID).Error("Sender ID not found")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
-		}
-	}
-
-	aliasAlreadyExists, err := rsAPI.SetRoomAlias(req.Context(), *senderID, *roomID, alias)
-	if err != nil {
+	var queryRes roomserverAPI.SetRoomAliasResponse
+	if err := rsAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
-	if aliasAlreadyExists {
+	if queryRes.AliasExists {
 		return util.JSONResponse{
 			Code: http.StatusConflict,
-			JSON: spec.Unknown("The alias " + alias + " already exists."),
+			JSON: jsonerror.Unknown("The alias " + alias + " already exists."),
 		}
 	}
 
@@ -241,91 +202,27 @@ func RemoveLocalAlias(
 	alias string,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 ) util.JSONResponse {
-	userID, err := spec.NewUserID(device.UserID, true)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{Err: "UserID for device is invalid"},
-		}
+	queryReq := roomserverAPI.RemoveRoomAliasRequest{
+		Alias:  alias,
+		UserID: device.UserID,
 	}
-
-	roomIDReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: alias}
-	roomIDRes := roomserverAPI.GetRoomIDForAliasResponse{}
-	err = rsAPI.GetRoomIDForAlias(req.Context(), &roomIDReq, &roomIDRes)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: spec.NotFound("The alias does not exist."),
-		}
-	}
-
-	validRoomID, err := spec.NewRoomID(roomIDRes.RoomID)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: spec.NotFound("The alias does not exist."),
-		}
-	}
-
-	// This seems like the kind of auth check that should be done in the roomserver, but
-	// if this check fails (user is not in the room), then there will be no SenderID for the user
-	// for pseudo-ID rooms - it will just return "". However, we can't use lack of a sender ID
-	// as meaning they are not in the room, since lacking a sender ID could be caused by other bugs.
-	// TODO: maybe have QuerySenderIDForUser return richer errors?
-	var queryResp roomserverAPI.QueryMembershipForUserResponse
-	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
-		RoomID: validRoomID.String(),
-		UserID: *userID,
-	}, &queryResp)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.QueryMembershipForUser failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
-		}
-	}
-	if !queryResp.IsInRoom {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("You do not have permission to remove this alias."),
-		}
-	}
-
-	deviceSenderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *userID)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: spec.NotFound("The alias does not exist."),
-		}
-	}
-	// TODO: how to handle this case? missing user/room keys seem to be a whole new class of errors
-	if deviceSenderID == nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
-		}
-	}
-
-	aliasFound, aliasRemoved, err := rsAPI.RemoveRoomAlias(req.Context(), *deviceSenderID, alias)
-	if err != nil {
+	var queryRes roomserverAPI.RemoveRoomAliasResponse
+	if err := rsAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
-		}
+		return jsonerror.InternalServerError()
 	}
 
-	if !aliasFound {
+	if !queryRes.Found {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
-			JSON: spec.NotFound("The alias does not exist."),
+			JSON: jsonerror.NotFound("The alias does not exist."),
 		}
 	}
 
-	if !aliasRemoved {
+	if !queryRes.Removed {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("You do not have permission to remove this alias."),
+			JSON: jsonerror.Forbidden("You do not have permission to remove this alias."),
 		}
 	}
 
@@ -350,15 +247,12 @@ func GetVisibility(
 	}, &res)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryPublishedRooms failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	var v roomVisibility
 	if len(res.RoomIDs) == 1 {
-		v.Visibility = spec.Public
+		v.Visibility = gomatrixserverlib.Public
 	} else {
 		v.Visibility = "private"
 	}
@@ -375,30 +269,7 @@ func SetVisibility(
 	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, dev *userapi.Device,
 	roomID string,
 ) util.JSONResponse {
-	deviceUserID, err := spec.NewUserID(dev.UserID, true)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("userID for this device is invalid"),
-		}
-	}
-	validRoomID, err := spec.NewRoomID(roomID)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("roomID is invalid")
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("RoomID is invalid"),
-		}
-	}
-	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
-	if err != nil || senderID == nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.Unknown("failed to find senderID for this user"),
-		}
-	}
-
-	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
+	resErr := checkMemberInRoom(req.Context(), rsAPI, dev.UserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}
@@ -406,26 +277,23 @@ func SetVisibility(
 	queryEventsReq := roomserverAPI.QueryLatestEventsAndStateRequest{
 		RoomID: roomID,
 		StateToFetch: []gomatrixserverlib.StateKeyTuple{{
-			EventType: spec.MRoomPowerLevels,
+			EventType: gomatrixserverlib.MRoomPowerLevels,
 			StateKey:  "",
 		}},
 	}
 	var queryEventsRes roomserverAPI.QueryLatestEventsAndStateResponse
-	err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
+	err := rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
 	if err != nil || len(queryEventsRes.StateEvents) == 0 {
 		util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 
 	// NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event
-	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].PDU)
-	if power.UserLevel(*senderID) < power.EventLevel(spec.MRoomCanonicalAlias, true) {
+	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
+	if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomCanonicalAlias, true) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
+			JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),
 		}
 	}
 
@@ -434,15 +302,16 @@ func SetVisibility(
 		return *reqErr
 	}
 
-	if err = rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
+	var publishRes roomserverAPI.PerformPublishResponse
+	if err := rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
 		RoomID:     roomID,
 		Visibility: v.Visibility,
-	}); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("failed to publish room")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+	}, &publishRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
+	if publishRes.Error != nil {
+		util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
+		return publishRes.Error.JSONResponse()
 	}
 
 	return util.JSONResponse{
@@ -458,7 +327,7 @@ func SetVisibilityAS(
 	if dev.AccountType != userapi.AccountTypeAppService {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("Only appservice may use this endpoint"),
+			JSON: jsonerror.Forbidden("Only appservice may use this endpoint"),
 		}
 	}
 	var v roomVisibility
@@ -471,17 +340,18 @@ func SetVisibilityAS(
 			return *reqErr
 		}
 	}
+	var publishRes roomserverAPI.PerformPublishResponse
 	if err := rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
 		RoomID:       roomID,
 		Visibility:   v.Visibility,
 		NetworkID:    networkID,
 		AppserviceID: dev.AppserviceID,
-	}); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("failed to publish room")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+	}, &publishRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
+	if publishRes.Error != nil {
+		util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
+		return publishRes.Error.JSONResponse()
 	}
 
 	return util.JSONResponse{

clientapi/routing/directory_public.go

@@ -23,19 +23,19 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/matrix-org/gomatrixserverlib/fclient"
-	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 var (
 	cacheMu          sync.Mutex
-	publicRoomsCache []fclient.PublicRoom
+	publicRoomsCache []gomatrixserverlib.PublicRoom
 )
 
 type PublicRoomReq struct {
@@ -56,7 +56,7 @@ type filter struct {
 func GetPostPublicRooms(
 	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
-	federation fclient.FederationClient,
+	federation *gomatrixserverlib.FederationClient,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var request PublicRoomReq
@@ -67,11 +67,11 @@ func GetPostPublicRooms(
 	if request.IncludeAllNetworks && request.NetworkID != "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam("include_all_networks and third_party_instance_id can not be used together"),
+			JSON: jsonerror.InvalidParam("include_all_networks and third_party_instance_id can not be used together"),
 		}
 	}
 
-	serverName := spec.ServerName(request.Server)
+	serverName := gomatrixserverlib.ServerName(request.Server)
 	if serverName != "" && !cfg.Matrix.IsLocalServerName(serverName) {
 		res, err := federation.GetPublicRoomsFiltered(
 			req.Context(), cfg.Matrix.ServerName, serverName,
@@ -81,10 +81,7 @@ func GetPostPublicRooms(
 		)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("failed to get public rooms")
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{},
-			}
+			return jsonerror.InternalServerError()
 		}
 		return util.JSONResponse{
 			Code: http.StatusOK,
@@ -95,10 +92,7 @@ func GetPostPublicRooms(
 	response, err := publicRooms(req.Context(), request, rsAPI, extRoomsProvider)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Errorf("failed to work out public rooms")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
+		return jsonerror.InternalServerError()
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -108,10 +102,10 @@ func GetPostPublicRooms(
 
 func publicRooms(
 	ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
-) (*fclient.RespPublicRooms, error) {
+) (*gomatrixserverlib.RespPublicRooms, error) {
 
-	response := fclient.RespPublicRooms{
-		Chunk: []fclient.PublicRoom{},
+	response := gomatrixserverlib.RespPublicRooms{
+		Chunk: []gomatrixserverlib.PublicRoom{},
 	}
 	var limit int64
 	var offset int64
@@ -128,7 +122,7 @@ func publicRooms(
 	}
 	err = nil
 
-	var rooms []fclient.PublicRoom
+	var rooms []gomatrixserverlib.PublicRoom
 	if request.Since == "" {
 		rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider, request)
 	} else {
@@ -152,14 +146,14 @@ func publicRooms(
 	return &response, err
 }
 
-func filterRooms(rooms []fclient.PublicRoom, searchTerm string) []fclient.PublicRoom {
+func filterRooms(rooms []gomatrixserverlib.PublicRoom, searchTerm string) []gomatrixserverlib.PublicRoom {
 	if searchTerm == "" {
 		return rooms
 	}
 
 	normalizedTerm := strings.ToLower(searchTerm)
 
-	result := make([]fclient.PublicRoom, 0)
+	result := make([]gomatrixserverlib.PublicRoom, 0)
 	for _, room := range rooms {
 		if strings.Contains(strings.ToLower(room.Name), normalizedTerm) ||
 			strings.Contains(strings.ToLower(room.Topic), normalizedTerm) ||
@@ -178,7 +172,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 	if httpReq.Method != "GET" && httpReq.Method != "POST" {
 		return &util.JSONResponse{
 			Code: http.StatusMethodNotAllowed,
-			JSON: spec.NotFound("Bad method"),
+			JSON: jsonerror.NotFound("Bad method"),
 		}
 	}
 	if httpReq.Method == "GET" {
@@ -189,7 +183,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 			util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
 			return &util.JSONResponse{
 				Code: 400,
-				JSON: spec.BadJSON("limit param is not a number"),
+				JSON: jsonerror.BadJSON("limit param is not a number"),
 			}
 		}
 		request.Limit = int64(limit)
@@ -220,7 +214,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 //	 limit=3&since=6  => G     (prev='3', next='')
 //
 //	A value of '-1' for prev/next indicates no position.
-func sliceInto(slice []fclient.PublicRoom, since int64, limit int64) (subset []fclient.PublicRoom, prev, next int) {
+func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
 	prev = -1
 	next = -1
 
@@ -247,10 +241,10 @@ func sliceInto(slice []fclient.PublicRoom, since int64, limit int64) (subset []f
 func refreshPublicRoomCache(
 	ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
 	request PublicRoomReq,
-) []fclient.PublicRoom {
+) []gomatrixserverlib.PublicRoom {
 	cacheMu.Lock()
 	defer cacheMu.Unlock()
-	var extraRooms []fclient.PublicRoom
+	var extraRooms []gomatrixserverlib.PublicRoom
 	if extRoomsProvider != nil {
 		extraRooms = extRoomsProvider.Rooms()
 	}
@@ -275,7 +269,7 @@ func refreshPublicRoomCache(
 		util.GetLogger(ctx).WithError(err).Error("PopulatePublicRooms failed")
 		return publicRoomsCache
 	}
-	publicRoomsCache = []fclient.PublicRoom{}
+	publicRoomsCache = []gomatrixserverlib.PublicRoom{}
 	publicRoomsCache = append(publicRoomsCache, pubRooms...)
 	publicRoomsCache = append(publicRoomsCache, extraRooms...)
 	publicRoomsCache = dedupeAndShuffle(publicRoomsCache)
@@ -287,16 +281,16 @@ func refreshPublicRoomCache(
 	return publicRoomsCache
 }
 
-func getPublicRoomsFromCache() []fclient.PublicRoom {
+func getPublicRoomsFromCache() []gomatrixserverlib.PublicRoom {
 	cacheMu.Lock()
 	defer cacheMu.Unlock()
 	return publicRoomsCache
 }
 
-func dedupeAndShuffle(in []fclient.PublicRoom) []fclient.PublicRoom {
+func dedupeAndShuffle(in []gomatrixserverlib.PublicRoom) []gomatrixserverlib.PublicRoom {
 	// de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers
 	// are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit)
-	var publicRooms []fclient.PublicRoom
+	var publicRooms []gomatrixserverlib.PublicRoom
 	haveRoomIDs := make(map[string]bool)
 	rand.Shuffle(len(in), func(i, j int) {
 		in[i], in[j] = in[j], in[i]

clientapi/routing/directory_public_test.go

@@ -4,17 +4,17 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func pubRoom(name string) fclient.PublicRoom {
-	return fclient.PublicRoom{
+func pubRoom(name string) gomatrixserverlib.PublicRoom {
+	return gomatrixserverlib.PublicRoom{
 		Name: name,
 	}
 }
 
 func TestSliceInto(t *testing.T) {
-	slice := []fclient.PublicRoom{
+	slice := []gomatrixserverlib.PublicRoom{
 		pubRoom("a"), pubRoom("b"), pubRoom("c"), pubRoom("d"), pubRoom("e"), pubRoom("f"), pubRoom("g"),
 	}
 	limit := int64(3)
@@ -22,7 +22,7 @@ func TestSliceInto(t *testing.T) {
 		since      int64
 		wantPrev   int
 		wantNext   int
-		wantSubset []fclient.PublicRoom
+		wantSubset []gomatrixserverlib.PublicRoom
 	}{
 		{
 			since:      0,