Merge pull request #66 from globekeeper/release/upstream-v0.12.0

Release/upstream v0.12.0

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -17,7 +17,6 @@ see: https://www.matrix.org/security-disclosure-policy/
 ### Background information
 <!-- Please include versions of all software when known e.g database versions, docker versions, client versions -->
 - **Dendrite version or git SHA**:
-- **Monolith or Polylith?**:
 - **SQLite3 or Postgres?**:
 - **Running in Docker?**:
 - **`go version`**:

.github/codecov.yaml

@@ -0,0 +1,20 @@
+flag_management:
+  default_rules:
+    carryforward: true
+
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 0%
+        base: auto
+        flags:
+          - unittests
+    patch:
+      default:
+        target: 75%
+        threshold: 0%
+        base: auto
+        flags:
+          - unittests

.github/workflows/dendrite.yml

@@ -4,7 +4,15 @@ on:
   push:
     branches:
       - main
+    paths:
+      - '**.go'                 # only execute on changes to go files
+      - 'go.sum'                # or dependency updates
+      - '.github/workflows/**'  # or workflow changes
   pull_request:
+    paths:
+      - '**.go'
+      - 'go.sum'                # or dependency updates
+      - '.github/workflows/**'
   release:
     types: [published]
   workflow_dispatch:
@@ -25,7 +33,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: "stable"
           cache: true
 
       - name: Install Node
@@ -62,14 +70,14 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: "stable"
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
 
   # run go test with go 1.19
   test:
-    timeout-minutes: 5
+    timeout-minutes: 10
-    name: Unit tests (Go ${{ matrix.go }})
+    name: Unit tests
     runs-on: ubuntu-latest
     # Service containers to run with `container-job`
     services:
@@ -91,17 +99,21 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-    strategy:
-      fail-fast: false
-      matrix:
-        go: ["1.19"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: "stable"
-          cache: true
+      - uses: actions/cache@v3
+        # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-stable-unit-
       - name: Set up gotestfmt
         uses: gotesttools/gotestfmt-action@v2
         with:
@@ -122,7 +134,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.19"]
         goos: ["linux"]
         goarch: ["amd64"]
     steps:
@@ -130,15 +141,15 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: "stable"
       - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
+            key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
       - name: Install dependencies x86
         if: ${{ matrix.goarch == '386' }}
         run: sudo apt update && sudo apt-get install -y gcc-multilib
@@ -156,23 +167,22 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ["1.18", "1.19"]
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
       - uses: actions/checkout@v3
-      - name: Setup Go ${{ matrix.go }}
+      - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: "stable"
       - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
+            key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
       - env:
@@ -194,6 +204,63 @@ jobs:
         with:
           jobs: ${{ toJSON(needs) }}
 
+  # run go test with different go versions
+  integration:
+    timeout-minutes: 20
+    needs: initial-tests-done
+    name: Integration tests
+    runs-on: ubuntu-latest
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:13-alpine
+        # Provide the password for postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "stable"
+      - name: Set up gotestfmt
+        uses: gotesttools/gotestfmt-action@v2
+        with:
+          # Optional: pass GITHUB_TOKEN to avoid rate limiting.
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-stable-test-race-
+      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: unittests
+          fail_ci_if_error: false
+
   # run database upgrade tests
   upgrade_test:
     name: Upgrade tests
@@ -205,7 +272,7 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "stable"
           cache: true
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
@@ -223,7 +290,7 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "stable"
           cache: true
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
@@ -243,18 +310,14 @@ jobs:
           - label: PostgreSQL
             postgres: postgres
 
-          - label: PostgreSQL, full HTTP APIs
-            postgres: postgres
-            api: full-http
     container:
-      image: matrixdotorg/sytest-dendrite:latest
+      image: matrixdotorg/sytest-dendrite
       volumes:
         - ${{ github.workspace }}:/src
         - /root/.cache/go-build:/github/home/.cache/go-build
         - /root/.cache/go-mod:/gopath/pkg/mod
       env:
         POSTGRES: ${{ matrix.postgres && 1}}
-        API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
         CGO_ENABLED: ${{ matrix.cgo && 1 }}
     steps:
@@ -302,11 +365,6 @@ jobs:
           - label: PostgreSQL
             postgres: Postgres
             cgo: 0
-
-          - label: PostgreSQL, full HTTP APIs
-            postgres: Postgres
-            api: full-http
-            cgo: 0
     steps:
       # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
       # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
@@ -349,7 +407,7 @@ jobs:
             (wget -O - "https://github.com/globekeeper/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
           done
       # Build initial Dendrite image
-      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
+      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
         working-directory: dendrite
         env:
           DOCKER_BUILDKIT: 1
@@ -361,8 +419,8 @@ jobs:
         shell: bash
         name: Run Complement Tests
         env:
-          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
+          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
-          API: ${{ matrix.api && 1 }}
+          COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
         working-directory: complement
 
   integration-tests-done:
@@ -372,6 +430,7 @@ jobs:
         initial-tests-done,
         sytest,
         complement,
+        integration
       ]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped

.github/workflows/docker.yml

@@ -61,7 +61,6 @@ jobs:
           cache-to: type=gha,mode=max
           context: .
           build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -77,7 +76,6 @@ jobs:
           cache-to: type=gha,mode=max
           context: .
           build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -98,86 +96,6 @@ jobs:
         with:
           sarif_file: "trivy-results.sarif"
 
-  polylith:
-    name: Polylith image
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-      security-events: write # To upload Trivy sarif files
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
-        if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ env.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Login to GitHub Containers
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build main polylith image
-        if: github.ref_name == 'main'
-        id: docker_build_polylith
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: polylith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-
-      - name: Build release polylith image
-        if: github.event_name == 'release' # Only for GitHub releases
-        id: docker_build_polylith_release
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: polylith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
-
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
   demo-pinecone:
     name: Pinecone demo image
     runs-on: ubuntu-latest

.github/workflows/gh-pages.yml

@@ -0,0 +1,52 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Deploy GitHub Pages dependencies preinstalled
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["gh-pages"]
+    paths:
+      - 'docs/**' # only execute if we have docs changes
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow one concurrent deployment
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Pages
+        uses: actions/configure-pages@v2
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./docs
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v1
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v1

.github/workflows/helm.yml

@@ -0,0 +1,39 @@
+name: Release Charts
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'helm/**' # only execute if we have helm chart changes
+
+jobs:
+  release:
+    # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions
+    # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.10.0
+
+      - name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.4.1
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        with:
+          config: helm/cr.yaml
+          charts_dir: helm/

.github/workflows/k8s.yml

@@ -0,0 +1,90 @@
+name: k8s
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - 'helm/**' # only execute if we have helm chart changes
+  pull_request:
+    branches: ["main"]
+    paths:
+      - 'helm/**'
+
+jobs:
+  lint:
+    name: Lint Helm chart
+    runs-on: ubuntu-latest
+    outputs:
+      changed: ${{ steps.list-changed.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: azure/setup-helm@v3
+        with:
+          version: v3.10.0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          check-latest: true
+      - uses: helm/chart-testing-action@v2.3.1
+      - name: Get changed status
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config helm/ct.yaml --target-branch ${{ github.event.repository.default_branch }})
+          if [[ -n "$changed" ]]; then
+              echo "::set-output name=changed::true"
+          fi
+
+      - name: Run lint
+        run: ct lint --config helm/ct.yaml
+
+  # only bother to run if lint step reports a change to the helm chart
+  install:
+    needs:
+      - lint
+    if: ${{ needs.lint.outputs.changed == 'true' }}
+    name: Install Helm charts
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.checkoutCommit }}
+      - name: Install Kubernetes tools
+        uses: yokawasa/action-setup-kube-tools@v0.8.2
+        with:
+          setup-tools: |
+            helmv3
+          helm: "3.10.3"
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.3.1
+      - name: Create k3d cluster
+        uses: nolar/setup-k3d-k3s@v1
+        with:
+          version: v1.21
+      - name: Remove node taints
+        run: |
+          kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true
+      - name: Run chart-testing (install)
+        run: ct install --config helm/ct.yaml
+
+      # Install the chart using helm directly and test with create-account
+      - name: Install chart
+        run: |
+          helm install --values helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml dendrite helm/dendrite
+      - name: Wait for Postgres and Dendrite to be up
+        run: |
+          kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=postgresql || kubectl get pods -A
+          kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=dendrite || kubectl get pods -A
+          kubectl get pods -A
+          kubectl get services
+          kubectl get ingress
+      - name: Run create account
+        run: |
+          podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name)
+          kubectl exec "${podName}" -- /usr/bin/create-account -username alice -password somerandompassword

.github/workflows/schedules.yaml

@@ -10,107 +10,43 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # run go test with different go versions
-  test:
-    timeout-minutes: 20
-    name: Unit tests (Go ${{ matrix.go }})
-    runs-on: ubuntu-latest
-    # Service containers to run with `container-job`
-    services:
-      # Label used to access the service container
-      postgres:
-        # Docker Hub image
-        image: postgres:13-alpine
-        # Provide the password for postgres
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: dendrite
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    strategy:
-      fail-fast: false
-      matrix:
-        go: ["1.18", "1.19"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ matrix.go }}
-      - name: Set up gotestfmt
-        uses: gotesttools/gotestfmt-action@v2
-        with:
-          # Optional: pass GITHUB_TOKEN to avoid rate limiting.
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go${{ matrix.go }}-test-race-
-      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
-        env:
-          POSTGRES_HOST: localhost
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: dendrite
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-
-  # Dummy step to gate other tests on without repeating the whole list
-  initial-tests-done:
-    name: Initial tests passed
-    needs: [test]
-    runs-on: ubuntu-latest
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    steps:
-      - name: Check initial tests passed
-        uses: re-actors/alls-green@release/v1
-        with:
-          jobs: ${{ toJSON(needs) }}
-
   # run Sytest in different variations
   sytest:
     timeout-minutes: 60
-    needs: initial-tests-done
     name: "Sytest (${{ matrix.label }})"
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         include:
-          - label: SQLite
+          - label: SQLite native
 
-          - label: SQLite, full HTTP APIs
+          - label: SQLite Cgo
-            api: full-http
+            cgo: 1
 
           - label: PostgreSQL
             postgres: postgres
-
-          - label: PostgreSQL, full HTTP APIs
-            postgres: postgres
-            api: full-http
     container:
       image: matrixdotorg/sytest-dendrite:latest
       volumes:
         - ${{ github.workspace }}:/src
+        - /root/.cache/go-build:/github/home/.cache/go-build
+        - /root/.cache/go-mod:/gopath/pkg/mod
       env:
         POSTGRES: ${{ matrix.postgres && 1}}
-        API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
         RACE_DETECTION: 1
+        COVER: 1
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            /gopath/pkg/mod
+          key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-sytest-
       - name: Run Sytest
         run: /bootstrap.sh dendrite
         working-directory: /src
@@ -133,3 +69,192 @@ jobs:
           path: |
             /logs/results.tap
             /logs/**/*.log*
+
+  sytest-coverage:
+    timeout-minutes: 5
+    name: "Sytest Coverage"
+    runs-on: ubuntu-latest
+    needs: sytest # only run once Sytest is done
+    if: ${{ always() }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+          cache: true
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+      - name: Install gocovmerge
+        run: go install github.com/wadey/gocovmerge@latest
+      - name: Run gocovmerge
+        run: |
+          find -name 'integrationcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > sytest.cov
+          go tool cover -func=sytest.cov
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./sytest.cov
+          flags: sytest
+          fail_ci_if_error: true
+
+  # run Complement
+  complement:
+    name: "Complement (${{ matrix.label }})"
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: SQLite native
+            cgo: 0
+
+          - label: SQLite Cgo
+            cgo: 1
+
+          - label: PostgreSQL
+            postgres: Postgres
+            cgo: 0
+    steps:
+      # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
+      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+      - name: "Set Go Version"
+        run: |
+          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+          echo "~/go/bin" >> $GITHUB_PATH
+      - name: "Install Complement Dependencies"
+        # We don't need to install Go because it is included on the Ubuntu 20.04 image:
+        # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
+        run: |
+          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
+          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+      - name: Run actions/checkout@v3 for dendrite
+        uses: actions/checkout@v3
+        with:
+          path: dendrite
+
+      # Attempt to check out the same branch of Complement as the PR. If it
+      # doesn't exist, fallback to main.
+      - name: Checkout complement
+        shell: bash
+        run: |
+          mkdir -p complement
+          # Attempt to use the version of complement which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default complement branch ("master").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
+            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+          done
+      # Build initial Dendrite image
+      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
+        working-directory: dendrite
+        env:
+          DOCKER_BUILDKIT: 1
+
+      - name: Create post test script
+        run: |
+          cat <<EOF > /tmp/posttest.sh
+          #!/bin/bash
+          mkdir -p /tmp/Complement/logs/\$2/\$1/
+          docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
+          EOF
+          
+          chmod +x /tmp/posttest.sh
+      # Run Complement
+      - run: |
+          set -o pipefail &&
+          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+        shell: bash
+        name: Run Complement Tests
+        env:
+          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
+          COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
+          COMPLEMENT_DENDRITE_COVER: 1
+          COMPLEMENT_POST_TEST_SCRIPT: /tmp/posttest.sh
+        working-directory: complement
+
+      - name: Upload Complement logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
+          path: |
+            /tmp/Complement/**/complementcover.log
+
+  complement-coverage:
+    timeout-minutes: 5
+    name: "Complement Coverage"
+    runs-on: ubuntu-latest
+    needs: complement # only run once Complement is done
+    if: ${{ always() }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+          cache: true
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+      - name: Install gocovmerge
+        run: go install github.com/wadey/gocovmerge@latest
+      - name: Run gocovmerge
+        run: |
+          find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
+          go tool cover -func=complement.cov
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./complement.cov
+          flags: complement
+          fail_ci_if_error: true
+
+  element_web:
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tecolicom/actions-use-apt-tools@v1
+        with:
+          # Our test suite includes some screenshot tests with unusual diacritics, which are
+          # supposed to be covered by STIXGeneral.
+          tools: fonts-stix
+      - uses: actions/checkout@v2
+        with:
+          repository: matrix-org/matrix-react-sdk
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+      - name: Fetch layered build
+        run: scripts/ci/layered.sh
+      - name: Copy config
+        run: cp element.io/develop/config.json config.json
+        working-directory: ./element-web
+      - name: Build
+        env:
+          CI_PACKAGE: true
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        run: yarn build
+        working-directory: ./element-web
+      - name: Edit Test Config
+        run: |
+          sed -i '/HOMESERVER/c\        HOMESERVER: "dendrite",' cypress.config.ts
+      - name: "Run cypress tests"
+        uses: cypress-io/github-action@v4.1.1
+        with:
+          browser: chrome
+          start: npx serve -p 8080 ./element-web/webapp
+          wait-on: 'http://localhost:8080'
+        env:
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          TMPDIR: ${{ runner.temp }}

.gitignore

@@ -58,6 +58,7 @@ dendrite.yaml
 
 # Database files
 *.db
+*.db-journal
 
 # Log files
 *.log*

.vscode/launch.json

@@ -5,7 +5,7 @@
             "type": "go",
             "request": "launch",
             "mode": "auto",
-            "program": "${workspaceFolder}/cmd/dendrite-monolith-server",
+            "program": "${workspaceFolder}/cmd/dendrite",
             "args": [
                 "-really-enable-open-registration",
                 "-config",

CHANGES.md

@@ -1,5 +1,80 @@
 # Changelog
 
+## Dendrite 0.12.0 (2023-03-13)
+
+### Features
+
+- The userapi and keyserver have been merged (no actions needed regarding the database)
+- The internal NATS JetStream server is now using logrus for logging (contributed by [dvob](https://github.com/dvob))
+- The roomserver database has been refactored to have separate interfaces when working with rooms and events. Also includes increased usage of the cache to avoid database round trips. (database is unchanged)
+- The pinecone demo now shuts down more cleanly
+- The Helm chart now has the ability to deploy a Grafana chart as well (contributed by [genofire](https://github.com/genofire))
+- Support for listening on unix sockets has been added (contributed by [cyberb](https://github.com/cyberb))
+- The internal NATS server was updated to v2.9.15
+- Initial support for `runtime/trace` has been added, to further track down long-running tasks
+
+### Fixes
+
+- The `session_id` is now correctly set when using SQLite
+- An issue where device keys could be removed if a device ID is reused has been fixed
+- A possible DoS issue related to relations has been fixed (reported by [sleroq](https://github.com/sleroq))
+- When backfilling events, errors are now ignored if we still could fetch events
+
+### Other
+
+- **⚠️ DEPRECATION: Polylith/HTTP API mode has been removed**
+- The default endpoint to report usages stats to has been updated
+
+## Dendrite 0.11.1 (2023-02-10)
+
+**⚠️ DEPRECATION WARNING: This is the last release to have polylith and HTTP API mode. Future releases are monolith only.**
+
+### Features
+
+* Dendrite can now be compiled against Go 1.20
+* Initial store and forward support has been added
+* A landing page showing that Dendrite is running has been added (contributed by [LukasLJL](https://github.com/LukasLJL))
+
+### Fixes
+
+- `/sync` is now using significantly less database round trips when using Postgres, resulting in faster initial syncs, allowing larger accounts to login again
+- Many under the hood pinecone improvements
+- Publishing rooms is now possible again
+
+## Dendrite 0.11.0 (2023-01-20)
+
+The last three missing federation API Sytests have been fixed - bringing us to 100% server-server Synapse parity, with client-server parity at 93% 🎉
+
+### Features
+
+* Added `/_dendrite/admin/purgeRoom/{roomID}` to clean up the database
+* The default room version was updated to 10 (contributed by [FSG-Cat](https://github.com/FSG-Cat))
+
+### Fixes
+
+* An oversight in the `create-config` binary, which now correctly sets the media path if specified (contributed by [BieHDC](https://github.com/BieHDC))
+* The Helm chart now uses the `$.Chart.AppVersion` as the default image version to pull, with the possibility to override it (contributed by [genofire](https://github.com/genofire))
+
+## Dendrite 0.10.9 (2023-01-17)
+
+### Features
+
+* Stale device lists are now cleaned up on startup, removing entries for users the server doesn't share a room with anymore
+* Dendrite now has its own Helm chart
+* Guest access is now handled correctly (disallow joins, kick guests on revocation of guest access, as well as over federation)
+
+### Fixes
+
+* Push rules have seen several tweaks and fixes, which should, for example, fix notifications for `m.read_receipts`
+* Outgoing presence will now correctly be sent to newly joined hosts
+* Fixes the `/_dendrite/admin/resetPassword/{userID}` admin endpoint to use the correct variable
+* Federated backfilling for medium/large rooms has been fixed
+* `/login` causing wrong device list updates has been resolved
+* `/sync` should now return the correct room summary heroes
+* The default config options for `recaptcha_sitekey_class` and `recaptcha_form_field` are now set correctly 
+* `/messages` now omits empty `state` to be more spec compliant (contributed by [handlerug](https://github.com/handlerug))
+* `/sync` has been optimised to only query state events for history visibility if they are really needed
+
 ## Dendrite 0.10.8 (2022-11-29)
 
 ### Features

Dockerfile

@@ -1,26 +1,31 @@
-FROM docker.io/golang:1.19-alpine AS base
+#
-
+# base installs required dependencies and runs go mod download to cache dependencies
-RUN apk --update --no-cache add bash build-base
+#
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base
+RUN apk --update --no-cache add bash build-base curl
 
 WORKDIR /build
 COPY . /build
 
-RUN mkdir -p bin
+#
-RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
+# Builds the Dendrite image containing all required binaries
-RUN go build -trimpath -o bin/ ./cmd/create-account
+#
-RUN go build -trimpath -o bin/ ./cmd/generate-keys
-
 FROM alpine:latest
-LABEL org.opencontainers.image.title="Dendrite (Monolith)"
+RUN apk --update --no-cache add curl
+LABEL org.opencontainers.image.title="Dendrite"
 LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
 LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
 LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
 LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
 
-COPY --from=base /build/bin/* /usr/bin/
+COPY --from=build /out/create-account /usr/bin/create-account
+COPY --from=build /out/generate-config /usr/bin/generate-config
+COPY --from=build /out/generate-keys /usr/bin/generate-keys
+COPY --from=build /out/dendrite /usr/bin/dendrite
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
 
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
+ENTRYPOINT ["/usr/bin/dendrite"]
+EXPOSE 8008 8448

README.md

@@ -24,7 +24,6 @@ This does not mean:
 - Dendrite is ready for massive homeserver deployments. There is no sharding of microservices (although it is possible to run them on separate machines) and there is no high-availability/clustering support.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
-In the future, we will be able to scale up to gigantic servers (equivalent to `matrix.org`) via polylith mode.
 
 If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or join us in:
 
@@ -72,10 +71,10 @@ $ ./bin/generate-keys --tls-cert server.crt --tls-key server.key
 
 # Copy and modify the config file - you'll need to set a server name and paths to the keys
 # at the very least, along with setting up the database connection strings.
-$ cp dendrite-sample.monolith.yaml dendrite.yaml
+$ cp dendrite-sample.yaml dendrite.yaml
 
 # Build and run the server:
-$ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
+$ ./bin/dendrite --tls-cert server.crt --tls-key server.key --config dendrite.yaml
 
 # Create an user account (add -admin for an admin user).
 # Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
@@ -88,7 +87,7 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
+updates with CI. As of January 2023, we have 100% server-server parity with Synapse, and the client-server parity is at 93% , though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs).
 

appservice/appservice.go

@@ -21,30 +21,24 @@ import (
 	"sync"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/appservice/consumers"
-	"github.com/matrix-org/dendrite/appservice/inthttp"
 	"github.com/matrix-org/dendrite/appservice/query"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// AddInternalRoutes registers HTTP handlers for internal API calls
-func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI) {
-	inthttp.AddRoutes(queryAPI, router)
-}
-
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
 	base *base.BaseDendrite,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.AppserviceUserAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceInternalAPI {
 	client := &http.Client{

appservice/appservice_test.go

@@ -0,0 +1,203 @@
+package appservice_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/userapi"
+
+	"github.com/matrix-org/dendrite/test/testrig"
+)
+
+func TestAppserviceInternalAPI(t *testing.T) {
+
+	// Set expected results
+	existingProtocol := "irc"
+	wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}}
+	wantProtocolResult := map[string]api.ASProtocolResponse{
+		existingProtocol: wantProtocolResponse,
+	}
+
+	// create a dummy AS url, handling some cases
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.Contains(r.URL.Path, "location"):
+			// Check if we've got an existing protocol, if so, return a proper response.
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "user"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "protocol"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode(nil); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		default:
+			t.Logf("hit location: %s", r.URL.Path)
+		}
+	}))
+
+	// The test cases to run
+	runCases := func(t *testing.T, testAPI api.AppServiceInternalAPI) {
+		t.Run("UserIDExists", func(t *testing.T) {
+			testUserIDExists(t, testAPI, "@as-testing:test", true)
+			testUserIDExists(t, testAPI, "@as1-testing:test", false)
+		})
+
+		t.Run("AliasExists", func(t *testing.T) {
+			testAliasExists(t, testAPI, "@asroom-testing:test", true)
+			testAliasExists(t, testAPI, "@asroom1-testing:test", false)
+		})
+
+		t.Run("Locations", func(t *testing.T) {
+			testLocations(t, testAPI, existingProtocol, wantLocationResponse)
+			testLocations(t, testAPI, "abc", nil)
+		})
+
+		t.Run("User", func(t *testing.T) {
+			testUser(t, testAPI, existingProtocol, wantUserResponse)
+			testUser(t, testAPI, "abc", nil)
+		})
+
+		t.Run("Protocols", func(t *testing.T) {
+			testProtocol(t, testAPI, existingProtocol, wantProtocolResult)
+			testProtocol(t, testAPI, existingProtocol, wantProtocolResult) // tests the cache
+			testProtocol(t, testAPI, "", wantProtocolResult)               // tests getting all protocols
+			testProtocol(t, testAPI, "abc", nil)
+		})
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, closeBase := testrig.CreateBaseDendrite(t, dbType)
+		defer closeBase()
+
+		// Create a dummy application service
+		base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
+			{
+				ID:              "someID",
+				URL:             srv.URL,
+				ASToken:         "",
+				HSToken:         "",
+				SenderLocalpart: "senderLocalPart",
+				NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+					"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
+					"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
+				},
+				Protocols: []string{existingProtocol},
+			},
+		}
+
+		// Create required internal APIs
+		rsAPI := roomserver.NewInternalAPI(base)
+		usrAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
+
+		runCases(t, asAPI)
+	})
+}
+
+func testUserIDExists(t *testing.T, asAPI api.AppServiceInternalAPI, userID string, wantExists bool) {
+	ctx := context.Background()
+	userResp := &api.UserIDExistsResponse{}
+
+	if err := asAPI.UserIDExists(ctx, &api.UserIDExistsRequest{
+		UserID: userID,
+	}, userResp); err != nil {
+		t.Errorf("failed to get userID: %s", err)
+	}
+	if userResp.UserIDExists != wantExists {
+		t.Errorf("unexpected result for UserIDExists(%s): %v, expected %v", userID, userResp.UserIDExists, wantExists)
+	}
+}
+
+func testAliasExists(t *testing.T, asAPI api.AppServiceInternalAPI, alias string, wantExists bool) {
+	ctx := context.Background()
+	aliasResp := &api.RoomAliasExistsResponse{}
+
+	if err := asAPI.RoomAliasExists(ctx, &api.RoomAliasExistsRequest{
+		Alias: alias,
+	}, aliasResp); err != nil {
+		t.Errorf("failed to get alias: %s", err)
+	}
+	if aliasResp.AliasExists != wantExists {
+		t.Errorf("unexpected result for RoomAliasExists(%s): %v, expected %v", alias, aliasResp.AliasExists, wantExists)
+	}
+}
+
+func testLocations(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASLocationResponse) {
+	ctx := context.Background()
+	locationResp := &api.LocationResponse{}
+
+	if err := asAPI.Locations(ctx, &api.LocationRequest{
+		Protocol: proto,
+	}, locationResp); err != nil {
+		t.Errorf("failed to get locations: %s", err)
+	}
+	if !reflect.DeepEqual(locationResp.Locations, wantResult) {
+		t.Errorf("unexpected result for Locations(%s): %+v, expected %+v", proto, locationResp.Locations, wantResult)
+	}
+}
+
+func testUser(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASUserResponse) {
+	ctx := context.Background()
+	userResp := &api.UserResponse{}
+
+	if err := asAPI.User(ctx, &api.UserRequest{
+		Protocol: proto,
+	}, userResp); err != nil {
+		t.Errorf("failed to get user: %s", err)
+	}
+	if !reflect.DeepEqual(userResp.Users, wantResult) {
+		t.Errorf("unexpected result for User(%s): %+v, expected %+v", proto, userResp.Users, wantResult)
+	}
+}
+
+func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult map[string]api.ASProtocolResponse) {
+	ctx := context.Background()
+	protoResp := &api.ProtocolResponse{}
+
+	if err := asAPI.Protocols(ctx, &api.ProtocolRequest{
+		Protocol: proto,
+	}, protoResp); err != nil {
+		t.Errorf("failed to get Protocols: %s", err)
+	}
+	if !reflect.DeepEqual(protoResp.Protocols, wantResult) {
+		t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
+	}
+}

appservice/consumers/roomserver.go

@@ -122,6 +122,7 @@ func (s *OutputRoomEventConsumer) onMessage(
 			if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
 				newEventID := output.NewRoomEvent.Event.EventID()
 				eventsReq := &api.QueryEventsByIDRequest{
+					RoomID:   output.NewRoomEvent.Event.RoomID(),
 					EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
 				}
 				eventsRes := &api.QueryEventsByIDResponse{}

appservice/inthttp/client.go

@@ -1,84 +0,0 @@
-package inthttp
-
-import (
-	"context"
-	"errors"
-	"net/http"
-
-	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
-)
-
-// HTTP paths for the internal HTTP APIs
-const (
-	AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists"
-	AppServiceUserIDExistsPath    = "/appservice/UserIDExists"
-	AppServiceLocationsPath       = "/appservice/locations"
-	AppServiceUserPath            = "/appservice/users"
-	AppServiceProtocolsPath       = "/appservice/protocols"
-)
-
-// httpAppServiceQueryAPI contains the URL to an appservice query API and a
-// reference to a httpClient used to reach it
-type httpAppServiceQueryAPI struct {
-	appserviceURL string
-	httpClient    *http.Client
-}
-
-// NewAppserviceClient creates a AppServiceQueryAPI implemented by talking
-// to a HTTP POST API.
-// If httpClient is nil an error is returned
-func NewAppserviceClient(
-	appserviceURL string,
-	httpClient *http.Client,
-) (api.AppServiceInternalAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is <nil>")
-	}
-	return &httpAppServiceQueryAPI{appserviceURL, httpClient}, nil
-}
-
-// RoomAliasExists implements AppServiceQueryAPI
-func (h *httpAppServiceQueryAPI) RoomAliasExists(
-	ctx context.Context,
-	request *api.RoomAliasExistsRequest,
-	response *api.RoomAliasExistsResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"RoomAliasExists", h.appserviceURL+AppServiceRoomAliasExistsPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-// UserIDExists implements AppServiceQueryAPI
-func (h *httpAppServiceQueryAPI) UserIDExists(
-	ctx context.Context,
-	request *api.UserIDExistsRequest,
-	response *api.UserIDExistsResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"UserIDExists", h.appserviceURL+AppServiceUserIDExistsPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error {
-	return httputil.CallInternalRPCAPI(
-		"ASLocation", h.appserviceURL+AppServiceLocationsPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error {
-	return httputil.CallInternalRPCAPI(
-		"ASUser", h.appserviceURL+AppServiceUserPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error {
-	return httputil.CallInternalRPCAPI(
-		"ASProtocols", h.appserviceURL+AppServiceProtocolsPath,
-		h.httpClient, ctx, request, response,
-	)
-}

appservice/inthttp/server.go

@@ -1,36 +0,0 @@
-package inthttp
-
-import (
-	"github.com/gorilla/mux"
-
-	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
-)
-
-// AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
-func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
-	internalAPIMux.Handle(
-		AppServiceRoomAliasExistsPath,
-		httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", a.RoomAliasExists),
-	)
-
-	internalAPIMux.Handle(
-		AppServiceUserIDExistsPath,
-		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
-	)
-
-	internalAPIMux.Handle(
-		AppServiceProtocolsPath,
-		httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
-	)
-
-	internalAPIMux.Handle(
-		AppServiceLocationsPath,
-		httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
-	)
-
-	internalAPIMux.Handle(
-		AppServiceUserPath,
-		httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
-	)
-}

appservice/query/query.go

@@ -25,10 +25,10 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/opentracing/opentracing-go"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
@@ -50,8 +50,8 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 	request *api.RoomAliasExistsRequest,
 	response *api.RoomAliasExistsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ApplicationServiceRoomAlias")
+	trace, ctx := internal.StartRegion(ctx, "ApplicationServiceRoomAlias")
-	defer span.Finish()
+	defer trace.EndRegion()
 
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
@@ -117,8 +117,8 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	request *api.UserIDExistsRequest,
 	response *api.UserIDExistsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ApplicationServiceUserID")
+	trace, ctx := internal.StartRegion(ctx, "ApplicationServiceUserID")
-	defer span.Finish()
+	defer trace.EndRegion()
 
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {

are-we-synapse-yet.list

@@ -937,3 +937,11 @@ fst Room state after a rejected state event is the same as before
 fpb Federation publicRoom Name/topic keys are correct
 fed New federated private chats get full presence information (SYN-115) (10 subtests)
 dvk Rejects invalid device keys
+rmv User can create and send/receive messages in a room with version 10
+rmv local user can join room with version 10
+rmv User can invite local user to room with version 10
+rmv remote user can join room with version 10
+rmv User can invite remote user to room with version 10
+rmv Remote user can backfill in a room with version 10
+rmv Can reject invites over federation for rooms with version 10
+rmv Can receive redactions from regular users over federation in room version 10

build/dendritejs-pinecone/jsServer.go

@@ -1,104 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"bufio"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"syscall/js"
-)
-
-// JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it.
-type JSServer struct {
-	// The router which will service requests
-	Mux http.Handler
-}
-
-// OnRequestFromJS is the function that JS will invoke when there is a new request.
-// The JS function signature is:
-//
-//	function(reqString: string): Promise<{result: string, error: string}>
-//
-// Usage is like:
-//
-//	const res = await global._go_js_server.fetch(reqString);
-//	if (res.error) {
-//	  // handle error: this is a 'network' error, not a non-2xx error.
-//	}
-//	const rawHttpResponse = res.result;
-func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
-	// we HAVE to spawn a new goroutine and return immediately or else Go will deadlock
-	// if this request blocks at all e.g for /sync calls
-	httpStr := args[0].String()
-	promise := js.Global().Get("Promise").New(js.FuncOf(func(pthis js.Value, pargs []js.Value) interface{} {
-		// The initial callback code for new Promise() is also called on the critical path, which is why
-		// we need to put this in an immediately invoked goroutine.
-		go func() {
-			resolve := pargs[0]
-			resStr, err := h.handle(httpStr)
-			errStr := ""
-			if err != nil {
-				errStr = err.Error()
-			}
-			resolve.Invoke(map[string]interface{}{
-				"result": resStr,
-				"error":  errStr,
-			})
-		}()
-		return nil
-	}))
-	return promise
-}
-
-// handle invokes the http.ServeMux for this request and returns the raw HTTP response.
-func (h *JSServer) handle(httpStr string) (resStr string, err error) {
-	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(httpStr)))
-	if err != nil {
-		return
-	}
-	w := httptest.NewRecorder()
-
-	h.Mux.ServeHTTP(w, req)
-
-	res := w.Result()
-	var resBuffer strings.Builder
-	err = res.Write(&resBuffer)
-	return resBuffer.String(), err
-}
-
-// ListenAndServe registers a variable in JS-land with the given namespace. This variable is
-// a function which JS-land can call to 'send' HTTP requests. The function is attached to
-// a global object called "_go_js_server". See OnRequestFromJS for more info.
-func (h *JSServer) ListenAndServe(namespace string) {
-	globalName := "_go_js_server"
-	// register a hook in JS-land for it to invoke stuff
-	server := js.Global().Get(globalName)
-	if !server.Truthy() {
-		server = js.Global().Get("Object").New()
-		js.Global().Set(globalName, server)
-	}
-
-	server.Set(namespace, js.FuncOf(h.OnRequestFromJS))
-
-	fmt.Printf("Listening for requests from JS on function %s.%s\n", globalName, namespace)
-	// Block forever to mimic http.ListenAndServe
-	select {}
-}

build/dendritejs-pinecone/main.go

@@ -1,234 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"crypto/ed25519"
-	"encoding/hex"
-	"fmt"
-	"syscall/js"
-
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi"
-
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/sirupsen/logrus"
-
-	_ "github.com/matrix-org/go-sqlite3-js"
-
-	pineconeConnections "github.com/matrix-org/pinecone/connections"
-	pineconeRouter "github.com/matrix-org/pinecone/router"
-	pineconeSessions "github.com/matrix-org/pinecone/sessions"
-)
-
-var GitCommit string
-
-func init() {
-	fmt.Printf("[%s] dendrite.js starting...\n", GitCommit)
-}
-
-const publicPeer = "wss://pinecone.matrix.org/public"
-const keyNameEd25519 = "_go_ed25519_key"
-
-func readKeyFromLocalStorage() (key ed25519.PrivateKey, err error) {
-	localforage := js.Global().Get("localforage")
-	if !localforage.Truthy() {
-		err = fmt.Errorf("readKeyFromLocalStorage: no localforage")
-		return
-	}
-	// https://localforage.github.io/localForage/
-	item, ok := await(localforage.Call("getItem", keyNameEd25519))
-	if !ok || !item.Truthy() {
-		err = fmt.Errorf("readKeyFromLocalStorage: no key in localforage")
-		return
-	}
-	fmt.Println("Found key in localforage")
-	// extract []byte and make an ed25519 key
-	seed := make([]byte, 32, 32)
-	js.CopyBytesToGo(seed, item)
-
-	return ed25519.NewKeyFromSeed(seed), nil
-}
-
-func writeKeyToLocalStorage(key ed25519.PrivateKey) error {
-	localforage := js.Global().Get("localforage")
-	if !localforage.Truthy() {
-		return fmt.Errorf("writeKeyToLocalStorage: no localforage")
-	}
-
-	// make a Uint8Array from the key's seed
-	seed := key.Seed()
-	jsSeed := js.Global().Get("Uint8Array").New(len(seed))
-	js.CopyBytesToJS(jsSeed, seed)
-	// write it
-	localforage.Call("setItem", keyNameEd25519, jsSeed)
-	return nil
-}
-
-// taken from https://go-review.googlesource.com/c/go/+/150917
-
-// await waits until the promise v has been resolved or rejected and returns the promise's result value.
-// The boolean value ok is true if the promise has been resolved, false if it has been rejected.
-// If v is not a promise, v itself is returned as the value and ok is true.
-func await(v js.Value) (result js.Value, ok bool) {
-	if v.Type() != js.TypeObject || v.Get("then").Type() != js.TypeFunction {
-		return v, true
-	}
-	done := make(chan struct{})
-	onResolve := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
-		result = args[0]
-		ok = true
-		close(done)
-		return nil
-	})
-	defer onResolve.Release()
-	onReject := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
-		result = args[0]
-		ok = false
-		close(done)
-		return nil
-	})
-	defer onReject.Release()
-	v.Call("then", onResolve, onReject)
-	<-done
-	return
-}
-
-func generateKey() ed25519.PrivateKey {
-	// attempt to look for a seed in JS-land and if it exists use it.
-	priv, err := readKeyFromLocalStorage()
-	if err == nil {
-		fmt.Println("Read key from localStorage")
-		return priv
-	}
-	// generate a new key
-	fmt.Println(err, " : Generating new ed25519 key")
-	_, priv, err = ed25519.GenerateKey(nil)
-	if err != nil {
-		logrus.Fatalf("Failed to generate ed25519 key: %s", err)
-	}
-	if err := writeKeyToLocalStorage(priv); err != nil {
-		fmt.Println("failed to write key to localStorage: ", err)
-		// non-fatal, we'll just have amnesia for a while
-	}
-	return priv
-}
-
-func main() {
-	startup()
-
-	// We want to block forever to let the fetch and libp2p handler serve the APIs
-	select {}
-}
-
-func startup() {
-	sk := generateKey()
-	pk := sk.Public().(ed25519.PublicKey)
-
-	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
-	pSessions := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
-	pManager := pineconeConnections.NewConnectionManager(pRouter)
-	pManager.AddPeer("wss://pinecone.matrix.org/public")
-
-	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
-	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
-	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
-	cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
-	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
-	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
-	cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db"
-	cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db"
-	cfg.Global.JetStream.StoragePath = "file:/idb/dendritejs/"
-	cfg.Global.TrustedIDServers = []string{}
-	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Global.PrivateKey = sk
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
-	cfg.ClientAPI.RegistrationDisabled = false
-	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-
-	if err := cfg.Derive(); err != nil {
-		logrus.Fatalf("Failed to derive values from config: %s", err)
-	}
-	base := base.NewBaseDendrite(cfg, "Monolith")
-	defer base.Close() // nolint: errcheck
-
-	federation := conn.CreateFederationClient(base, pSessions)
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
-
-	serverKeyAPI := &signing.YggdrasilKeys{}
-	keyRing := serverKeyAPI.KeyRing()
-
-	rsAPI := roomserver.NewInternalAPI(base)
-
-	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(userAPI)
-
-	asQuery := appservice.NewInternalAPI(
-		base, userAPI, rsAPI,
-	)
-	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, keyRing, true)
-	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
-
-	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    conn.CreateClient(base, pSessions),
-		FedClient: federation,
-		KeyRing:   keyRing,
-
-		AppserviceAPI: asQuery,
-		FederationAPI: fedSenderAPI,
-		RoomserverAPI: rsAPI,
-		UserAPI:       userAPI,
-		KeyAPI:        keyAPI,
-		//ServerKeyAPI:        serverKeyAPI,
-		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation),
-	}
-	monolith.AddAllPublicRoutes(base)
-
-	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-
-	p2pRouter := pSessions.Protocol("matrix").HTTP().Mux()
-	p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux)
-	p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux)
-
-	// Expose the matrix APIs via fetch - for local traffic
-	go func() {
-		logrus.Info("Listening for service-worker fetch traffic")
-		s := JSServer{
-			Mux: httpRouter,
-		}
-		s.ListenAndServe("fetch")
-	}()
-}

build/dendritejs-pinecone/main_noop.go

@@ -1,24 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build !wasm
-// +build !wasm
-
-package main
-
-import "fmt"
-
-func main() {
-	fmt.Println("dendritejs: no-op when not compiling for WebAssembly")
-}

build/docker/Dockerfile.demo-pinecone

@@ -17,6 +17,7 @@ RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
+RUN apk --update --no-cache add curl
 LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
 LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
 LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"

build/docker/README.md

@@ -5,7 +5,6 @@ These are Docker images for Dendrite!
 They can be found on Docker Hub:
 
 - [matrixdotorg/dendrite-monolith](https://hub.docker.com/r/matrixdotorg/dendrite-monolith) for monolith deployments
-- [matrixdotorg/dendrite-polylith](https://hub.docker.com/r/matrixdotorg/dendrite-polylith) for polylith deployments
 
 ## Dockerfiles
 
@@ -15,7 +14,6 @@ repository, run:
 
 ```
 docker build . --target monolith -t matrixdotorg/dendrite-monolith
-docker build . --target polylith -t matrixdotorg/dendrite-monolith
 docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone
 docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil
 ```
@@ -25,7 +23,6 @@ docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil
 There are two sample `docker-compose` files:
 
 - `docker-compose.monolith.yml` which runs a monolith Dendrite deployment
-- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment
 
 ## Configuration
 
@@ -51,9 +48,9 @@ docker run --rm --entrypoint="" \
 
 The key files will now exist in your current working directory, and can be mounted into place.
 
-## Starting Dendrite as a monolith deployment
+## Starting Dendrite
 
-Create your config based on the [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) sample configuration file.
+Create your config based on the [`dendrite-sample.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.yaml) sample configuration file.
 
 Then start the deployment:
 
@@ -61,16 +58,6 @@ Then start the deployment:
 docker-compose -f docker-compose.monolith.yml up
 ```
 
-## Starting Dendrite as a polylith deployment
-
-Create your config based on the [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) sample configuration file.
-
-Then start the deployment:
-
-```
-docker-compose -f docker-compose.polylith.yml up
-```
-
 ## Building the images
 
 The `build/docker/images-build.sh` script will build the base image, followed by

build/docker/docker-compose.polylith.yml

@@ -1,143 +0,0 @@
-version: "3.4"
-services:
-  postgres:
-    hostname: postgres
-    image: postgres:14
-    restart: always
-    volumes:
-      - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
-      # To persist your PostgreSQL databases outside of the Docker image, 
-      # to prevent data loss, modify the following ./path_to path:
-      - ./path_to/postgresql:/var/lib/postgresql/data
-    environment:
-      POSTGRES_PASSWORD: itsasecret
-      POSTGRES_USER: dendrite
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U dendrite"]
-      interval: 5s
-      timeout: 5s
-      retries: 5
-    networks:
-      - internal
-
-  jetstream:
-    hostname: jetstream
-    image: nats:latest
-    command: |
-      --jetstream
-      --store_dir /var/lib/nats
-      --cluster_name Dendrite
-    volumes:
-      # To persist your NATS JetStream streams outside of the Docker image,
-      # prevent data loss, modify the following ./path_to path:
-      - ./path_to/nats:/var/lib/nats
-    networks:
-      - internal
-
-  client_api:
-    hostname: client_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: clientapi
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-
-  media_api:
-    hostname: media_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: mediaapi
-    volumes:
-      - ./config:/etc/dendrite
-      - ./media:/var/dendrite/media
-    networks:
-      - internal
-    restart: unless-stopped
-
-  sync_api:
-    hostname: sync_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: syncapi
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-
-  room_server:
-    hostname: room_server
-    image: matrixdotorg/dendrite-polylith:latest
-    command: roomserver
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-
-  federation_api:
-    hostname: federation_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: federationapi
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-      
-  key_server:
-    hostname: key_server
-    image: matrixdotorg/dendrite-polylith:latest
-    command: keyserver
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-      
-  user_api:
-    hostname: user_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: userapi
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-
-  appservice_api:
-    hostname: appservice_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: appservice
-    volumes:
-      - ./config:/etc/dendrite
-    networks:
-      - internal
-    depends_on:
-      - jetstream
-      - postgres
-      - room_server
-      - user_api
-    restart: unless-stopped
-
-networks:
-  internal:
-    attachable: true

build/docker/images-build.sh

@@ -7,6 +7,5 @@ TAG=${1:-latest}
 echo "Building tag '${TAG}'"
 
 docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG}
-docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG}
 docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG}
 docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil:${TAG}

build/docker/images-pull.sh

@@ -5,4 +5,3 @@ TAG=${1:-latest}
 echo "Pulling tag '${TAG}'"
 
 docker pull matrixdotorg/dendrite-monolith:${TAG}
-docker pull matrixdotorg/dendrite-polylith:${TAG}

build/docker/images-push.sh

@@ -5,4 +5,3 @@ TAG=${1:-latest}
 echo "Pushing tag '${TAG}'"
 
 docker push matrixdotorg/dendrite-monolith:${TAG}
-docker push matrixdotorg/dendrite-polylith:${TAG}

build/gobind-pinecone/build.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-
-TARGET=""
-
-while getopts "ai" option
-do
-    case "$option"
-    in
-    a) gomobile bind -v -target android -trimpath -ldflags="-s -w" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
-    i) gomobile bind -v -target ios -trimpath -ldflags="" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
-    *) echo "No target specified, specify -a or -i"; exit 1 ;;
-    esac
-done

build/gobind-pinecone/monolith.go

@@ -1,519 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gobind
-
-import (
-	"context"
-	"crypto/ed25519"
-	"crypto/rand"
-	"crypto/tls"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"time"
-
-	"go.uber.org/atomic"
-
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/clientapi/userutil"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/dendrite/userapi"
-	userapiAPI "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
-	"golang.org/x/net/http2"
-	"golang.org/x/net/http2/h2c"
-
-	pineconeConnections "github.com/matrix-org/pinecone/connections"
-	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
-	pineconeRouter "github.com/matrix-org/pinecone/router"
-	pineconeEvents "github.com/matrix-org/pinecone/router/events"
-	pineconeSessions "github.com/matrix-org/pinecone/sessions"
-	"github.com/matrix-org/pinecone/types"
-
-	_ "golang.org/x/mobile/bind"
-)
-
-const (
-	PeerTypeRemote    = pineconeRouter.PeerTypeRemote
-	PeerTypeMulticast = pineconeRouter.PeerTypeMulticast
-	PeerTypeBluetooth = pineconeRouter.PeerTypeBluetooth
-	PeerTypeBonjour   = pineconeRouter.PeerTypeBonjour
-)
-
-type DendriteMonolith struct {
-	logger            logrus.Logger
-	PineconeRouter    *pineconeRouter.Router
-	PineconeMulticast *pineconeMulticast.Multicast
-	PineconeQUIC      *pineconeSessions.Sessions
-	PineconeManager   *pineconeConnections.ConnectionManager
-	StorageDirectory  string
-	CacheDirectory    string
-	listener          net.Listener
-	httpServer        *http.Server
-	processContext    *process.ProcessContext
-	userAPI           userapiAPI.UserInternalAPI
-}
-
-func (m *DendriteMonolith) PublicKey() string {
-	return m.PineconeRouter.PublicKey().String()
-}
-
-func (m *DendriteMonolith) BaseURL() string {
-	return fmt.Sprintf("http://%s", m.listener.Addr().String())
-}
-
-func (m *DendriteMonolith) PeerCount(peertype int) int {
-	return m.PineconeRouter.PeerCount(peertype)
-}
-
-func (m *DendriteMonolith) SessionCount() int {
-	return len(m.PineconeQUIC.Protocol("matrix").Sessions())
-}
-
-type InterfaceInfo struct {
-	Name         string
-	Index        int
-	Mtu          int
-	Up           bool
-	Broadcast    bool
-	Loopback     bool
-	PointToPoint bool
-	Multicast    bool
-	Addrs        string
-}
-
-type InterfaceRetriever interface {
-	CacheCurrentInterfaces() int
-	GetCachedInterface(index int) *InterfaceInfo
-}
-
-func (m *DendriteMonolith) RegisterNetworkCallback(intfCallback InterfaceRetriever) {
-	callback := func() []pineconeMulticast.InterfaceInfo {
-		count := intfCallback.CacheCurrentInterfaces()
-		intfs := []pineconeMulticast.InterfaceInfo{}
-		for i := 0; i < count; i++ {
-			iface := intfCallback.GetCachedInterface(i)
-			if iface != nil {
-				intfs = append(intfs, pineconeMulticast.InterfaceInfo{
-					Name:         iface.Name,
-					Index:        iface.Index,
-					Mtu:          iface.Mtu,
-					Up:           iface.Up,
-					Broadcast:    iface.Broadcast,
-					Loopback:     iface.Loopback,
-					PointToPoint: iface.PointToPoint,
-					Multicast:    iface.Multicast,
-					Addrs:        iface.Addrs,
-				})
-			}
-		}
-		return intfs
-	}
-	m.PineconeMulticast.RegisterNetworkCallback(callback)
-}
-
-func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
-	if enabled {
-		m.PineconeMulticast.Start()
-	} else {
-		m.PineconeMulticast.Stop()
-		m.DisconnectType(int(pineconeRouter.PeerTypeMulticast))
-	}
-}
-
-func (m *DendriteMonolith) SetStaticPeer(uri string) {
-	m.PineconeManager.RemovePeers()
-	for _, uri := range strings.Split(uri, ",") {
-		m.PineconeManager.AddPeer(strings.TrimSpace(uri))
-	}
-}
-
-func (m *DendriteMonolith) DisconnectType(peertype int) {
-	for _, p := range m.PineconeRouter.Peers() {
-		if int(peertype) == p.PeerType {
-			m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
-		}
-	}
-}
-
-func (m *DendriteMonolith) DisconnectZone(zone string) {
-	for _, p := range m.PineconeRouter.Peers() {
-		if zone == p.Zone {
-			m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
-		}
-	}
-}
-
-func (m *DendriteMonolith) DisconnectPort(port int) {
-	m.PineconeRouter.Disconnect(types.SwitchPortID(port), nil)
-}
-
-func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) {
-	l, r := net.Pipe()
-	conduit := &Conduit{conn: r, port: 0}
-	go func() {
-		conduit.portMutex.Lock()
-		defer conduit.portMutex.Unlock()
-
-		logrus.Errorf("Attempting authenticated connect")
-		var err error
-		if conduit.port, err = m.PineconeRouter.Connect(
-			l,
-			pineconeRouter.ConnectionZone(zone),
-			pineconeRouter.ConnectionPeerType(peertype),
-		); err != nil {
-			logrus.Errorf("Authenticated connect failed: %s", err)
-			_ = l.Close()
-			_ = r.Close()
-			_ = conduit.Close()
-			return
-		}
-		logrus.Infof("Authenticated connect succeeded (port %d)", conduit.port)
-	}()
-	return conduit, nil
-}
-
-func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, error) {
-	pubkey := m.PineconeRouter.PublicKey()
-	userID := userutil.MakeUserID(
-		localpart,
-		gomatrixserverlib.ServerName(hex.EncodeToString(pubkey[:])),
-	)
-	userReq := &userapiAPI.PerformAccountCreationRequest{
-		AccountType: userapiAPI.AccountTypeUser,
-		Localpart:   localpart,
-		Password:    password,
-	}
-	userRes := &userapiAPI.PerformAccountCreationResponse{}
-	if err := m.userAPI.PerformAccountCreation(context.Background(), userReq, userRes); err != nil {
-		return userID, fmt.Errorf("userAPI.PerformAccountCreation: %w", err)
-	}
-	return userID, nil
-}
-
-func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, error) {
-	accessTokenBytes := make([]byte, 16)
-	n, err := rand.Read(accessTokenBytes)
-	if err != nil {
-		return "", fmt.Errorf("rand.Read: %w", err)
-	}
-	loginReq := &userapiAPI.PerformDeviceCreationRequest{
-		Localpart:   localpart,
-		DeviceID:    &deviceID,
-		AccessToken: hex.EncodeToString(accessTokenBytes[:n]),
-	}
-	loginRes := &userapiAPI.PerformDeviceCreationResponse{}
-	if err := m.userAPI.PerformDeviceCreation(context.Background(), loginReq, loginRes); err != nil {
-		return "", fmt.Errorf("userAPI.PerformDeviceCreation: %w", err)
-	}
-	if !loginRes.DeviceCreated {
-		return "", fmt.Errorf("device was not created")
-	}
-	return loginRes.Device.AccessToken, nil
-}
-
-// nolint:gocyclo
-func (m *DendriteMonolith) Start() {
-	var sk ed25519.PrivateKey
-	var pk ed25519.PublicKey
-
-	keyfile := filepath.Join(m.StorageDirectory, "p2p.pem")
-	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
-		oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
-		if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
-			if err = test.NewMatrixKey(keyfile); err != nil {
-				panic("failed to generate a new PEM key: " + err.Error())
-			}
-			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
-				panic("failed to load PEM key: " + err.Error())
-			}
-			if len(sk) != ed25519.PrivateKeySize {
-				panic("the private key is not long enough")
-			}
-		} else {
-			if sk, err = os.ReadFile(oldkeyfile); err != nil {
-				panic("failed to read the old private key: " + err.Error())
-			}
-			if len(sk) != ed25519.PrivateKeySize {
-				panic("the private key is not long enough")
-			}
-			if err = test.SaveMatrixKey(keyfile, sk); err != nil {
-				panic("failed to convert the private key to PEM format: " + err.Error())
-			}
-		}
-	} else {
-		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
-			panic("failed to load PEM key: " + err.Error())
-		}
-		if len(sk) != ed25519.PrivateKeySize {
-			panic("the private key is not long enough")
-		}
-	}
-
-	pk = sk.Public().(ed25519.PublicKey)
-
-	var err error
-	m.listener, err = net.Listen("tcp", "localhost:65432")
-	if err != nil {
-		panic(err)
-	}
-
-	m.logger = logrus.Logger{
-		Out: BindLogger{},
-	}
-	m.logger.SetOutput(BindLogger{})
-	logrus.SetOutput(BindLogger{})
-
-	pineconeEventChannel := make(chan pineconeEvents.Event)
-	m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
-	m.PineconeRouter.EnableHopLimiting()
-	m.PineconeRouter.EnableWakeupBroadcasts()
-	m.PineconeRouter.Subscribe(pineconeEventChannel)
-
-	m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
-	m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
-	m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil)
-
-	prefix := hex.EncodeToString(pk)
-	cfg := &config.Dendrite{}
-	cfg.Defaults(config.DefaultOpts{
-		Generate:   true,
-		Monolithic: true,
-	})
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
-	cfg.Global.PrivateKey = sk
-	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Global.JetStream.InMemory = false
-	cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(m.CacheDirectory, prefix))
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.MediaAPI.BasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
-	cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
-	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
-	cfg.ClientAPI.RegistrationDisabled = false
-	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-	cfg.SyncAPI.Fulltext.Enabled = true
-	cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(m.CacheDirectory, "search"))
-	if err = cfg.Derive(); err != nil {
-		panic(err)
-	}
-
-	base := base.NewBaseDendrite(cfg, "Monolith")
-	defer base.Close() // nolint: errcheck
-
-	federation := conn.CreateFederationClient(base, m.PineconeQUIC)
-
-	serverKeyAPI := &signing.YggdrasilKeys{}
-	keyRing := serverKeyAPI.KeyRing()
-
-	rsAPI := roomserver.NewInternalAPI(base)
-
-	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, keyRing, true,
-	)
-
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
-	m.userAPI = userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(m.userAPI)
-
-	asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI)
-
-	// The underlying roomserver implementation needs to be able to call the fedsender.
-	// This is different to rsAPI which can be the http client which doesn't need this dependency
-	rsAPI.SetFederationAPI(fsAPI, keyRing)
-
-	userProvider := users.NewPineconeUserProvider(m.PineconeRouter, m.PineconeQUIC, m.userAPI, federation)
-	roomProvider := rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation)
-
-	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    conn.CreateClient(base, m.PineconeQUIC),
-		FedClient: federation,
-		KeyRing:   keyRing,
-
-		AppserviceAPI:            asAPI,
-		FederationAPI:            fsAPI,
-		RoomserverAPI:            rsAPI,
-		UserAPI:                  m.userAPI,
-		KeyAPI:                   keyAPI,
-		ExtPublicRoomsProvider:   roomProvider,
-		ExtUserDirectoryProvider: userProvider,
-	}
-	monolith.AddAllPublicRoutes(base)
-
-	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-	httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
-
-	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
-	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
-	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-
-	pHTTP := m.PineconeQUIC.Protocol("matrix").HTTP()
-	pHTTP.Mux().Handle(users.PublicURL, pMux)
-	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
-	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
-
-	// Build both ends of a HTTP multiplex.
-	h2s := &http2.Server{}
-	m.httpServer = &http.Server{
-		Addr:         ":0",
-		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
-		ReadTimeout:  10 * time.Second,
-		WriteTimeout: 10 * time.Second,
-		IdleTimeout:  30 * time.Second,
-		BaseContext: func(_ net.Listener) context.Context {
-			return context.Background()
-		},
-		Handler: h2c.NewHandler(pMux, h2s),
-	}
-
-	m.processContext = base.ProcessContext
-
-	go func() {
-		m.logger.Info("Listening on ", cfg.Global.ServerName)
-
-		switch m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")) {
-		case net.ErrClosed, http.ErrServerClosed:
-			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
-		default:
-			m.logger.Fatal(err)
-		}
-	}()
-	go func() {
-		logrus.Info("Listening on ", m.listener.Addr())
-
-		switch http.Serve(m.listener, httpRouter) {
-		case net.ErrClosed, http.ErrServerClosed:
-			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
-		default:
-			m.logger.Fatal(err)
-		}
-	}()
-
-	go func(ch <-chan pineconeEvents.Event) {
-		eLog := logrus.WithField("pinecone", "events")
-
-		for event := range ch {
-			switch e := event.(type) {
-			case pineconeEvents.PeerAdded:
-			case pineconeEvents.PeerRemoved:
-			case pineconeEvents.TreeParentUpdate:
-			case pineconeEvents.SnakeDescUpdate:
-			case pineconeEvents.TreeRootAnnUpdate:
-			case pineconeEvents.SnakeEntryAdded:
-			case pineconeEvents.SnakeEntryRemoved:
-			case pineconeEvents.BroadcastReceived:
-				eLog.Info("Broadcast received from: ", e.PeerID)
-
-				req := &api.PerformWakeupServersRequest{
-					ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
-				}
-				res := &api.PerformWakeupServersResponse{}
-				if err := fsAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
-					logrus.WithError(err).Error("Failed to wakeup destination", e.PeerID)
-				}
-			case pineconeEvents.BandwidthReport:
-			default:
-			}
-		}
-	}(pineconeEventChannel)
-}
-
-func (m *DendriteMonolith) Stop() {
-	m.processContext.ShutdownDendrite()
-	_ = m.listener.Close()
-	m.PineconeMulticast.Stop()
-	_ = m.PineconeQUIC.Close()
-	_ = m.PineconeRouter.Close()
-	m.processContext.WaitForComponentsToFinish()
-}
-
-const MaxFrameSize = types.MaxFrameSize
-
-type Conduit struct {
-	closed    atomic.Bool
-	conn      net.Conn
-	port      types.SwitchPortID
-	portMutex sync.Mutex
-}
-
-func (c *Conduit) Port() int {
-	c.portMutex.Lock()
-	defer c.portMutex.Unlock()
-	return int(c.port)
-}
-
-func (c *Conduit) Read(b []byte) (int, error) {
-	if c.closed.Load() {
-		return 0, io.EOF
-	}
-	return c.conn.Read(b)
-}
-
-func (c *Conduit) ReadCopy() ([]byte, error) {
-	if c.closed.Load() {
-		return nil, io.EOF
-	}
-	var buf [65535 * 2]byte
-	n, err := c.conn.Read(buf[:])
-	if err != nil {
-		return nil, err
-	}
-	return buf[:n], nil
-}
-
-func (c *Conduit) Write(b []byte) (int, error) {
-	if c.closed.Load() {
-		return 0, io.EOF
-	}
-	return c.conn.Write(b)
-}
-
-func (c *Conduit) Close() error {
-	if c.closed.Load() {
-		return io.ErrClosedPipe
-	}
-	c.closed.Store(true)
-	return c.conn.Close()
-}

build/gobind-pinecone/platform_ios.go

@@ -1,40 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build ios
-// +build ios
-
-package gobind
-
-/*
-#cgo CFLAGS: -x objective-c
-#cgo LDFLAGS: -framework Foundation
-#import <Foundation/Foundation.h>
-void Log(const char *text) {
-  NSString *nss = [NSString stringWithUTF8String:text];
-  NSLog(@"%@", nss);
-}
-*/
-import "C"
-import "unsafe"
-
-type BindLogger struct {
-}
-
-func (nsl BindLogger) Write(p []byte) (n int, err error) {
-	p = append(p, 0)
-	cstr := (*C.char)(unsafe.Pointer(&p[0]))
-	C.Log(cstr)
-	return len(p), nil
-}

build/gobind-pinecone/platform_other.go

@@ -1,27 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build !ios
-// +build !ios
-
-package gobind
-
-import "log"
-
-type BindLogger struct{}
-
-func (nsl BindLogger) Write(p []byte) (n int, err error) {
-	log.Println(string(p))
-	return len(p), nil
-}

build/gobind-yggdrasil/monolith.go

@@ -20,7 +20,6 @@ import (
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
@@ -128,7 +127,7 @@ func (m *DendriteMonolith) Start() {
 	cfg := &config.Dendrite{}
 	cfg.Defaults(config.DefaultOpts{
 		Generate:       true,
-		Monolithic: true,
+		SingleDatabase: true,
 	})
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
@@ -149,7 +148,8 @@ func (m *DendriteMonolith) Start() {
 		panic(err)
 	}
 
-	base := base.NewBaseDendrite(cfg, "Monolith")
+	base := base.NewBaseDendrite(cfg)
+	base.ConfigureAdminEndpoints()
 	m.processContext = base.ProcessContext
 	defer base.Close() // nolint: errcheck
 
@@ -164,9 +164,7 @@ func (m *DendriteMonolith) Start() {
 		base, federation, rsAPI, base.Caches, keyRing, true,
 	)
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
+	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
-	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(userAPI)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
@@ -185,7 +183,6 @@ func (m *DendriteMonolith) Start() {
 		FederationAPI: fsAPI,
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
-		KeyAPI:        keyAPI,
 		ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider(
 			ygg, fsAPI, federation,
 		),
@@ -193,9 +190,10 @@ func (m *DendriteMonolith) Start() {
 	monolith.AddAllPublicRoutes(base)
 
 	httpRouter := mux.NewRouter()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 
 	yggRouter := mux.NewRouter()
 	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)

build/scripts/Complement.Dockerfile

@@ -16,13 +16,16 @@ RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
-    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server
+    CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
+    cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
 
 ENV SERVER_NAME=localhost
 ENV API=0
+ENV COVER=0
 EXPOSE 8008 8448
 
 # At runtime, generate TLS cert based on the CA now mounted at /ca
@@ -30,4 +33,4 @@ EXPOSE 8008 8448
 CMD ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
+    exec /complement-cmd.sh

build/scripts/ComplementLocal.Dockerfile

@@ -12,18 +12,20 @@ FROM golang:1.18-stretch
 RUN apt-get update && apt-get install -y sqlite3
 
 ENV SERVER_NAME=localhost
+ENV COVER=0
 EXPOSE 8008 8448
 
 WORKDIR /runtime
 # This script compiles Dendrite for us.
 RUN echo '\
     #!/bin/bash -eux \n\
-    if test -f "/runtime/dendrite-monolith-server"; then \n\
+    if test -f "/runtime/dendrite" && test -f "/runtime/dendrite-cover"; then \n\
     echo "Skipping compilation; binaries exist" \n\
     exit 0 \n\
     fi \n\
     cd /dendrite \n\
-    go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
+    go build -v -o /runtime /dendrite/cmd/dendrite \n\
+    go test -c -cover -covermode=atomic -o /runtime/dendrite-cover -coverpkg "github.com/matrix-org/..." /dendrite/cmd/dendrite \n\
     ' > compile.sh && chmod +x compile.sh
 
 # This script runs Dendrite for us. Must be run in the /runtime directory.
@@ -33,7 +35,8 @@ RUN echo '\
     ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
-    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+    [ ${COVER} -eq 1 ] && exec ./dendrite-cover --test.coverprofile=integrationcover.log --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+    exec ./dendrite --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
     ' > run.sh && chmod +x run.sh
 
 

build/scripts/ComplementPostgres.Dockerfile

@@ -34,13 +34,16 @@ RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
-    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server
+    CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
+    cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
 
 ENV SERVER_NAME=localhost
 ENV API=0
+ENV COVER=0
 EXPOSE 8008 8448
 
 
@@ -51,4 +54,4 @@ CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NA
     # Bump max_open_conns up here in the global database config
     sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
+    exec /complement-cmd.sh

build/scripts/complement-cmd.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+# This script is intended to be used inside a docker container for Complement
+
+if [[ "${COVER}" -eq 1 ]]; then
+  echo "Running with coverage"
+  exec /dendrite/dendrite-cover \
+    --really-enable-open-registration \
+    --tls-cert server.crt \
+    --tls-key server.key \
+    --config dendrite.yaml \
+    --test.coverprofile=complementcover.log
+else
+  echo "Not running with coverage"
+  exec /dendrite/dendrite \
+    --really-enable-open-registration \
+    --tls-cert server.crt \
+    --tls-key server.key \
+    --config dendrite.yaml
+fi

clientapi/admin_test.go

@@ -0,0 +1,230 @@
+package clientapi
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/tidwall/gjson"
+
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+func TestAdminResetPassword(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser))
+	vhUser := &test.User{ID: "@vhuser:vh1"}
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+
+		// add a vhost
+		base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
+		})
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		// Needed for changing the password/login
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]string{
+			aliceAdmin: "",
+			bob:        "",
+			vhUser:     "",
+		}
+		for u := range accessTokens {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+			password := util.RandomString(8)
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    password,
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+
+			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+				"type": authtypes.LoginTypePassword,
+				"identifier": map[string]interface{}{
+					"type": "m.id.user",
+					"user": u.ID,
+				},
+				"password": password,
+			}))
+			rec := httptest.NewRecorder()
+			base.PublicClientAPIMux.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("failed to login: %s", rec.Body.String())
+			}
+			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
+		}
+
+		testCases := []struct {
+			name           string
+			requestingUser *test.User
+			userID         string
+			requestOpt     test.HTTPRequestOpt
+			wantOK         bool
+			withHeader     bool
+		}{
+			{name: "Missing auth", requestingUser: bob, wantOK: false, userID: bob.ID},
+			{name: "Bob is denied access", requestingUser: bob, wantOK: false, withHeader: true, userID: bob.ID},
+			{name: "Alice is allowed access", requestingUser: aliceAdmin, wantOK: true, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": util.RandomString(8),
+			})},
+			{name: "missing userID does not call function", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: ""}, // this 404s
+			{name: "rejects empty password", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": "",
+			})},
+			{name: "rejects unknown server name", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: "@doesnotexist:localhost", requestOpt: test.WithJSONBody(t, map[string]interface{}{})},
+			{name: "rejects unknown user", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: "@doesnotexist:test", requestOpt: test.WithJSONBody(t, map[string]interface{}{})},
+			{name: "allows changing password for different vhost", requestingUser: aliceAdmin, wantOK: true, withHeader: true, userID: vhUser.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": util.RandomString(8),
+			})},
+			{name: "rejects existing user, missing body", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID},
+			{name: "rejects invalid userID", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: "!notauserid:test", requestOpt: test.WithJSONBody(t, map[string]interface{}{})},
+			{name: "rejects invalid json", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, `{invalidJSON}`)},
+			{name: "rejects too weak password", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": util.RandomString(6),
+			})},
+			{name: "rejects too long password", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": util.RandomString(513),
+			})},
+		}
+
+		for _, tc := range testCases {
+			tc := tc // ensure we don't accidentally only test the last test case
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/resetPassword/"+tc.userID)
+				if tc.requestOpt != nil {
+					req = test.NewRequest(t, http.MethodPost, "/_dendrite/admin/resetPassword/"+tc.userID, tc.requestOpt)
+				}
+
+				if tc.withHeader {
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser])
+				}
+
+				rec := httptest.NewRecorder()
+				base.DendriteAdminMux.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}
+
+func TestPurgeRoom(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, aliceAdmin, test.RoomPreset(test.PresetTrustedPrivateChat))
+
+	// Invite Bob
+	room.CreateAndInsert(t, aliceAdmin, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "invite",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+
+		fedClient := base.CreateFederationClient()
+		rsAPI := roomserver.NewInternalAPI(base)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+
+		// this starts the JetStream consumers
+		syncapi.AddPublicRoutes(base, userAPI, rsAPI)
+		federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true)
+		rsAPI.SetFederationAPI(nil, nil)
+
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]string{
+			aliceAdmin: "",
+		}
+		for u := range accessTokens {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+			password := util.RandomString(8)
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    password,
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+
+			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+				"type": authtypes.LoginTypePassword,
+				"identifier": map[string]interface{}{
+					"type": "m.id.user",
+					"user": u.ID,
+				},
+				"password": password,
+			}))
+			rec := httptest.NewRecorder()
+			base.PublicClientAPIMux.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("failed to login: %s", rec.Body.String())
+			}
+			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
+		}
+
+		testCases := []struct {
+			name   string
+			roomID string
+			wantOK bool
+		}{
+			{name: "Can purge existing room", wantOK: true, roomID: room.ID},
+			{name: "Can not purge non-existent room", wantOK: false, roomID: "!doesnotexist:localhost"},
+			{name: "rejects invalid room ID", wantOK: false, roomID: "@doesnotexist:localhost"},
+		}
+
+		for _, tc := range testCases {
+			tc := tc // ensure we don't accidentally only test the last test case
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/purgeRoom/"+tc.roomID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin])
+
+				rec := httptest.NewRecorder()
+				base.DendriteAdminMux.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+
+	})
+}

clientapi/auth/login_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/ratelimit"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -48,7 +49,7 @@ func TestLoginFromJSONReader(t *testing.T) {
 				"password": "herpassword",
 				"device_id": "adevice"
             }`,
-			WantUsername: "alice",
+			WantUsername: "@alice:example.com",
 			WantDeviceID: "adevice",
 		},
 		{
@@ -179,7 +180,7 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *
 		return nil
 	}
 	res.Exists = true
-	res.Account = &uapi.Account{}
+	res.Account = &uapi.Account{UserID: userutil.MakeUserID(req.Localpart, req.ServerName)}
 	return nil
 }
 

clientapi/auth/password.go

@@ -96,7 +96,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			}
 		}
 	} else {
-		username = strings.ToLower(r.Username())
+		username = r.Username()
 	}
 	if username == "" {
 		return nil, &util.JSONResponse{
@@ -146,6 +146,8 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		}
 	}
 
+	// If we couldn't find the user by the lower cased localpart, try the provided
+	// localpart as is.
 	if !res.Exists {
 		err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
 			Localpart:         localpart,
@@ -170,6 +172,9 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			}
 		}
 	}
+	// Set the user, so login.Username() can do the right thing
+	r.Identifier.User = res.Account.UserID
+	r.User = res.Account.UserID
 	r.Login.User = username
 	return &r.Login, nil
 }

clientapi/clientapi.go

@@ -15,18 +15,18 @@
 package clientapi
 
 import (
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/transactions"
-	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
@@ -39,7 +39,6 @@ func AddPublicRoutes(
 	fsAPI federationAPI.ClientFederationAPI,
 	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.QuerySearchProfilesAPI,
-	keyAPI keyserverAPI.ClientKeyAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 ) {
 	cfg := &base.Cfg.ClientAPI
@@ -58,13 +57,10 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		base.PublicClientAPIMux,
+		base,
-		base.PublicWellKnownAPIMux,
-		base.SynapseAdminMux,
-		base.DendriteAdminMux,
 		cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
-		syncProducer, transactionsCache, fsAPI, keyAPI,
+		syncProducer, transactionsCache, fsAPI,
 		extRoomsProvider, mscCfg, natsClient,
 	)
 }

clientapi/routing/admin.go

@@ -1,12 +1,14 @@
 package routing
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
@@ -14,14 +16,13 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 )
 
-func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -54,7 +55,7 @@ func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi
 	}
 }
 
-func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -97,26 +98,77 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi
 	}
 }
 
-func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
+func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	serverName := cfg.Matrix.ServerName
+	roomID, ok := vars["roomID"]
-	localpart, ok := vars["localpart"]
 	if !ok {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting user localpart."),
+			JSON: jsonerror.MissingArgument("Expecting room ID."),
 		}
 	}
-	if l, s, err := cfg.Matrix.SplitLocalID('@', localpart); err == nil {
+	res := &roomserverAPI.PerformAdminPurgeRoomResponse{}
-		localpart, serverName = l, s
+	if err := rsAPI.PerformAdminPurgeRoom(
+		context.Background(),
+		&roomserverAPI.PerformAdminPurgeRoomRequest{
+			RoomID: roomID,
+		},
+		res,
+	); err != nil {
+		return util.ErrorResponse(err)
+	}
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: res,
+	}
+}
+
+func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.Device, userAPI api.ClientUserAPI) util.JSONResponse {
+	if req.Body == nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("Missing request body"),
+		}
+	}
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	var localpart string
+	userID := vars["userID"]
+	localpart, serverName, err := cfg.Matrix.SplitLocalID('@', userID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+	accAvailableResp := &api.QueryAccountAvailabilityResponse{}
+	if err = userAPI.QueryAccountAvailability(req.Context(), &api.QueryAccountAvailabilityRequest{
+		Localpart:  localpart,
+		ServerName: serverName,
+	}, accAvailableResp); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalAPIError(req.Context(), err),
+		}
+	}
+	if accAvailableResp.Available {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.Unknown("User does not exist"),
+		}
 	}
 	request := struct {
 		Password string `json:"password"`
 	}{}
-	if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+	if err = json.NewDecoder(req.Body).Decode(&request); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.Unknown("Failed to decode request body: " + err.Error()),
@@ -128,13 +180,18 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
 			JSON: jsonerror.MissingArgument("Expecting non-empty password."),
 		}
 	}
-	updateReq := &userapi.PerformPasswordUpdateRequest{
+
+	if err = internal.ValidatePassword(request.Password); err != nil {
+		return *internal.PasswordResponse(err)
+	}
+
+	updateReq := &api.PerformPasswordUpdateRequest{
 		Localpart:     localpart,
 		ServerName:    serverName,
 		Password:      request.Password,
 		LogoutDevices: true,
 	}
-	updateRes := &userapi.PerformPasswordUpdateResponse{}
+	updateRes := &api.PerformPasswordUpdateResponse{}
 	if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -151,7 +208,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
 	}
 }
 
-func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, natsClient *nats.Conn) util.JSONResponse {
+func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *api.Device, natsClient *nats.Conn) util.JSONResponse {
 	_, err := natsClient.RequestMsg(nats.NewMsg(cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex)), time.Second*10)
 	if err != nil {
 		logrus.WithError(err).Error("failed to publish nats message")
@@ -197,7 +254,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien
 	}
 }
 
-func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)

clientapi/routing/auth_fallback.go

@@ -15,11 +15,11 @@
 package routing
 
 import (
+	"fmt"
 	"html/template"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/util"
 )
@@ -101,14 +101,28 @@ func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]s
 func AuthFallback(
 	w http.ResponseWriter, req *http.Request, authType string,
 	cfg *config.ClientAPI,
-) *util.JSONResponse {
+) {
-	sessionID := req.URL.Query().Get("session")
+	// We currently only support "m.login.recaptcha", so fail early if that's not requested
+	if authType == authtypes.LoginTypeRecaptcha {
+		if !cfg.RecaptchaEnabled {
+			writeHTTPMessage(w, req,
+				"Recaptcha login is disabled on this Homeserver",
+				http.StatusBadRequest,
+			)
+			return
+		}
+	} else {
+		writeHTTPMessage(w, req, fmt.Sprintf("Unknown authtype %q", authType), http.StatusNotImplemented)
+		return
+	}
 
+	sessionID := req.URL.Query().Get("session")
 	if sessionID == "" {
-		return writeHTTPMessage(w, req,
+		writeHTTPMessage(w, req,
 			"Session ID not provided",
 			http.StatusBadRequest,
 		)
+		return
 	}
 
 	serveRecaptcha := func() {
@@ -130,70 +144,44 @@ func AuthFallback(
 
 	if req.Method == http.MethodGet {
 		// Handle Recaptcha
-		if authType == authtypes.LoginTypeRecaptcha {
-			if err := checkRecaptchaEnabled(cfg, w, req); err != nil {
-				return err
-			}
-
 		serveRecaptcha()
-			return nil
+		return
-		}
-		return &util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Unknown auth stage type"),
-		}
 	} else if req.Method == http.MethodPost {
 		// Handle Recaptcha
-		if authType == authtypes.LoginTypeRecaptcha {
-			if err := checkRecaptchaEnabled(cfg, w, req); err != nil {
-				return err
-			}
-
 		clientIP := req.RemoteAddr
 		err := req.ParseForm()
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("req.ParseForm failed")
-				res := jsonerror.InternalServerError()
+			w.WriteHeader(http.StatusBadRequest)
-				return &res
+			serveRecaptcha()
+			return
 		}
 
 		response := req.Form.Get(cfg.RecaptchaFormField)
-			if err := validateRecaptcha(cfg, response, clientIP); err != nil {
+		err = validateRecaptcha(cfg, response, clientIP)
-				util.GetLogger(req.Context()).Error(err)
+		switch err {
-				return err
+		case ErrMissingResponse:
+			w.WriteHeader(http.StatusBadRequest)
+			serveRecaptcha() // serve the initial page again, instead of nothing
+			return
+		case ErrInvalidCaptcha:
+			w.WriteHeader(http.StatusUnauthorized)
+			serveRecaptcha()
+			return
+		case nil:
+		default: // something else failed
+			util.GetLogger(req.Context()).WithError(err).Error("failed to validate recaptcha")
+			serveRecaptcha()
+			return
 		}
 
 		// Success. Add recaptcha as a completed login flow
 		sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
 
 		serveSuccess()
-			return nil
+		return
 	}
-
+	writeHTTPMessage(w, req, "Bad method", http.StatusMethodNotAllowed)
-		return &util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Unknown auth stage type"),
-		}
-	}
-	return &util.JSONResponse{
-		Code: http.StatusMethodNotAllowed,
-		JSON: jsonerror.NotFound("Bad method"),
-	}
-}
-
-// checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver.
-func checkRecaptchaEnabled(
-	cfg *config.ClientAPI,
-	w http.ResponseWriter,
-	req *http.Request,
-) *util.JSONResponse {
-	if !cfg.RecaptchaEnabled {
-		return writeHTTPMessage(w, req,
-			"Recaptcha login is disabled on this Homeserver",
-			http.StatusBadRequest,
-		)
-	}
-	return nil
 }
 
 // writeHTTPMessage writes the given header and message to the HTTP response writer.
@@ -201,13 +189,10 @@ func checkRecaptchaEnabled(
 func writeHTTPMessage(
 	w http.ResponseWriter, req *http.Request,
 	message string, header int,
-) *util.JSONResponse {
+) {
 	w.WriteHeader(header)
 	_, err := w.Write([]byte(message))
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("w.Write failed")
-		res := jsonerror.InternalServerError()
-		return &res
 	}
-	return nil
 }

clientapi/routing/auth_fallback_test.go

@@ -0,0 +1,149 @@
+package routing
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test/testrig"
+)
+
+func Test_AuthFallback(t *testing.T) {
+	base, _, _ := testrig.Base(nil)
+	defer base.Close()
+
+	for _, useHCaptcha := range []bool{false, true} {
+		for _, recaptchaEnabled := range []bool{false, true} {
+			for _, wantErr := range []bool{false, true} {
+				t.Run(fmt.Sprintf("useHCaptcha(%v) - recaptchaEnabled(%v) - wantErr(%v)", useHCaptcha, recaptchaEnabled, wantErr), func(t *testing.T) {
+					// Set the defaults for each test
+					base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
+					base.Cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled
+					base.Cfg.ClientAPI.RecaptchaPublicKey = "pub"
+					base.Cfg.ClientAPI.RecaptchaPrivateKey = "priv"
+					if useHCaptcha {
+						base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify"
+						base.Cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js"
+						base.Cfg.ClientAPI.RecaptchaFormField = "h-captcha-response"
+						base.Cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha"
+					}
+					cfgErrs := &config.ConfigErrors{}
+					base.Cfg.ClientAPI.Verify(cfgErrs)
+					if len(*cfgErrs) > 0 {
+						t.Fatalf("(hCaptcha=%v) unexpected config errors: %s", useHCaptcha, cfgErrs.Error())
+					}
+
+					req := httptest.NewRequest(http.MethodGet, "/?session=1337", nil)
+					rec := httptest.NewRecorder()
+
+					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+					if !recaptchaEnabled {
+						if rec.Code != http.StatusBadRequest {
+							t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
+						}
+						if rec.Body.String() != "Recaptcha login is disabled on this Homeserver" {
+							t.Fatalf("unexpected response body: %s", rec.Body.String())
+						}
+					} else {
+						if !strings.Contains(rec.Body.String(), base.Cfg.ClientAPI.RecaptchaSitekeyClass) {
+							t.Fatalf("body does not contain %s: %s", base.Cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String())
+						}
+					}
+
+					srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+						if wantErr {
+							_, _ = w.Write([]byte(`{"success":false}`))
+							return
+						}
+						_, _ = w.Write([]byte(`{"success":true}`))
+					}))
+					defer srv.Close() // nolint: errcheck
+
+					base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
+
+					// check the result after sending the captcha
+					req = httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
+					req.Form = url.Values{}
+					req.Form.Add(base.Cfg.ClientAPI.RecaptchaFormField, "someRandomValue")
+					rec = httptest.NewRecorder()
+					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+					if recaptchaEnabled {
+						if !wantErr {
+							if rec.Code != http.StatusOK {
+								t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusOK)
+							}
+							if rec.Body.String() != successTemplate {
+								t.Fatalf("unexpected response: %s, want %s", rec.Body.String(), successTemplate)
+							}
+						} else {
+							if rec.Code != http.StatusUnauthorized {
+								t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusUnauthorized)
+							}
+							wantString := "Authentication"
+							if !strings.Contains(rec.Body.String(), wantString) {
+								t.Fatalf("expected response to contain '%s', but didn't: %s", wantString, rec.Body.String())
+							}
+						}
+					} else {
+						if rec.Code != http.StatusBadRequest {
+							t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
+						}
+						if rec.Body.String() != "Recaptcha login is disabled on this Homeserver" {
+							t.Fatalf("unexpected response: %s, want %s", rec.Body.String(), "successTemplate")
+						}
+					}
+				})
+			}
+		}
+	}
+
+	t.Run("unknown fallbacks are handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, "DoesNotExist", &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusNotImplemented {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusNotImplemented)
+		}
+	})
+
+	t.Run("unknown methods are handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodDelete, "/?session=1337", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusMethodNotAllowed {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusMethodNotAllowed)
+		}
+	})
+
+	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
+		}
+	})
+
+	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
+		}
+	})
+
+	t.Run("missing 'response' is handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
+		}
+	})
+}

clientapi/routing/joinroom.go

@@ -37,6 +37,7 @@ func JoinRoomByIDOrAlias(
 	joinReq := roomserverAPI.PerformJoinRequest{
 		RoomIDOrAlias: roomIDOrAlias,
 		UserID:        device.UserID,
+		IsGuest:       device.AccountType == api.AccountTypeGuest,
 		Content:       map[string]interface{}{},
 	}
 	joinRes := roomserverAPI.PerformJoinResponse{}
@@ -84,7 +85,14 @@ func JoinRoomByIDOrAlias(
 		if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil {
 			done <- jsonerror.InternalAPIError(req.Context(), err)
 		} else if joinRes.Error != nil {
+			if joinRes.Error.Code == roomserverAPI.PerformErrorNotAllowed && device.AccountType == api.AccountTypeGuest {
+				done <- util.JSONResponse{
+					Code: http.StatusForbidden,
+					JSON: jsonerror.GuestAccessForbidden(joinRes.Error.Msg),
+				}
+			} else {
 				done <- joinRes.Error.JSONResponse()
+			}
 		} else {
 			done <- util.JSONResponse{
 				Code: http.StatusOK,

clientapi/routing/joinroom_test.go

@@ -0,0 +1,156 @@
+package routing
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+func TestJoinRoomByIDOrAlias(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	charlie := test.NewUser(t, test.WithAccountType(uapi.AccountTypeGuest))
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
+
+		// Create the users in the userapi
+		for _, u := range []*test.User{alice, bob, charlie} {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    "someRandomPassword",
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+
+		}
+
+		aliceDev := &uapi.Device{UserID: alice.ID}
+		bobDev := &uapi.Device{UserID: bob.ID}
+		charlieDev := &uapi.Device{UserID: charlie.ID, AccountType: uapi.AccountTypeGuest}
+
+		// create a room with disabled guest access and invite Bob
+		resp := createRoom(ctx, createRoomRequest{
+			Name:          "testing",
+			IsDirect:      true,
+			Topic:         "testing",
+			Visibility:    "public",
+			Preset:        presetPublicChat,
+			RoomAliasName: "alias",
+			Invite:        []string{bob.ID},
+			GuestCanJoin:  false,
+		}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
+		crResp, ok := resp.JSON.(createRoomResponse)
+		if !ok {
+			t.Fatalf("response is not a createRoomResponse: %+v", resp)
+		}
+
+		// create a room with guest access enabled and invite Charlie
+		resp = createRoom(ctx, createRoomRequest{
+			Name:         "testing",
+			IsDirect:     true,
+			Topic:        "testing",
+			Visibility:   "public",
+			Preset:       presetPublicChat,
+			Invite:       []string{charlie.ID},
+			GuestCanJoin: true,
+		}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
+		crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse)
+		if !ok {
+			t.Fatalf("response is not a createRoomResponse: %+v", resp)
+		}
+
+		// Dummy request
+		body := &bytes.Buffer{}
+		req, err := http.NewRequest(http.MethodPost, "/?server_name=test", body)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		testCases := []struct {
+			name        string
+			device      *uapi.Device
+			roomID      string
+			wantHTTP200 bool
+		}{
+			{
+				name:        "User can join successfully by alias",
+				device:      bobDev,
+				roomID:      crResp.RoomAlias,
+				wantHTTP200: true,
+			},
+			{
+				name:        "User can join successfully by roomID",
+				device:      bobDev,
+				roomID:      crResp.RoomID,
+				wantHTTP200: true,
+			},
+			{
+				name:   "join is forbidden if user is guest",
+				device: charlieDev,
+				roomID: crResp.RoomID,
+			},
+			{
+				name:   "room does not exist",
+				device: aliceDev,
+				roomID: "!doesnotexist:test",
+			},
+			{
+				name:   "user from different server",
+				device: &uapi.Device{UserID: "@wrong:server"},
+				roomID: crResp.RoomAlias,
+			},
+			{
+				name:   "user doesn't exist locally",
+				device: &uapi.Device{UserID: "@doesnotexist:test"},
+				roomID: crResp.RoomAlias,
+			},
+			{
+				name:   "invalid room ID",
+				device: aliceDev,
+				roomID: "invalidRoomID",
+			},
+			{
+				name:   "roomAlias does not exist",
+				device: aliceDev,
+				roomID: "#doesnotexist:test",
+			},
+			{
+				name:   "room with guest_access event",
+				device: charlieDev,
+				roomID: crRespWithGuestAccess.RoomID,
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				joinResp := JoinRoomByIDOrAlias(req, tc.device, rsAPI, userAPI, tc.roomID)
+				if tc.wantHTTP200 && !joinResp.Is2xx() {
+					t.Fatalf("expected join room to succeed, but didn't: %+v", joinResp)
+				}
+			})
+		}
+	})
+}

clientapi/routing/key_crosssigning.go

@@ -21,9 +21,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
@@ -34,8 +33,8 @@ type crossSigningRequest struct {
 
 func UploadCrossSigningDeviceKeys(
 	req *http.Request, userInteractiveAuth *auth.UserInteractive,
-	keyserverAPI api.ClientKeyAPI, device *userapi.Device,
+	keyserverAPI api.ClientKeyAPI, device *api.Device,
-	accountAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	accountAPI api.ClientUserAPI, cfg *config.ClientAPI,
 ) util.JSONResponse {
 	uploadReq := &crossSigningRequest{}
 	uploadRes := &api.PerformUploadDeviceKeysResponse{}
@@ -107,7 +106,7 @@ func UploadCrossSigningDeviceKeys(
 	}
 }
 
-func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
+func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse {
 	uploadReq := &api.PerformUploadDeviceSignaturesRequest{}
 	uploadRes := &api.PerformUploadDeviceSignaturesResponse{}
 

clientapi/routing/keys.go

@@ -23,8 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 type uploadKeysRequest struct {
@@ -32,7 +31,7 @@ type uploadKeysRequest struct {
 	OneTimeKeys map[string]json.RawMessage `json:"one_time_keys"`
 }
 
-func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
+func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse {
 	var r uploadKeysRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
@@ -106,7 +105,7 @@ func (r *queryKeysRequest) GetTimeout() time.Duration {
 	return timeout
 }
 
-func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
+func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse {
 	var r queryKeysRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {

clientapi/routing/login_test.go

@@ -0,0 +1,149 @@
+package routing
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+func TestLogin(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bobUser := &test.User{ID: "@bob:test", AccountType: uapi.AccountTypeUser}
+	charlie := &test.User{ID: "@Charlie:test", AccountType: uapi.AccountTypeUser}
+	vhUser := &test.User{ID: "@vhuser:vh1"}
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+		base.Cfg.ClientAPI.RateLimiting.Enabled = false
+		// add a vhost
+		base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
+		})
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		// Needed for /login
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+
+		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
+		Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, &base.Cfg.MSCs, nil)
+
+		// Create password
+		password := util.RandomString(8)
+
+		// create the users
+		for _, u := range []*test.User{aliceAdmin, bobUser, vhUser, charlie} {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    password,
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+			if !userRes.AccountCreated {
+				t.Fatalf("account not created")
+			}
+		}
+
+		testCases := []struct {
+			name   string
+			userID string
+			wantOK bool
+		}{
+			{
+				name:   "aliceAdmin can login",
+				userID: aliceAdmin.ID,
+				wantOK: true,
+			},
+			{
+				name:   "bobUser can login",
+				userID: bobUser.ID,
+				wantOK: true,
+			},
+			{
+				name:   "vhuser can login",
+				userID: vhUser.ID,
+				wantOK: true,
+			},
+			{
+				name:   "bob with uppercase can login",
+				userID: "@Bob:test",
+				wantOK: true,
+			},
+			{
+				name:   "Charlie can login (existing uppercase)",
+				userID: charlie.ID,
+				wantOK: true,
+			},
+			{
+				name:   "Charlie can not login with lowercase userID",
+				userID: strings.ToLower(charlie.ID),
+				wantOK: false,
+			},
+		}
+
+		ctx := context.Background()
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+					"type": authtypes.LoginTypePassword,
+					"identifier": map[string]interface{}{
+						"type": "m.id.user",
+						"user": tc.userID,
+					},
+					"password": password,
+				}))
+				rec := httptest.NewRecorder()
+				base.PublicClientAPIMux.ServeHTTP(rec, req)
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("failed to login: %s", rec.Body.String())
+				}
+
+				t.Logf("Response: %s", rec.Body.String())
+				// get the response
+				resp := loginResponse{}
+				if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+					t.Fatal(err)
+				}
+				// everything OK
+				if !tc.wantOK && resp.AccessToken == "" {
+					return
+				}
+				if tc.wantOK && resp.AccessToken == "" {
+					t.Fatalf("expected accessToken after successful login but got none: %+v", resp)
+				}
+
+				devicesResp := &uapi.QueryDevicesResponse{}
+				if err := userAPI.QueryDevices(ctx, &uapi.QueryDevicesRequest{UserID: resp.UserID}, devicesResp); err != nil {
+					t.Fatal(err)
+				}
+				for _, dev := range devicesResp.Devices {
+					// We expect the userID on the device to be the same as resp.UserID
+					if dev.UserID != resp.UserID {
+						t.Fatalf("unexpected userID on device: %s", dev.UserID)
+					}
+				}
+			})
+		}
+	})
+}

clientapi/routing/password.go

@@ -9,6 +9,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -147,8 +148,8 @@ func Password(
 	}
 
 	// Check the new password strength.
-	if resErr = validatePassword(r.NewPassword); resErr != nil {
+	if err := internal.ValidatePassword(r.NewPassword); err != nil {
-		return *resErr
+		return *internal.PasswordResponse(err)
 	}
 
 	// Ask the user API to perform the password change.

clientapi/routing/redaction.go

@@ -57,7 +57,7 @@ func SendRedaction(
 		}
 	}
 
-	ev := roomserverAPI.GetEvent(req.Context(), rsAPI, eventID)
+	ev := roomserverAPI.GetEvent(req.Context(), rsAPI, roomID, eventID)
 	if ev == nil {
 		return util.JSONResponse{
 			Code: 400,

clientapi/routing/register.go

@@ -18,18 +18,19 @@ package routing
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
-	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/internal/eventutil"
@@ -60,12 +61,7 @@ var (
 	)
 )
 
-const (
+const sessionIDLength = 24
-	minPasswordLength = 8   // http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based
-	maxPasswordLength = 512 // https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
-	maxUsernameLength = 254 // http://matrix.org/speculator/spec/HEAD/intro.html#user-identifiers TODO account for domain
-	sessionIDLength   = 24
-)
 
 // sessionsDict keeps track of completed auth stages for each session.
 // It shouldn't be passed by value because it contains a mutex.
@@ -201,7 +197,6 @@ func (d *sessionsDict) getDeviceToDelete(sessionID string) (string, bool) {
 
 var (
 	sessions = newSessionsDict()
-	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 )
 
 // registerRequest represents the submitted registration request.
@@ -267,7 +262,6 @@ func newUserInteractiveResponse(
 type registerResponse struct {
 	UserID      string `json:"user_id"`
 	AccessToken string `json:"access_token,omitempty"`
-	HomeServer  gomatrixserverlib.ServerName `json:"home_server"`
 	DeviceID    string `json:"device_id,omitempty"`
 }
 
@@ -279,83 +273,28 @@ type recaptchaResponse struct {
 	ErrorCodes  []int     `json:"error-codes"`
 }
 
-// validateUsername returns an error response if the username is invalid
+var (
-func validateUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
+	ErrInvalidCaptcha  = errors.New("invalid captcha response")
-	// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
+	ErrMissingResponse = errors.New("captcha response is required")
-	if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
+	ErrCaptchaDisabled = errors.New("captcha registration is disabled")
-		return &util.JSONResponse{
+)
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
-		}
-	} else if !validUsernameRegex.MatchString(localpart) {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
-		}
-	} else if localpart[0] == '_' { // Regex checks its not a zero length string
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username cannot start with a '_'"),
-		}
-	}
-	return nil
-}
-
-// validateApplicationServiceUsername returns an error response if the username is invalid for an application service
-func validateApplicationServiceUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
-	if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
-		}
-	} else if !validUsernameRegex.MatchString(localpart) {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
-		}
-	}
-	return nil
-}
-
-// validatePassword returns an error response if the password is invalid
-func validatePassword(password string) *util.JSONResponse {
-	// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
-	if len(password) > maxPasswordLength {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(fmt.Sprintf("'password' >%d characters", maxPasswordLength)),
-		}
-	} else if len(password) > 0 && len(password) < minPasswordLength {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.WeakPassword(fmt.Sprintf("password too weak: min %d chars", minPasswordLength)),
-		}
-	}
-	return nil
-}
 
 // validateRecaptcha returns an error response if the captcha response is invalid
 func validateRecaptcha(
 	cfg *config.ClientAPI,
 	response string,
 	clientip string,
-) *util.JSONResponse {
+) error {
 	ip, _, _ := net.SplitHostPort(clientip)
 	if !cfg.RecaptchaEnabled {
-		return &util.JSONResponse{
+		return ErrCaptchaDisabled
-			Code: http.StatusConflict,
-			JSON: jsonerror.Unknown("Captcha registration is disabled"),
-		}
 	}
 
 	if response == "" {
-		return &util.JSONResponse{
+		return ErrMissingResponse
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Captcha response is required"),
-		}
 	}
 
-	// Make a POST request to Google's API to check the captcha response
+	// Make a POST request to the captcha provider API to check the captcha response
 	resp, err := http.PostForm(cfg.RecaptchaSiteVerifyAPI,
 		url.Values{
 			"secret":   {cfg.RecaptchaPrivateKey},
@@ -365,10 +304,7 @@ func validateRecaptcha(
 	)
 
 	if err != nil {
-		return &util.JSONResponse{
+		return err
-			Code: http.StatusInternalServerError,
-			JSON: jsonerror.BadJSON("Error in requesting validation of captcha response"),
-		}
 	}
 
 	// Close the request once we're finishing reading from it
@@ -378,25 +314,16 @@ func validateRecaptcha(
 	var r recaptchaResponse
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return &util.JSONResponse{
+		return err
-			Code: http.StatusGatewayTimeout,
-			JSON: jsonerror.Unknown("Error in contacting captcha server" + err.Error()),
-		}
 	}
 	err = json.Unmarshal(body, &r)
 	if err != nil {
-		return &util.JSONResponse{
+		return err
-			Code: http.StatusInternalServerError,
-			JSON: jsonerror.BadJSON("Error in unmarshaling captcha server's response: " + err.Error()),
-		}
 	}
 
 	// Check that we received a "success"
 	if !r.Success {
-		return &util.JSONResponse{
+		return ErrInvalidCaptcha
-			Code: http.StatusUnauthorized,
-			JSON: jsonerror.BadJSON("Invalid captcha response. Please try again."),
-		}
 	}
 	return nil
 }
@@ -528,8 +455,8 @@ func validateApplicationService(
 	}
 
 	// Check username application service is trying to register is valid
-	if err := validateApplicationServiceUsername(username, cfg.Matrix.ServerName); err != nil {
+	if err := internal.ValidateApplicationServiceUsername(username, cfg.Matrix.ServerName); err != nil {
-		return "", err
+		return "", internal.UsernameResponse(err)
 	}
 
 	// No errors, registration valid
@@ -584,15 +511,12 @@ func Register(
 	if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil {
 		return *resErr
 	}
-	if l, d, err := cfg.Matrix.SplitLocalID('@', r.Username); err == nil {
-		r.Username, r.ServerName = l, d
-	}
 	if req.URL.Query().Get("kind") == "guest" {
 		return handleGuestRegistration(req, r, cfg, userAPI)
 	}
 
 	// Don't allow numeric usernames less than MAX_INT64.
-	if _, err := strconv.ParseInt(r.Username, 10, 64); err == nil {
+	if _, err = strconv.ParseInt(r.Username, 10, 64); err == nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.InvalidUsername("Numeric user IDs are reserved"),
@@ -604,7 +528,7 @@ func Register(
 			ServerName: r.ServerName,
 		}
 		nres := &userapi.QueryNumericLocalpartResponse{}
-		if err := userAPI.QueryNumericLocalpart(req.Context(), nreq, nres); err != nil {
+		if err = userAPI.QueryNumericLocalpart(req.Context(), nreq, nres); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryNumericLocalpart failed")
 			return jsonerror.InternalServerError()
 		}
@@ -621,8 +545,8 @@ func Register(
 	case r.Type == authtypes.LoginTypeApplicationService && accessTokenErr == nil:
 		// Spec-compliant case (the access_token is specified and the login type
 		// is correctly set, so it's an appservice registration)
-		if resErr := validateApplicationServiceUsername(r.Username, r.ServerName); resErr != nil {
+		if err = internal.ValidateApplicationServiceUsername(r.Username, r.ServerName); err != nil {
-			return *resErr
+			return *internal.UsernameResponse(err)
 		}
 	case accessTokenErr == nil:
 		// Non-spec-compliant case (the access_token is specified but the login
@@ -634,12 +558,12 @@ func Register(
 	default:
 		// Spec-compliant case (neither the access_token nor the login type are
 		// specified, so it's a normal user registration)
-		if resErr := validateUsername(r.Username, r.ServerName); resErr != nil {
+		if err = internal.ValidateUsername(r.Username, r.ServerName); err != nil {
-			return *resErr
+			return *internal.UsernameResponse(err)
 		}
 	}
-	if resErr := validatePassword(r.Password); resErr != nil {
+	if err = internal.ValidatePassword(r.Password); err != nil {
-		return *resErr
+		return *internal.PasswordResponse(err)
 	}
 
 	logger := util.GetLogger(req.Context())
@@ -717,7 +641,6 @@ func handleGuestRegistration(
 		JSON: registerResponse{
 			UserID:      devRes.Device.UserID,
 			AccessToken: devRes.Device.AccessToken,
-			HomeServer:  res.Account.ServerName,
 			DeviceID:    devRes.Device.ID,
 		},
 	}
@@ -782,9 +705,18 @@ func handleRegistrationFlow(
 	switch r.Auth.Type {
 	case authtypes.LoginTypeRecaptcha:
 		// Check given captcha response
-		resErr := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
+		err := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
-		if resErr != nil {
+		switch err {
-			return *resErr
+		case ErrCaptchaDisabled:
+			return util.JSONResponse{Code: http.StatusForbidden, JSON: jsonerror.Unknown(err.Error())}
+		case ErrMissingResponse:
+			return util.JSONResponse{Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(err.Error())}
+		case ErrInvalidCaptcha:
+			return util.JSONResponse{Code: http.StatusUnauthorized, JSON: jsonerror.BadJSON(err.Error())}
+		case nil:
+		default:
+			util.GetLogger(req.Context()).WithError(err).Error("failed to validate recaptcha")
+			return util.JSONResponse{Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError()}
 		}
 
 		// Add Recaptcha to the list of completed registration stages
@@ -874,7 +806,7 @@ func handleApplicationServiceRegistration(
 	// Don't need to worry about appending to registration stages as
 	// application service registration is entirely separate.
 	return completeRegistration(
-		req.Context(), userAPI, r.Username, r.ServerName, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session,
+		req.Context(), userAPI, r.Username, r.ServerName, "", "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session,
 		r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService, nil,
 	)
 }
@@ -894,7 +826,7 @@ func checkAndCompleteFlow(
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
 		return completeRegistration(
-			req.Context(), userAPI, r.Username, r.ServerName, r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID,
+			req.Context(), userAPI, r.Username, r.ServerName, "", r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID,
 			r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser, threePid,
 		)
 	}
@@ -917,10 +849,10 @@ func checkAndCompleteFlow(
 func completeRegistration(
 	ctx context.Context,
 	userAPI userapi.ClientUserAPI,
-	username string, serverName gomatrixserverlib.ServerName,
+	username string, serverName gomatrixserverlib.ServerName, displayName string,
 	password, appserviceID, ipAddr, userAgent, sessionID string,
 	inhibitLogin eventutil.WeakBoolean,
-	displayName, deviceID *string,
+	deviceDisplayName, deviceID *string,
 	accType userapi.AccountType,
 	threePid *authtypes.ThreePID,
 ) util.JSONResponse {
@@ -985,7 +917,6 @@ func completeRegistration(
 			Code: http.StatusOK,
 			JSON: registerResponse{
 				UserID: userutil.MakeUserID(username, accRes.Account.ServerName),
-				HomeServer: accRes.Account.ServerName,
 			},
 		}
 	}
@@ -998,12 +929,28 @@ func completeRegistration(
 		}
 	}
 
+	if displayName != "" {
+		nameReq := userapi.PerformUpdateDisplayNameRequest{
+			Localpart:   username,
+			ServerName:  serverName,
+			DisplayName: displayName,
+		}
+		var nameRes userapi.PerformUpdateDisplayNameResponse
+		err = userAPI.SetDisplayName(ctx, &nameReq, &nameRes)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: jsonerror.Unknown("failed to set display name: " + err.Error()),
+			}
+		}
+	}
+
 	var devRes userapi.PerformDeviceCreationResponse
 	err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
 		Localpart:         username,
 		ServerName:        serverName,
 		AccessToken:       token,
-		DeviceDisplayName: displayName,
+		DeviceDisplayName: deviceDisplayName,
 		DeviceID:          deviceID,
 		IPAddr:            ipAddr,
 		UserAgent:         userAgent,
@@ -1018,7 +965,6 @@ func completeRegistration(
 	result := registerResponse{
 		UserID:      devRes.Device.UserID,
 		AccessToken: devRes.Device.AccessToken,
-		HomeServer:  accRes.Account.ServerName,
 		DeviceID:    devRes.Device.ID,
 	}
 	sessions.addCompletedRegistration(sessionID, result)
@@ -1114,8 +1060,8 @@ func RegisterAvailable(
 		}
 	}
 
-	if err := validateUsername(username, domain); err != nil {
+	if err := internal.ValidateUsername(username, domain); err != nil {
-		return *err
+		return *internal.UsernameResponse(err)
 	}
 
 	// Check if this username is reserved by an application service
@@ -1177,11 +1123,11 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
 	// downcase capitals
 	ssrr.User = strings.ToLower(ssrr.User)
 
-	if resErr := validateUsername(ssrr.User, cfg.Matrix.ServerName); resErr != nil {
+	if err = internal.ValidateUsername(ssrr.User, cfg.Matrix.ServerName); err != nil {
-		return *resErr
+		return *internal.UsernameResponse(err)
 	}
-	if resErr := validatePassword(ssrr.Password); resErr != nil {
+	if err = internal.ValidatePassword(ssrr.Password); err != nil {
-		return *resErr
+		return *internal.PasswordResponse(err)
 	}
 	deviceID := "shared_secret_registration"
 
@@ -1189,5 +1135,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
 	if ssrr.Admin {
 		accType = userapi.AccountTypeAdmin
 	}
-	return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType, nil)
+	return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.DisplayName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType, nil)
 }

clientapi/routing/register_secret.go

@@ -24,6 +24,7 @@ type SharedSecretRegistrationRequest struct {
 	MacBytes    []byte
 	MacStr      string `json:"mac"`
 	Admin       bool   `json:"admin"`
+	DisplayName string `json:"displayname,omitempty"`
 }
 
 func NewSharedSecretRegistrationRequest(reader io.ReadCloser) (*SharedSecretRegistrationRequest, error) {

clientapi/routing/register_secret_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestSharedSecretRegister(t *testing.T) {
 	// these values have come from a local synapse instance to ensure compatibility
-	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`)
+	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
 	sharedSecret := "dendritetest"
 
 	req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))

clientapi/routing/register_test.go

@@ -15,12 +15,30 @@
 package routing
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
 	"regexp"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+	"github.com/patrickmn/go-cache"
+	"github.com/stretchr/testify/assert"
 )
 
 var (
@@ -183,7 +201,7 @@ func TestValidationOfApplicationServices(t *testing.T) {
 	fakeConfig := &config.Dendrite{}
 	fakeConfig.Defaults(config.DefaultOpts{
 		Generate:       true,
-		Monolithic: true,
+		SingleDatabase: true,
 	})
 	fakeConfig.Global.ServerName = "localhost"
 	fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}
@@ -264,3 +282,378 @@ func TestSessionCleanUp(t *testing.T) {
 		}
 	})
 }
+
+func Test_register(t *testing.T) {
+	testCases := []struct {
+		name                 string
+		kind                 string
+		password             string
+		username             string
+		loginType            string
+		forceEmpty           bool
+		registrationDisabled bool
+		guestsDisabled       bool
+		enableRecaptcha      bool
+		captchaBody          string
+		wantResponse         util.JSONResponse
+	}{
+		{
+			name:           "disallow guests",
+			kind:           "guest",
+			guestsDisabled: true,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden(`Guest registration is disabled on "test"`),
+			},
+		},
+		{
+			name: "allow guests",
+			kind: "guest",
+		},
+		{
+			name:      "unknown login type",
+			loginType: "im.not.known",
+			wantResponse: util.JSONResponse{
+				Code: http.StatusNotImplemented,
+				JSON: jsonerror.Unknown("unknown/unimplemented auth type"),
+			},
+		},
+		{
+			name:                 "disabled registration",
+			registrationDisabled: true,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden(`Registration is disabled on "test"`),
+			},
+		},
+		{
+			name:       "successful registration, numeric ID",
+			username:   "",
+			password:   "someRandomPassword",
+			forceEmpty: true,
+		},
+		{
+			name:     "successful registration",
+			username: "success",
+		},
+		{
+			name:     "failing registration - user already exists",
+			username: "success",
+			wantResponse: util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.UserInUse("Desired user ID is already taken."),
+			},
+		},
+		{
+			name:     "successful registration uppercase username",
+			username: "LOWERCASED", // this is going to be lower-cased
+		},
+		{
+			name:         "invalid username",
+			username:     "#totalyNotValid",
+			wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
+		},
+		{
+			name:     "numeric username is forbidden",
+			username: "1337",
+			wantResponse: util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidUsername("Numeric user IDs are reserved"),
+			},
+		},
+		{
+			name:      "disabled recaptcha login",
+			loginType: authtypes.LoginTypeRecaptcha,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Unknown(ErrCaptchaDisabled.Error()),
+			},
+		},
+		{
+			name:            "enabled recaptcha, no response defined",
+			enableRecaptcha: true,
+			loginType:       authtypes.LoginTypeRecaptcha,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON(ErrMissingResponse.Error()),
+			},
+		},
+		{
+			name:            "invalid captcha response",
+			enableRecaptcha: true,
+			loginType:       authtypes.LoginTypeRecaptcha,
+			captchaBody:     `notvalid`,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: jsonerror.BadJSON(ErrInvalidCaptcha.Error()),
+			},
+		},
+		{
+			name:            "valid captcha response",
+			enableRecaptcha: true,
+			loginType:       authtypes.LoginTypeRecaptcha,
+			captchaBody:     `success`,
+		},
+		{
+			name:            "captcha invalid from remote",
+			enableRecaptcha: true,
+			loginType:       authtypes.LoginTypeRecaptcha,
+			captchaBody:     `i should fail for other reasons`,
+			wantResponse:    util.JSONResponse{Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError()},
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				if tc.enableRecaptcha {
+					srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+						if err := r.ParseForm(); err != nil {
+							t.Fatal(err)
+						}
+						response := r.Form.Get("response")
+
+						// Respond with valid JSON or no JSON at all to test happy/error cases
+						switch response {
+						case "success":
+							json.NewEncoder(w).Encode(recaptchaResponse{Success: true})
+						case "notvalid":
+							json.NewEncoder(w).Encode(recaptchaResponse{Success: false})
+						default:
+
+						}
+					}))
+					defer srv.Close()
+					base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
+				}
+
+				if err := base.Cfg.Derive(); err != nil {
+					t.Fatalf("failed to derive config: %s", err)
+				}
+
+				base.Cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha
+				base.Cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled
+				base.Cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled
+
+				if tc.kind == "" {
+					tc.kind = "user"
+				}
+				if tc.password == "" && !tc.forceEmpty {
+					tc.password = "someRandomPassword"
+				}
+				if tc.username == "" && !tc.forceEmpty {
+					tc.username = "valid"
+				}
+				if tc.loginType == "" {
+					tc.loginType = "m.login.dummy"
+				}
+
+				reg := registerRequest{
+					Password: tc.password,
+					Username: tc.username,
+				}
+
+				body := &bytes.Buffer{}
+				err := json.NewEncoder(body).Encode(reg)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/?kind=%s", tc.kind), body)
+
+				resp := Register(req, userAPI, &base.Cfg.ClientAPI)
+				t.Logf("Resp: %+v", resp)
+
+				// The first request should return a userInteractiveResponse
+				switch r := resp.JSON.(type) {
+				case userInteractiveResponse:
+					// Check that the flows are the ones we configured
+					if !reflect.DeepEqual(r.Flows, base.Cfg.Derived.Registration.Flows) {
+						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, base.Cfg.Derived.Registration.Flows)
+					}
+				case *jsonerror.MatrixError:
+					if !reflect.DeepEqual(tc.wantResponse, resp) {
+						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse)
+					}
+					return
+				case registerResponse:
+					// this should only be possible on guest user registration, never for normal users
+					if tc.kind != "guest" {
+						t.Fatalf("got register response on first request: %+v", r)
+					}
+					// assert we've got a UserID, AccessToken and DeviceID
+					if r.UserID == "" {
+						t.Fatalf("missing userID in response")
+					}
+					if r.AccessToken == "" {
+						t.Fatalf("missing accessToken in response")
+					}
+					if r.DeviceID == "" {
+						t.Fatalf("missing deviceID in response")
+					}
+					return
+				default:
+					t.Logf("Got response: %T", resp.JSON)
+				}
+
+				// If we reached this, we should have received a UIA response
+				uia, ok := resp.JSON.(userInteractiveResponse)
+				if !ok {
+					t.Fatalf("did not receive a userInteractiveResponse: %T", resp.JSON)
+				}
+				t.Logf("%+v", uia)
+
+				// Register the user
+				reg.Auth = authDict{
+					Type:    authtypes.LoginType(tc.loginType),
+					Session: uia.Session,
+				}
+
+				if tc.captchaBody != "" {
+					reg.Auth.Response = tc.captchaBody
+				}
+
+				dummy := "dummy"
+				reg.DeviceID = &dummy
+				reg.InitialDisplayName = &dummy
+				reg.Type = authtypes.LoginType(tc.loginType)
+
+				err = json.NewEncoder(body).Encode(reg)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				req = httptest.NewRequest(http.MethodPost, "/", body)
+
+				resp = Register(req, userAPI, &base.Cfg.ClientAPI)
+
+				switch resp.JSON.(type) {
+				case *jsonerror.MatrixError:
+					if !reflect.DeepEqual(tc.wantResponse, resp) {
+						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
+					}
+					return
+				case util.JSONResponse:
+					if !reflect.DeepEqual(tc.wantResponse, resp) {
+						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
+					}
+					return
+				}
+
+				rr, ok := resp.JSON.(registerResponse)
+				if !ok {
+					t.Fatalf("expected a registerresponse, got %T", resp.JSON)
+				}
+
+				// validate the response
+				if tc.forceEmpty {
+					// when not supplying a username, one will be generated. Given this _SHOULD_ be
+					// the second user, set the username accordingly
+					reg.Username = "2"
+				}
+				wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test"))
+				if wantUserID != rr.UserID {
+					t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID)
+				}
+				if rr.DeviceID != *reg.DeviceID {
+					t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID)
+				}
+				if rr.AccessToken == "" {
+					t.Fatalf("missing accessToken in response")
+				}
+			})
+		}
+	})
+}
+
+func TestRegisterUserWithDisplayName(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+		base.Cfg.Global.ServerName = "server"
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		deviceName, deviceID := "deviceName", "deviceID"
+		expectedDisplayName := "DisplayName"
+		response := completeRegistration(
+			base.Context(),
+			userAPI,
+			"user",
+			"server",
+			expectedDisplayName,
+			"password",
+			"",
+			"localhost",
+			"user agent",
+			"session",
+			false,
+			&deviceName,
+			&deviceID,
+			api.AccountTypeAdmin,
+			nil,
+		)
+
+		assert.Equal(t, http.StatusOK, response.Code)
+
+		req := api.QueryProfileRequest{UserID: "@user:server"}
+		var res api.QueryProfileResponse
+		err := userAPI.QueryProfile(base.Context(), &req, &res)
+		assert.NoError(t, err)
+		assert.Equal(t, expectedDisplayName, res.DisplayName)
+	})
+}
+
+func TestRegisterAdminUsingSharedSecret(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+		base.Cfg.Global.ServerName = "server"
+		sharedSecret := "dendritetest"
+		base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+
+		expectedDisplayName := "rabbit"
+		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
+		req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))
+		assert.NoError(t, err)
+		if err != nil {
+			t.Fatalf("failed to read request: %s", err)
+		}
+
+		r := NewSharedSecretRegistration(sharedSecret)
+
+		// force the nonce to be known
+		r.nonces.Set(req.Nonce, true, cache.DefaultExpiration)
+
+		_, err = r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes)
+		assert.NoError(t, err)
+
+		body := &bytes.Buffer{}
+		err = json.NewEncoder(body).Encode(req)
+		assert.NoError(t, err)
+		ssrr := httptest.NewRequest(http.MethodPost, "/", body)
+
+		response := handleSharedSecretRegistration(
+			&base.Cfg.ClientAPI,
+			userAPI,
+			r,
+			ssrr,
+		)
+		assert.Equal(t, http.StatusOK, response.Code)
+
+		profilReq := api.QueryProfileRequest{UserID: "@alice:server"}
+		var profileRes api.QueryProfileResponse
+		err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes)
+		assert.NoError(t, err)
+		assert.Equal(t, expectedDisplayName, profileRes.DisplayName)
+	})
+}

clientapi/routing/routing.go

@@ -18,8 +18,11 @@ import (
 	"context"
 	"net/http"
 	"strings"
+	"sync"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/setup/base"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
@@ -36,11 +39,9 @@ import (
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/transactions"
-	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
@@ -50,7 +51,7 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	publicAPIMux, wkMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
+	base *base.BaseDendrite,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
@@ -60,11 +61,17 @@ func Setup(
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
 	federationSender federationAPI.ClientFederationAPI,
-	keyAPI keyserverAPI.ClientKeyAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	mscCfg *config.MSCs, natsClient *nats.Conn,
 ) {
+	publicAPIMux := base.PublicClientAPIMux
+	wkMux := base.PublicWellKnownAPIMux
+	synapseAdminRouter := base.SynapseAdminMux
+	dendriteAdminRouter := base.DendriteAdminMux
+
+	if base.EnableMetrics {
 		prometheus.MustRegister(amtRegUsers, sendEventDuration)
+	}
 
 	rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
 	rateLimitsFailedLogin := ratelimit.NewRtFailedLogin(&cfg.RtFailedLogin)
@@ -159,6 +166,12 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/purgeRoom/{roomID}",
+		httputil.MakeAdminAPI("admin_purge_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminPurgeRoom(req, cfg, device, rsAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	dendriteAdminRouter.Handle("/admin/resetPassword/{userID}",
 		httputil.MakeAdminAPI("admin_reset_password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return AdminResetPassword(req, cfg, device, userAPI)
@@ -179,25 +192,31 @@ func Setup(
 
 	dendriteAdminRouter.Handle("/admin/refreshDevices/{userID}",
 		httputil.MakeAdminAPI("admin_refresh_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminMarkAsStale(req, cfg, keyAPI)
+			return AdminMarkAsStale(req, cfg, userAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
-		serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
+		var serverNotificationSender *userapi.Device
-		if err != nil {
+		var err error
-			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+		notificationSenderOnce := &sync.Once{}
-		}
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
 			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+				notificationSenderOnce.Do(func() {
+					serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
+					if err != nil {
+						logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+					}
+				})
 				// not specced, but ensure we're rate limiting requests to this endpoint
 				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
 				}
-				vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+				var vars map[string]string
+				vars, err = httputil.URLDecodeMapValues(mux.Vars(req))
 				if err != nil {
 					return util.ErrorResponse(err)
 				}
@@ -213,6 +232,12 @@ func Setup(
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice",
 			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+				notificationSenderOnce.Do(func() {
+					serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
+					if err != nil {
+						logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+					}
+				})
 				// not specced, but ensure we're rate limiting requests to this endpoint
 				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
@@ -644,9 +669,9 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	v3mux.Handle("/auth/{authType}/fallback/web",
-		httputil.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse {
+		httputil.MakeHTMLAPI("auth_fallback", base.EnableMetrics, func(w http.ResponseWriter, req *http.Request) {
 			vars := mux.Vars(req)
-			return AuthFallback(w, req, vars["authType"], cfg)
+			AuthFallback(w, req, vars["authType"], cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
@@ -1358,11 +1383,11 @@ func Setup(
 	// Cross-signing device keys
 
 	postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, keyAPI, device, userAPI, cfg)
+		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg)
 	})
 
 	postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceSignatures(req, keyAPI, device)
+		return UploadCrossSigningDeviceSignatures(req, userAPI, device)
 	}, httputil.WithAllowGuests())
 
 	v3mux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions)
@@ -1374,22 +1399,22 @@ func Setup(
 	// Supplying a device ID is deprecated.
 	v3mux.Handle("/keys/upload/{deviceID}",
 		httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return UploadKeys(req, keyAPI, device)
+			return UploadKeys(req, userAPI, device)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/keys/upload",
 		httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return UploadKeys(req, keyAPI, device)
+			return UploadKeys(req, userAPI, device)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/keys/query",
 		httputil.MakeAuthAPI("keys_query", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return QueryKeys(req, keyAPI, device)
+			return QueryKeys(req, userAPI, device)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/keys/claim",
 		httputil.MakeAuthAPI("keys_claim", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return ClaimKeys(req, keyAPI)
+			return ClaimKeys(req, userAPI)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",

cmd/create-account/main.go

@@ -25,10 +25,10 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"regexp"
 	"strings"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/tidwall/gjson"
 
 	"github.com/sirupsen/logrus"
@@ -65,7 +65,6 @@ var (
 	isAdmin       = flag.Bool("admin", false, "Create an admin account")
 	resetPassword = flag.Bool("reset-password", false, "Deprecated")
 	serverURL     = flag.String("url", "http://localhost:8008", "The URL to connect to.")
-	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 	timeout       = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
 )
 
@@ -95,20 +94,21 @@ func main() {
 		os.Exit(1)
 	}
 
-	if !validUsernameRegex.MatchString(*username) {
+	if err := internal.ValidateUsername(*username, cfg.Global.ServerName); err != nil {
-		logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='")
+		logrus.WithError(err).Error("Specified username is invalid")
 		os.Exit(1)
 	}
 
-	if len(fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName)) > 255 {
-		logrus.Fatalf("Username can not be longer than 255 characters: %s", fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName))
-	}
-
 	pass, err := getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
 	if err != nil {
 		logrus.Fatalln(err)
 	}
 
+	if err = internal.ValidatePassword(pass); err != nil {
+		logrus.WithError(err).Error("Specified password is invalid")
+		os.Exit(1)
+	}
+
 	cl.Timeout = *timeout
 
 	accessToken, err := sharedSecretRegister(cfg.ClientAPI.RegistrationSharedSecret, *serverURL, *username, pass, *isAdmin)
@@ -177,7 +177,7 @@ func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, a
 	defer regResp.Body.Close() // nolint: errcheck
 	if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
 		body, _ = io.ReadAll(regResp.Body)
-		return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
+		return "", fmt.Errorf("got HTTP %d error from server: %s", regResp.StatusCode, string(body))
 	}
 	r, err := io.ReadAll(regResp.Body)
 	if err != nil {

cmd/dendrite-demo-pinecone/README.md

@@ -1,26 +0,0 @@
-# Pinecone Demo
-
-This is the Dendrite Pinecone demo! It's easy to get started.
-
-To run the homeserver, start at the root of the Dendrite repository and run:
-
-```
-go run ./cmd/dendrite-demo-pinecone
-```
-
-To connect to the static Pinecone peer used by the mobile demos run:
-
-```
-go run ./cmd/dendrite-demo-pinecone -peer wss://pinecone.matrix.org/public
-```
-
-The following command line arguments are accepted:
-
-* `-peer tcp://a.b.c.d:e` to specify a static Pinecone peer to connect to - you will need to supply this if you do not have another Pinecone node on your network
-* `-port 12345` to specify a port to listen on for client connections
-
-Then point your favourite Matrix client to  the homeserver URL`http://localhost:8008` (or whichever `-port` you specified), create an account and log in.
-
-If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer.
-
-Once logged in, you should be able to open the room directory or join a room by its ID.

cmd/dendrite-demo-pinecone/conn/client.go

@@ -1,107 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package conn
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"net/http"
-	"strings"
-
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/gomatrixserverlib"
-	"nhooyr.io/websocket"
-
-	pineconeRouter "github.com/matrix-org/pinecone/router"
-	pineconeSessions "github.com/matrix-org/pinecone/sessions"
-)
-
-func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error {
-	var parent net.Conn
-	if strings.HasPrefix(peer, "ws://") || strings.HasPrefix(peer, "wss://") {
-		ctx := context.Background()
-		c, _, err := websocket.Dial(ctx, peer, nil)
-		if err != nil {
-			return fmt.Errorf("websocket.DefaultDialer.Dial: %w", err)
-		}
-		parent = websocket.NetConn(ctx, c, websocket.MessageBinary)
-	} else {
-		var err error
-		parent, err = net.Dial("tcp", peer)
-		if err != nil {
-			return fmt.Errorf("net.Dial: %w", err)
-		}
-	}
-	if parent == nil {
-		return fmt.Errorf("failed to wrap connection")
-	}
-	_, err := pRouter.Connect(
-		parent,
-		pineconeRouter.ConnectionZone("static"),
-		pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
-		pineconeRouter.ConnectionURI(peer),
-	)
-	return err
-}
-
-type RoundTripper struct {
-	inner *http.Transport
-}
-
-func (y *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.URL.Scheme = "http"
-	return y.inner.RoundTrip(req)
-}
-
-func createTransport(s *pineconeSessions.Sessions) *http.Transport {
-	proto := s.Protocol("matrix")
-	tr := &http.Transport{
-		DisableKeepAlives: false,
-		Dial:              proto.Dial,
-		DialContext:       proto.DialContext,
-		DialTLS:           proto.DialTLS,
-		DialTLSContext:    proto.DialTLSContext,
-	}
-	tr.RegisterProtocol(
-		"matrix", &RoundTripper{
-			inner: &http.Transport{
-				DisableKeepAlives: false,
-				Dial:              proto.Dial,
-				DialContext:       proto.DialContext,
-				DialTLS:           proto.DialTLS,
-				DialTLSContext:    proto.DialTLSContext,
-			},
-		},
-	)
-	return tr
-}
-
-func CreateClient(
-	base *base.BaseDendrite, s *pineconeSessions.Sessions,
-) *gomatrixserverlib.Client {
-	return gomatrixserverlib.NewClient(
-		gomatrixserverlib.WithTransport(createTransport(s)),
-	)
-}
-
-func CreateFederationClient(
-	base *base.BaseDendrite, s *pineconeSessions.Sessions,
-) *gomatrixserverlib.FederationClient {
-	return gomatrixserverlib.NewFederationClient(
-		base.Cfg.Global.SigningIdentities(),
-		gomatrixserverlib.WithTransport(createTransport(s)),
-	)
-}

cmd/dendrite-demo-pinecone/conn/ws.go

@@ -1,95 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package conn
-
-import (
-	"io"
-	"net"
-	"time"
-
-	"github.com/gorilla/websocket"
-)
-
-func WrapWebSocketConn(c *websocket.Conn) *WebSocketConn {
-	return &WebSocketConn{c: c}
-}
-
-type WebSocketConn struct {
-	r io.Reader
-	c *websocket.Conn
-}
-
-func (c *WebSocketConn) Write(p []byte) (int, error) {
-	err := c.c.WriteMessage(websocket.BinaryMessage, p)
-	if err != nil {
-		return 0, err
-	}
-	return len(p), nil
-}
-
-func (c *WebSocketConn) Read(p []byte) (int, error) {
-	for {
-		if c.r == nil {
-			// Advance to next message.
-			var err error
-			_, c.r, err = c.c.NextReader()
-			if err != nil {
-				return 0, err
-			}
-		}
-		n, err := c.r.Read(p)
-		if err == io.EOF {
-			// At end of message.
-			c.r = nil
-			if n > 0 {
-				return n, nil
-			} else {
-				// No data read, continue to next message.
-				continue
-			}
-		}
-		return n, err
-	}
-}
-
-func (c *WebSocketConn) Close() error {
-	return c.c.Close()
-}
-
-func (c *WebSocketConn) LocalAddr() net.Addr {
-	return c.c.LocalAddr()
-}
-
-func (c *WebSocketConn) RemoteAddr() net.Addr {
-	return c.c.RemoteAddr()
-}
-
-func (c *WebSocketConn) SetDeadline(t time.Time) error {
-	if err := c.SetReadDeadline(t); err != nil {
-		return err
-	}
-	if err := c.SetWriteDeadline(t); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (c *WebSocketConn) SetReadDeadline(t time.Time) error {
-	return c.c.SetReadDeadline(t)
-}
-
-func (c *WebSocketConn) SetWriteDeadline(t time.Time) error {
-	return c.c.SetWriteDeadline(t)
-}

cmd/dendrite-demo-pinecone/defaults/defaults.go

@@ -1,21 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package defaults
-
-import "github.com/matrix-org/gomatrixserverlib"
-
-var DefaultServerNames = map[gomatrixserverlib.ServerName]struct{}{
-	"3bf0258d23c60952639cc4c69c71d1508a7d43a0475d9000ff900a1848411ec7": {},
-}

cmd/dendrite-demo-pinecone/embed/embed_elementweb.go

@@ -1,98 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build elementweb
-// +build elementweb
-
-package embed
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"regexp"
-
-	"github.com/gorilla/mux"
-	"github.com/tidwall/sjson"
-)
-
-// From within the Element Web directory:
-// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_elementweb.go -private -pkg embed .
-
-var cssFile = regexp.MustCompile("\\.css$")
-var jsFile = regexp.MustCompile("\\.js$")
-
-type mimeFixingHandler struct {
-	fs http.Handler
-}
-
-func (h mimeFixingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	ruri := r.RequestURI
-	fmt.Println(ruri)
-	switch {
-	case cssFile.MatchString(ruri):
-		w.Header().Set("Content-Type", "text/css")
-	case jsFile.MatchString(ruri):
-		w.Header().Set("Content-Type", "application/javascript")
-	default:
-	}
-	h.fs.ServeHTTP(w, r)
-}
-
-func Embed(rootMux *mux.Router, listenPort int, serverName string) {
-	embeddedFS := _escFS(false)
-	embeddedServ := mimeFixingHandler{http.FileServer(embeddedFS)}
-
-	rootMux.NotFoundHandler = embeddedServ
-	rootMux.HandleFunc("/config.json", func(w http.ResponseWriter, r *http.Request) {
-		url := fmt.Sprintf("http://%s:%d", r.Header("Host"), listenPort)
-		configFile, err := embeddedFS.Open("/config.sample.json")
-		if err != nil {
-			w.WriteHeader(500)
-			io.WriteString(w, "Couldn't open the file: "+err.Error())
-			return
-		}
-		configFileInfo, err := configFile.Stat()
-		if err != nil {
-			w.WriteHeader(500)
-			io.WriteString(w, "Couldn't stat the file: "+err.Error())
-			return
-		}
-		buf := make([]byte, configFileInfo.Size())
-		n, err := configFile.Read(buf)
-		if err != nil {
-			w.WriteHeader(500)
-			io.WriteString(w, "Couldn't read the file: "+err.Error())
-			return
-		}
-		if int64(n) != configFileInfo.Size() {
-			w.WriteHeader(500)
-			io.WriteString(w, "The returned file size didn't match what we expected")
-			return
-		}
-		js, _ := sjson.SetBytes(buf, "default_server_config.m\\.homeserver.base_url", url)
-		js, _ = sjson.SetBytes(js, "default_server_config.m\\.homeserver.server_name", serverName)
-		js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Element %s", serverName))
-		js, _ = sjson.SetBytes(js, "disable_guests", true)
-		js, _ = sjson.SetBytes(js, "disable_3pid_login", true)
-		js, _ = sjson.DeleteBytes(js, "welcomeUserId")
-		_, _ = w.Write(js)
-	})
-
-	fmt.Println("*-------------------------------*")
-	fmt.Println("| This build includes Element Web! |")
-	fmt.Println("*-------------------------------*")
-	fmt.Println("Point your browser to:", url)
-	fmt.Println()
-}

cmd/dendrite-demo-pinecone/embed/embed_other.go

@@ -1,24 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build !elementweb
-// +build !elementweb
-
-package embed
-
-import "github.com/gorilla/mux"
-
-func Embed(_ *mux.Router, _ int, _ string) {
-
-}

cmd/dendrite-demo-pinecone/main.go

@@ -1,332 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"crypto/ed25519"
-	"crypto/tls"
-	"encoding/hex"
-	"flag"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/gorilla/mux"
-	"github.com/gorilla/websocket"
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/embed"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	pineconeConnections "github.com/matrix-org/pinecone/connections"
-	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
-	pineconeRouter "github.com/matrix-org/pinecone/router"
-	pineconeEvents "github.com/matrix-org/pinecone/router/events"
-	pineconeSessions "github.com/matrix-org/pinecone/sessions"
-
-	"github.com/sirupsen/logrus"
-)
-
-var (
-	instanceName   = flag.String("name", "dendrite-p2p-pinecone", "the name of this P2P demo instance")
-	instancePort   = flag.Int("port", 8008, "the port that the client API will listen on")
-	instancePeer   = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list")
-	instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to")
-	instanceDir    = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
-)
-
-// nolint:gocyclo
-func main() {
-	flag.Parse()
-	internal.SetupPprof()
-
-	var pk ed25519.PublicKey
-	var sk ed25519.PrivateKey
-
-	// iterate through the cli args and check if the config flag was set
-	configFlagSet := false
-	for _, arg := range os.Args {
-		if arg == "--config" || arg == "-config" {
-			configFlagSet = true
-			break
-		}
-	}
-
-	cfg := &config.Dendrite{}
-
-	// use custom config if config flag is set
-	if configFlagSet {
-		cfg = setup.ParseFlags(true)
-		sk = cfg.Global.PrivateKey
-		pk = sk.Public().(ed25519.PublicKey)
-	} else {
-		keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem"
-		if _, err := os.Stat(keyfile); os.IsNotExist(err) {
-			oldkeyfile := *instanceName + ".key"
-			if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
-				if err = test.NewMatrixKey(keyfile); err != nil {
-					panic("failed to generate a new PEM key: " + err.Error())
-				}
-				if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
-					panic("failed to load PEM key: " + err.Error())
-				}
-				if len(sk) != ed25519.PrivateKeySize {
-					panic("the private key is not long enough")
-				}
-			} else {
-				if sk, err = os.ReadFile(oldkeyfile); err != nil {
-					panic("failed to read the old private key: " + err.Error())
-				}
-				if len(sk) != ed25519.PrivateKeySize {
-					panic("the private key is not long enough")
-				}
-				if err := test.SaveMatrixKey(keyfile, sk); err != nil {
-					panic("failed to convert the private key to PEM format: " + err.Error())
-				}
-			}
-		} else {
-			var err error
-			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
-				panic("failed to load PEM key: " + err.Error())
-			}
-			if len(sk) != ed25519.PrivateKeySize {
-				panic("the private key is not long enough")
-			}
-		}
-
-		pk = sk.Public().(ed25519.PublicKey)
-
-		cfg.Defaults(config.DefaultOpts{
-			Generate:   true,
-			Monolithic: true,
-		})
-		cfg.Global.PrivateKey = sk
-		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(*instanceDir, *instanceName)))
-		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
-		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.ClientAPI.RegistrationDisabled = false
-		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-		cfg.MediaAPI.BasePath = config.Path(*instanceDir)
-		cfg.SyncAPI.Fulltext.Enabled = true
-		cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir)
-		if err := cfg.Derive(); err != nil {
-			panic(err)
-		}
-	}
-
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
-	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-
-	base := base.NewBaseDendrite(cfg, "Monolith")
-	defer base.Close() // nolint: errcheck
-
-	pineconeEventChannel := make(chan pineconeEvents.Event)
-	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
-	pRouter.EnableHopLimiting()
-	pRouter.EnableWakeupBroadcasts()
-	pRouter.Subscribe(pineconeEventChannel)
-
-	pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
-	pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
-	pManager := pineconeConnections.NewConnectionManager(pRouter, nil)
-	pMulticast.Start()
-	if instancePeer != nil && *instancePeer != "" {
-		for _, peer := range strings.Split(*instancePeer, ",") {
-			pManager.AddPeer(strings.Trim(peer, " \t\r\n"))
-		}
-	}
-
-	go func() {
-		listener, err := net.Listen("tcp", *instanceListen)
-		if err != nil {
-			panic(err)
-		}
-
-		fmt.Println("Listening on", listener.Addr())
-
-		for {
-			conn, err := listener.Accept()
-			if err != nil {
-				logrus.WithError(err).Error("listener.Accept failed")
-				continue
-			}
-
-			port, err := pRouter.Connect(
-				conn,
-				pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
-			)
-			if err != nil {
-				logrus.WithError(err).Error("pSwitch.Connect failed")
-				continue
-			}
-
-			fmt.Println("Inbound connection", conn.RemoteAddr(), "is connected to port", port)
-		}
-	}()
-
-	federation := conn.CreateFederationClient(base, pQUIC)
-
-	serverKeyAPI := &signing.YggdrasilKeys{}
-	keyRing := serverKeyAPI.KeyRing()
-
-	rsComponent := roomserver.NewInternalAPI(base)
-	rsAPI := rsComponent
-	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, keyRing, true,
-	)
-
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
-	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(userAPI)
-
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-
-	rsComponent.SetFederationAPI(fsAPI, keyRing)
-
-	userProvider := users.NewPineconeUserProvider(pRouter, pQUIC, userAPI, federation)
-	roomProvider := rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation)
-
-	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    conn.CreateClient(base, pQUIC),
-		FedClient: federation,
-		KeyRing:   keyRing,
-
-		AppserviceAPI:            asAPI,
-		FederationAPI:            fsAPI,
-		RoomserverAPI:            rsAPI,
-		UserAPI:                  userAPI,
-		KeyAPI:                   keyAPI,
-		ExtPublicRoomsProvider:   roomProvider,
-		ExtUserDirectoryProvider: userProvider,
-	}
-	monolith.AddAllPublicRoutes(base)
-
-	wsUpgrader := websocket.Upgrader{
-		CheckOrigin: func(_ *http.Request) bool {
-			return true
-		},
-	}
-	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-	httpRouter.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
-		c, err := wsUpgrader.Upgrade(w, r, nil)
-		if err != nil {
-			logrus.WithError(err).Error("Failed to upgrade WebSocket connection")
-			return
-		}
-		conn := conn.WrapWebSocketConn(c)
-		if _, err = pRouter.Connect(
-			conn,
-			pineconeRouter.ConnectionZone("websocket"),
-			pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
-		); err != nil {
-			logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
-		}
-	})
-	httpRouter.HandleFunc("/pinecone", pRouter.ManholeHandler)
-	embed.Embed(httpRouter, *instancePort, "Pinecone Demo")
-
-	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
-	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
-	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-
-	pHTTP := pQUIC.Protocol("matrix").HTTP()
-	pHTTP.Mux().Handle(users.PublicURL, pMux)
-	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
-	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
-
-	// Build both ends of a HTTP multiplex.
-	httpServer := &http.Server{
-		Addr:         ":0",
-		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
-		ReadTimeout:  10 * time.Second,
-		WriteTimeout: 10 * time.Second,
-		IdleTimeout:  60 * time.Second,
-		BaseContext: func(_ net.Listener) context.Context {
-			return context.Background()
-		},
-		Handler: pMux,
-	}
-
-	go func() {
-		pubkey := pRouter.PublicKey()
-		logrus.Info("Listening on ", hex.EncodeToString(pubkey[:]))
-		logrus.Fatal(httpServer.Serve(pQUIC.Protocol("matrix")))
-	}()
-	go func() {
-		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
-		logrus.Info("Listening on ", httpBindAddr)
-		logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
-	}()
-
-	go func(ch <-chan pineconeEvents.Event) {
-		eLog := logrus.WithField("pinecone", "events")
-
-		for event := range ch {
-			switch e := event.(type) {
-			case pineconeEvents.PeerAdded:
-			case pineconeEvents.PeerRemoved:
-			case pineconeEvents.TreeParentUpdate:
-			case pineconeEvents.SnakeDescUpdate:
-			case pineconeEvents.TreeRootAnnUpdate:
-			case pineconeEvents.SnakeEntryAdded:
-			case pineconeEvents.SnakeEntryRemoved:
-			case pineconeEvents.BroadcastReceived:
-				eLog.Info("Broadcast received from: ", e.PeerID)
-
-				req := &api.PerformWakeupServersRequest{
-					ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
-				}
-				res := &api.PerformWakeupServersResponse{}
-				if err := fsAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
-					logrus.WithError(err).Error("Failed to wakeup destination", e.PeerID)
-				}
-			case pineconeEvents.BandwidthReport:
-			default:
-			}
-		}
-	}(pineconeEventChannel)
-
-	base.WaitForShutdown()
-}

cmd/dendrite-demo-pinecone/rooms/rooms.go

@@ -1,123 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rooms
-
-import (
-	"context"
-	"sync"
-	"time"
-
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults"
-	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-
-	pineconeRouter "github.com/matrix-org/pinecone/router"
-	pineconeSessions "github.com/matrix-org/pinecone/sessions"
-)
-
-type PineconeRoomProvider struct {
-	r         *pineconeRouter.Router
-	s         *pineconeSessions.Sessions
-	fedSender api.FederationInternalAPI
-	fedClient *gomatrixserverlib.FederationClient
-}
-
-func NewPineconeRoomProvider(
-	r *pineconeRouter.Router,
-	s *pineconeSessions.Sessions,
-	fedSender api.FederationInternalAPI,
-	fedClient *gomatrixserverlib.FederationClient,
-) *PineconeRoomProvider {
-	p := &PineconeRoomProvider{
-		r:         r,
-		s:         s,
-		fedSender: fedSender,
-		fedClient: fedClient,
-	}
-	return p
-}
-
-func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
-	list := map[gomatrixserverlib.ServerName]struct{}{}
-	for k := range defaults.DefaultServerNames {
-		list[k] = struct{}{}
-	}
-	for _, k := range p.r.Peers() {
-		list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
-	}
-	return bulkFetchPublicRoomsFromServers(
-		context.Background(), p.fedClient,
-		gomatrixserverlib.ServerName(p.r.PublicKey().String()), list,
-	)
-}
-
-// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
-// Returns a list of public rooms.
-func bulkFetchPublicRoomsFromServers(
-	ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
-	origin gomatrixserverlib.ServerName,
-	homeservers map[gomatrixserverlib.ServerName]struct{},
-) (publicRooms []gomatrixserverlib.PublicRoom) {
-	limit := 200
-	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
-	// goroutines send rooms to this channel
-	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
-	// signalling channel to tell goroutines to stop sending rooms and quit
-	done := make(chan bool)
-	// signalling to say when we can close the room channel
-	var wg sync.WaitGroup
-	wg.Add(len(homeservers))
-	// concurrently query for public rooms
-	reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5)
-	for hs := range homeservers {
-		go func(homeserverDomain gomatrixserverlib.ServerName) {
-			defer wg.Done()
-			util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
-			fres, err := fedClient.GetPublicRooms(reqctx, origin, homeserverDomain, int(limit), "", false, "")
-			if err != nil {
-				util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn(
-					"bulkFetchPublicRoomsFromServers: failed to query hs",
-				)
-				return
-			}
-			for _, room := range fres.Chunk {
-				// atomically send a room or stop
-				select {
-				case roomCh <- room:
-				case <-done:
-				case <-reqctx.Done():
-					util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms")
-					return
-				}
-			}
-		}(hs)
-	}
-
-	select {
-	case <-time.After(5 * time.Second):
-	default:
-		wg.Wait()
-	}
-	reqcancel()
-	close(done)
-	close(roomCh)
-
-	for room := range roomCh {
-		publicRooms = append(publicRooms, room)
-	}
-
-	return publicRooms
-}

cmd/dendrite-demo-pinecone/users/users.go

@@ -1,163 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package users
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-
-	pineconeRouter "github.com/matrix-org/pinecone/router"
-	pineconeSessions "github.com/matrix-org/pinecone/sessions"
-)
-
-type PineconeUserProvider struct {
-	r         *pineconeRouter.Router
-	s         *pineconeSessions.Sessions
-	userAPI   userapi.QuerySearchProfilesAPI
-	fedClient *gomatrixserverlib.FederationClient
-}
-
-const PublicURL = "/_matrix/p2p/profiles"
-
-func NewPineconeUserProvider(
-	r *pineconeRouter.Router,
-	s *pineconeSessions.Sessions,
-	userAPI userapi.QuerySearchProfilesAPI,
-	fedClient *gomatrixserverlib.FederationClient,
-) *PineconeUserProvider {
-	p := &PineconeUserProvider{
-		r:         r,
-		s:         s,
-		userAPI:   userAPI,
-		fedClient: fedClient,
-	}
-	return p
-}
-
-func (p *PineconeUserProvider) FederatedUserProfiles(w http.ResponseWriter, r *http.Request) {
-	req := &userapi.QuerySearchProfilesRequest{Limit: 25}
-	res := &userapi.QuerySearchProfilesResponse{}
-	if err := clienthttputil.UnmarshalJSONRequest(r, &req); err != nil {
-		w.WriteHeader(400)
-		return
-	}
-	if err := p.userAPI.QuerySearchProfiles(r.Context(), req, res); err != nil {
-		w.WriteHeader(400)
-		return
-	}
-	j, err := json.Marshal(res)
-	if err != nil {
-		w.WriteHeader(400)
-		return
-	}
-	w.WriteHeader(200)
-	_, _ = w.Write(j)
-}
-
-func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error {
-	list := map[gomatrixserverlib.ServerName]struct{}{}
-	for k := range defaults.DefaultServerNames {
-		list[k] = struct{}{}
-	}
-	for _, k := range p.r.Peers() {
-		list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
-	}
-	res.Profiles = bulkFetchUserDirectoriesFromServers(context.Background(), req, p.fedClient, list)
-	return nil
-}
-
-// bulkFetchUserDirectoriesFromServers fetches users from the list of homeservers.
-// Returns a list of user profiles.
-func bulkFetchUserDirectoriesFromServers(
-	ctx context.Context, req *userapi.QuerySearchProfilesRequest,
-	fedClient *gomatrixserverlib.FederationClient,
-	homeservers map[gomatrixserverlib.ServerName]struct{},
-) (profiles []authtypes.Profile) {
-	jsonBody, err := json.Marshal(req)
-	if err != nil {
-		return nil
-	}
-
-	limit := 200
-	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
-	// goroutines send rooms to this channel
-	profileCh := make(chan authtypes.Profile, int(limit))
-	// signalling channel to tell goroutines to stop sending rooms and quit
-	done := make(chan bool)
-	// signalling to say when we can close the room channel
-	var wg sync.WaitGroup
-	wg.Add(len(homeservers))
-	// concurrently query for public rooms
-	reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5)
-	for hs := range homeservers {
-		go func(homeserverDomain gomatrixserverlib.ServerName) {
-			defer wg.Done()
-			util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for users")
-
-			jsonBodyReader := bytes.NewBuffer(jsonBody)
-			httpReq, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("matrix://%s%s", homeserverDomain, PublicURL), jsonBodyReader)
-			if err != nil {
-				util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn(
-					"bulkFetchUserDirectoriesFromServers: failed to create request",
-				)
-			}
-			res := &userapi.QuerySearchProfilesResponse{}
-			if err = fedClient.DoRequestAndParseResponse(reqctx, httpReq, res); err != nil {
-				util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn(
-					"bulkFetchUserDirectoriesFromServers: failed to query hs",
-				)
-				return
-			}
-			for _, profile := range res.Profiles {
-				profile.ServerName = string(homeserverDomain)
-				// atomically send a room or stop
-				select {
-				case profileCh <- profile:
-				case <-done:
-				case <-reqctx.Done():
-					util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending profiles")
-					return
-				}
-			}
-		}(hs)
-	}
-
-	select {
-	case <-time.After(5 * time.Second):
-	default:
-		wg.Wait()
-	}
-	reqcancel()
-	close(done)
-	close(profileCh)
-
-	for profile := range profileCh {
-		profiles = append(profiles, profile)
-	}
-
-	return profiles
-}

cmd/dendrite-demo-yggdrasil/main.go

@@ -39,7 +39,6 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
@@ -118,7 +117,7 @@ func main() {
 	} else {
 		cfg.Defaults(config.DefaultOpts{
 			Generate:       true,
-			Monolithic: true,
+			SingleDatabase: true,
 		})
 		cfg.Global.PrivateKey = sk
 		cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(*instanceDir, *instanceName))
@@ -143,7 +142,8 @@ func main() {
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
-	base := base.NewBaseDendrite(cfg, "Monolith")
+	base := base.NewBaseDendrite(cfg)
+	base.ConfigureAdminEndpoints()
 	defer base.Close() // nolint: errcheck
 
 	ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
@@ -156,15 +156,11 @@ func main() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
+	rsAPI := roomserver.NewInternalAPI(
-
-	rsComponent := roomserver.NewInternalAPI(
 		base,
 	)
-	rsAPI := rsComponent
 
-	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
+	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
-	keyAPI.SetUserAPI(userAPI)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
@@ -172,7 +168,7 @@ func main() {
 		base, federation, rsAPI, base.Caches, keyRing, true,
 	)
 
-	rsComponent.SetFederationAPI(fsAPI, keyRing)
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
@@ -184,7 +180,6 @@ func main() {
 		FederationAPI: fsAPI,
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
-		KeyAPI:        keyAPI,
 		ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider(
 			ygg, fsAPI, federation,
 		),
@@ -195,9 +190,10 @@ func main() {
 	}
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 	embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
 
 	yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()

cmd/dendrite-monolith-server/main.go

@@ -1,188 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"log"
-	"os"
-
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/mscs"
-	"github.com/matrix-org/dendrite/userapi"
-	uapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/sirupsen/logrus"
-)
-
-var (
-	httpBindAddr   = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
-	httpsBindAddr  = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
-	apiBindAddr    = flag.String("api-bind-address", "localhost:18008", "The HTTP listening port for the internal HTTP APIs (if -api is enabled)")
-	certFile       = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
-	keyFile        = flag.String("tls-key", "", "The PEM private key to use for TLS")
-	enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)")
-	traceInternal  = os.Getenv("DENDRITE_TRACE_INTERNAL") == "1"
-)
-
-func main() {
-	cfg := setup.ParseFlags(true)
-	httpAddr := config.HTTPAddress("http://" + *httpBindAddr)
-	for _, logging := range cfg.Logging {
-		if logging.Type == "std" {
-			level, err := logrus.ParseLevel(logging.Level)
-			if err != nil {
-				log.Fatal(err)
-			}
-			logrus.SetLevel(level)
-			logrus.SetFormatter(&logrus.JSONFormatter{})
-		}
-	}
-	httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr)
-	httpAPIAddr := httpAddr
-	options := []basepkg.BaseDendriteOptions{}
-	if *enableHTTPAPIs {
-		logrus.Warnf("DANGER! The -api option is enabled, exposing internal APIs on %q!", *apiBindAddr)
-		httpAPIAddr = config.HTTPAddress("http://" + *apiBindAddr)
-		// If the HTTP APIs are enabled then we need to update the Listen
-		// statements in the configuration so that we know where to find
-		// the API endpoints. They'll listen on the same port as the monolith
-		// itself.
-		cfg.AppServiceAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.KeyServer.InternalAPI.Connect = httpAPIAddr
-		cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.RoomServer.InternalAPI.Connect = httpAPIAddr
-		cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.UserAPI.InternalAPI.Connect = httpAPIAddr
-		options = append(options, basepkg.UseHTTPAPIs)
-	}
-
-	base := basepkg.NewBaseDendrite(cfg, "Monolith", options...)
-	defer base.Close() // nolint: errcheck
-
-	federation := base.CreateFederationClient()
-
-	rsImpl := roomserver.NewInternalAPI(base)
-	// call functions directly on the impl unless running in HTTP mode
-	rsAPI := rsImpl
-	if base.UseHTTPAPIs {
-		roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl)
-		rsAPI = base.RoomserverHTTPClient()
-	}
-	if traceInternal {
-		rsAPI = &api.RoomserverInternalAPITrace{
-			Impl: rsAPI,
-		}
-	}
-
-	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, nil, false,
-	)
-	fsImplAPI := fsAPI
-	if base.UseHTTPAPIs {
-		federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
-		fsAPI = base.FederationAPIHTTPClient()
-	}
-	keyRing := fsAPI.KeyRing()
-
-	keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
-	keyAPI := keyImpl
-	if base.UseHTTPAPIs {
-		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
-		keyAPI = base.KeyServerHTTPClient()
-	}
-
-	pgClient := base.PushGatewayHTTPClient()
-	userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
-	userAPI := userImpl
-	if base.UseHTTPAPIs {
-		userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
-		userAPI = base.UserAPIClient()
-	}
-	if traceInternal {
-		userAPI = &uapi.UserInternalAPITrace{
-			Impl: userAPI,
-		}
-	}
-
-	// TODO: This should use userAPI, not userImpl, but the appservice setup races with
-	// the listeners and panics at startup if it tries to create appservice accounts
-	// before the listeners are up.
-	asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI)
-	if base.UseHTTPAPIs {
-		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
-		asAPI = base.AppserviceHTTPClient()
-	}
-
-	// The underlying roomserver implementation needs to be able to call the fedsender.
-	// This is different to rsAPI which can be the http client which doesn't need this
-	// dependency. Other components also need updating after their dependencies are up.
-	rsImpl.SetFederationAPI(fsAPI, keyRing)
-	rsImpl.SetAppserviceAPI(asAPI)
-	rsImpl.SetUserAPI(userAPI)
-	keyImpl.SetUserAPI(userAPI)
-
-	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    base.CreateClient(),
-		FedClient: federation,
-		KeyRing:   keyRing,
-
-		AppserviceAPI: asAPI,
-		// always use the concrete impl here even in -http mode because adding public routes
-		// must be done on the concrete impl not an HTTP client else fedapi will call itself
-		FederationAPI: fsImplAPI,
-		RoomserverAPI: rsAPI,
-		UserAPI:       userAPI,
-		KeyAPI:        keyAPI,
-	}
-	monolith.AddAllPublicRoutes(base)
-
-	if len(base.Cfg.MSCs.MSCs) > 0 {
-		if err := mscs.Enable(base, &monolith); err != nil {
-			logrus.WithError(err).Fatalf("Failed to enable MSCs")
-		}
-	}
-
-	// Expose the matrix APIs directly rather than putting them under a /api path.
-	go func() {
-		base.SetupAndServeHTTP(
-			httpAPIAddr, // internal API
-			httpAddr,    // external API
-			nil, nil,    // TLS settings
-		)
-	}()
-	// Handle HTTPS if certificate and key are provided
-	if *certFile != "" && *keyFile != "" {
-		go func() {
-			base.SetupAndServeHTTP(
-				basepkg.NoListener, // internal API
-				httpsAddr,          // external API
-				certFile, keyFile,  // TLS settings
-			)
-		}()
-	}
-
-	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
-	base.WaitForShutdown()
-}

cmd/dendrite-polylith-multi/main.go

@@ -1,77 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"os"
-	"strings"
-
-	"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/sirupsen/logrus"
-)
-
-type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite)
-
-func main() {
-	cfg := setup.ParseFlags(false)
-
-	component := ""
-	if flag.NFlag() > 0 {
-		component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi
-	} else if len(os.Args) > 1 {
-		component = os.Args[1] // ./dendrite-polylith-multi clientapi
-	}
-
-	components := map[string]entrypoint{
-		"appservice":    personalities.Appservice,
-		"clientapi":     personalities.ClientAPI,
-		"federationapi": personalities.FederationAPI,
-		"keyserver":     personalities.KeyServer,
-		"mediaapi":      personalities.MediaAPI,
-		"roomserver":    personalities.RoomServer,
-		"syncapi":       personalities.SyncAPI,
-		"userapi":       personalities.UserAPI,
-	}
-
-	start, ok := components[component]
-	if !ok {
-		if component == "" {
-			logrus.Errorf("No component specified")
-			logrus.Info("The first argument on the command line must be the name of the component to run")
-		} else {
-			logrus.Errorf("Unknown component %q specified", component)
-		}
-
-		var list []string
-		for c := range components {
-			list = append(list, c)
-		}
-		logrus.Infof("Valid components: %s", strings.Join(list, ", "))
-
-		os.Exit(1)
-	}
-
-	logrus.Infof("Starting %q component", component)
-
-	base := base.NewBaseDendrite(cfg, component, base.PolylithMode) // TODO
-	defer base.Close()                                              // nolint: errcheck
-
-	go start(base, cfg)
-	base.WaitForShutdown()
-}

cmd/dendrite-polylith-multi/personalities/appservice.go

@@ -1,36 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/setup/base"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) {
-	userAPI := base.UserAPIClient()
-	rsAPI := base.RoomserverHTTPClient()
-
-	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener
-		basepkg.NoListener,                        // external listener
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/clientapi.go

@@ -1,44 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/clientapi"
-	"github.com/matrix-org/dendrite/internal/transactions"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	federation := base.CreateFederationClient()
-
-	asQuery := base.AppserviceHTTPClient()
-	rsAPI := base.RoomserverHTTPClient()
-	fsAPI := base.FederationAPIHTTPClient()
-	userAPI := base.UserAPIClient()
-	keyAPI := base.KeyServerHTTPClient()
-
-	clientapi.AddPublicRoutes(
-		base, federation, rsAPI, asQuery,
-		transactions.New(), fsAPI, userAPI, userAPI,
-		keyAPI, nil,
-	)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.ClientAPI.InternalAPI.Listen,
-		base.Cfg.ClientAPI.ExternalAPI.Listen,
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -1,44 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/federationapi"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	userAPI := base.UserAPIClient()
-	federation := base.CreateFederationClient()
-	rsAPI := base.RoomserverHTTPClient()
-	keyAPI := base.KeyServerHTTPClient()
-	fsAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, nil, true)
-	keyRing := fsAPI.KeyRing()
-
-	federationapi.AddPublicRoutes(
-		base,
-		userAPI, federation, keyRing,
-		rsAPI, fsAPI, keyAPI, nil,
-	)
-
-	federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.FederationAPI.InternalAPI.Listen,
-		base.Cfg.FederationAPI.ExternalAPI.Listen,
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -1,35 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/keyserver"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	fsAPI := base.FederationAPIHTTPClient()
-	intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
-	intAPI.SetUserAPI(base.UserAPIClient())
-
-	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.KeyServer.InternalAPI.Listen, // internal listener
-		basepkg.NoListener,                    // external listener
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/mediaapi.go

@@ -1,36 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/mediaapi"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func MediaAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	userAPI := base.UserAPIClient()
-	client := base.CreateClient()
-
-	mediaapi.AddPublicRoutes(
-		base, userAPI, client,
-	)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.MediaAPI.InternalAPI.Listen,
-		base.Cfg.MediaAPI.ExternalAPI.Listen,
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -1,36 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/roomserver"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	asAPI := base.AppserviceHTTPClient()
-	fsAPI := base.FederationAPIHTTPClient()
-	rsAPI := roomserver.NewInternalAPI(base)
-	rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing())
-	rsAPI.SetAppserviceAPI(asAPI)
-	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.RoomServer.InternalAPI.Listen, // internal listener
-		basepkg.NoListener,                     // external listener
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/syncapi.go

@@ -1,39 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/syncapi"
-)
-
-func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	userAPI := base.UserAPIClient()
-
-	rsAPI := base.RoomserverHTTPClient()
-
-	syncapi.AddPublicRoutes(
-		base,
-		userAPI, rsAPI,
-		base.KeyServerHTTPClient(),
-	)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.SyncAPI.InternalAPI.Listen,
-		base.Cfg.SyncAPI.ExternalAPI.Listen,
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/userapi.go

@@ -1,37 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi"
-)
-
-func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	userAPI := userapi.NewInternalAPI(
-		base, &cfg.UserAPI, cfg.Derived.ApplicationServices,
-		base.KeyServerHTTPClient(), base.RoomserverHTTPClient(),
-		base.PushGatewayHTTPClient(),
-	)
-
-	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.UserAPI.InternalAPI.Listen, // internal listener
-		basepkg.NoListener,                  // external listener
-		nil, nil,
-	)
-}

cmd/dendrite-upgrade-tests/main.go

@@ -7,6 +7,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -44,6 +45,10 @@ var (
 
 const HEAD = "HEAD"
 
+// The binary was renamed after v0.11.1, so everything after that should use the new name
+var binaryChangeVersion, _ = semver.NewVersion("v0.11.1")
+var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD"
+
 // Embed the Dockerfile to use when building dendrite versions.
 // We cannot use the dockerfile associated with the repo with each version sadly due to changes in
 // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
@@ -53,14 +58,16 @@ const HEAD = "HEAD"
 const DockerfilePostgreSQL = `FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
+ARG BINARY
 
 # Copy the build context to the repo as this is the right dendrite code. This is different to the
 # Complement Dockerfile which wgets a branch.
 COPY . .
 
-RUN go build ./cmd/dendrite-monolith-server
+RUN go build ./cmd/${BINARY}
 RUN go build ./cmd/generate-keys
 RUN go build ./cmd/generate-config
+RUN go build ./cmd/create-account
 RUN ./generate-config --ci > dendrite.yaml
 RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
 
@@ -86,24 +93,27 @@ done \n\
  \n\
 sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
 PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\
-./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\
+./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\
 ' > run_dendrite.sh && chmod +x run_dendrite.sh
 
 ENV SERVER_NAME=localhost
+ENV BINARY=dendrite
 EXPOSE 8008 8448
 CMD /build/run_dendrite.sh`
 
 const DockerfileSQLite = `FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
+ARG BINARY
 
 # Copy the build context to the repo as this is the right dendrite code. This is different to the
 # Complement Dockerfile which wgets a branch.
 COPY . .
 
-RUN go build ./cmd/dendrite-monolith-server
+RUN go build ./cmd/${BINARY}
 RUN go build ./cmd/generate-keys
 RUN go build ./cmd/generate-config
+RUN go build ./cmd/create-account
 RUN ./generate-config --ci > dendrite.yaml
 RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
 
@@ -115,10 +125,11 @@ RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postg
 RUN echo '\
 sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
 PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\
-./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\
+./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\
 ' > run_dendrite.sh && chmod +x run_dendrite.sh
 
 ENV SERVER_NAME=localhost
+ENV BINARY=dendrite
 EXPOSE 8008 8448
 CMD /build/run_dendrite.sh `
 
@@ -179,7 +190,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
 }
 
 // buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
-func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) {
+func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir string, branchOrTagName, binary string) (string, error) {
 	var tarball *bytes.Buffer
 	var err error
 	// If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI
@@ -213,6 +224,9 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
 	log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName)
 	res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
 		Tags: []string{"dendrite-upgrade"},
+		BuildArgs: map[string]*string{
+			"BINARY": &binary,
+		},
 	})
 	if err != nil {
 		return "", fmt.Errorf("failed to start building image: %s", err)
@@ -269,7 +283,7 @@ func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Ve
 	return semVers, nil
 }
 
-func calculateVersions(cli *http.Client, from, to string, direct bool) []string {
+func calculateVersions(cli *http.Client, from, to string, direct bool) []*semver.Version {
 	semvers, err := getAndSortVersionsFromGithub(cli)
 	if err != nil {
 		log.Fatalf("failed to collect semvers from github: %s", err)
@@ -317,28 +331,25 @@ func calculateVersions(cli *http.Client, from, to string, direct bool) []string
 		}
 		semvers = semvers[:i+1]
 	}
-	var versions []string
+
-	for _, sv := range semvers {
-		versions = append(versions, sv.Original())
-	}
 	if to == HEAD {
-		versions = append(versions, HEAD)
+		semvers = append(semvers, latest)
 	}
 	if direct {
-		versions = []string{versions[0], versions[len(versions)-1]}
+		semvers = []*semver.Version{semvers[0], semvers[len(semvers)-1]}
 	}
-	return versions
+	return semvers
 }
 
-func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, branchOrTagNames []string) map[string]string {
+func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, versions []*semver.Version) map[string]string {
 	// concurrently build all versions, this can be done in any order. The mutex protects the map
 	branchToImageID := make(map[string]string)
 	var mu sync.Mutex
 
 	var wg sync.WaitGroup
 	wg.Add(concurrency)
-	ch := make(chan string, len(branchOrTagNames))
+	ch := make(chan *semver.Version, len(versions))
-	for _, branchName := range branchOrTagNames {
+	for _, branchName := range versions {
 		ch <- branchName
 	}
 	close(ch)
@@ -346,11 +357,13 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b
 	for i := 0; i < concurrency; i++ {
 		go func() {
 			defer wg.Done()
-			for branchName := range ch {
+			for version := range ch {
+				branchName, binary := versionToBranchAndBinary(version)
+				log.Printf("Building version %s with binary %s", branchName, binary)
 				tmpDir := baseTempDir + alphaNumerics.ReplaceAllString(branchName, "")
-				imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName)
+				imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName, binary)
 				if err != nil {
-					log.Fatalf("%s: failed to build dendrite image: %s", branchName, err)
+					log.Fatalf("%s: failed to build dendrite image: %s", version, err)
 				}
 				mu.Lock()
 				branchToImageID[branchName] = imgID
@@ -362,13 +375,14 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b
 	return branchToImageID
 }
 
-func runImage(dockerClient *client.Client, volumeName, version, imageID string) (csAPIURL, containerID string, err error) {
+func runImage(dockerClient *client.Client, volumeName string, branchNameToImageID map[string]string, version *semver.Version) (csAPIURL, containerID string, err error) {
-	log.Printf("%s: running image %s\n", version, imageID)
+	branchName, binary := versionToBranchAndBinary(version)
+	imageID := branchNameToImageID[branchName]
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancel()
 	body, err := dockerClient.ContainerCreate(ctx, &container.Config{
 		Image: imageID,
-		Env:   []string{"SERVER_NAME=hs1"},
+		Env:   []string{"SERVER_NAME=hs1", fmt.Sprintf("BINARY=%s", binary)},
 		Labels: map[string]string{
 			dendriteUpgradeTestLabel: "yes",
 		},
@@ -381,7 +395,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
 				Target: "/var/lib/postgresql/9.6/main",
 			},
 		},
-	}, nil, nil, "dendrite_upgrade_test_"+version)
+	}, nil, nil, "dendrite_upgrade_test_"+branchName)
 	if err != nil {
 		return "", "", fmt.Errorf("failed to ContainerCreate: %s", err)
 	}
@@ -448,8 +462,8 @@ func destroyContainer(dockerClient *client.Client, containerID string) {
 	}
 }
 
-func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchToImageID map[string]string) error {
+func loadAndRunTests(dockerClient *client.Client, volumeName string, v *semver.Version, branchToImageID map[string]string) error {
-	csAPIURL, containerID, err := runImage(dockerClient, volumeName, v, branchToImageID[v])
+	csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, v)
 	if err != nil {
 		return fmt.Errorf("failed to run container for branch %v: %v", v, err)
 	}
@@ -458,12 +472,65 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo
 	if err = runTests(csAPIURL, v); err != nil {
 		return fmt.Errorf("failed to run tests on version %s: %s", v, err)
 	}
+
+	err = testCreateAccount(dockerClient, v, containerID)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
-func verifyTests(dockerClient *client.Client, volumeName string, versions []string, branchToImageID map[string]string) error {
+// test that create-account is working
+func testCreateAccount(dockerClient *client.Client, version *semver.Version, containerID string) error {
+	branchName, _ := versionToBranchAndBinary(version)
+	createUser := strings.ToLower("createaccountuser-" + branchName)
+	log.Printf("%s: Creating account %s with create-account\n", branchName, createUser)
+
+	respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{
+		AttachStderr: true,
+		AttachStdout: true,
+		Cmd: []string{
+			"/build/create-account",
+			"-username", createUser,
+			"-password", "someRandomPassword",
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to ContainerExecCreate: %w", err)
+	}
+
+	response, err := dockerClient.ContainerExecAttach(context.Background(), respID.ID, types.ExecStartCheck{})
+	if err != nil {
+		return fmt.Errorf("failed to attach to container: %w", err)
+	}
+	defer response.Close()
+
+	data, err := ioutil.ReadAll(response.Reader)
+	if err != nil {
+		return err
+	}
+
+	if !bytes.Contains(data, []byte("AccessToken")) {
+		return fmt.Errorf("failed to create-account: %s", string(data))
+	}
+	return nil
+}
+
+func versionToBranchAndBinary(version *semver.Version) (branchName, binary string) {
+	binary = "dendrite-monolith-server"
+	branchName = version.Original()
+	if version.GreaterThan(binaryChangeVersion) {
+		binary = "dendrite"
+		if version.Equal(latest) {
+			branchName = HEAD
+		}
+	}
+	return
+}
+
+func verifyTests(dockerClient *client.Client, volumeName string, versions []*semver.Version, branchToImageID map[string]string) error {
 	lastVer := versions[len(versions)-1]
-	csAPIURL, containerID, err := runImage(dockerClient, volumeName, lastVer, branchToImageID[lastVer])
+	csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, lastVer)
 	if err != nil {
 		return fmt.Errorf("failed to run container for branch %v: %v", lastVer, err)
 	}

cmd/dendrite-upgrade-tests/tests.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -22,7 +23,8 @@ type user struct {
 // - register alice and bob with branch name muxed into the localpart
 // - create a DM room for the 2 users and exchange messages
 // - create/join a public #global room and exchange messages
-func runTests(baseURL, branchName string) error {
+func runTests(baseURL string, v *semver.Version) error {
+	branchName, _ := versionToBranchAndBinary(v)
 	// register 2 users
 	users := []user{
 		{
@@ -164,15 +166,16 @@ func runTests(baseURL, branchName string) error {
 }
 
 // verifyTestsRan checks that the HS has the right rooms/messages
-func verifyTestsRan(baseURL string, branchNames []string) error {
+func verifyTestsRan(baseURL string, versions []*semver.Version) error {
 	log.Println("Verifying tests....")
 	// check we can login as all users
 	var resp *gomatrix.RespLogin
-	for _, branchName := range branchNames {
+	for _, version := range versions {
 		client, err := gomatrix.NewClient(baseURL, "", "")
 		if err != nil {
 			return err
 		}
+		branchName, _ := versionToBranchAndBinary(version)
 		userLocalparts := []string{
 			"alice" + branchName,
 			"bob" + branchName,
@@ -224,7 +227,7 @@ func verifyTestsRan(baseURL string, branchNames []string) error {
 			msgCount += 1
 		}
 	}
-	wantMsgCount := len(branchNames) * 4
+	wantMsgCount := len(versions) * 4
 	if msgCount != wantMsgCount {
 		return fmt.Errorf("got %d messages in global room, want %d", msgCount, wantMsgCount)
 	}

cmd/dendrite/Dockerfile.dev

@@ -5,4 +5,4 @@ COPY dendrite-monolith-server /usr/bin/
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
 
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
+ENTRYPOINT ["/usr/bin/dendrite"]

cmd/dendrite/main.go

@@ -0,0 +1,125 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"io/fs"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/mscs"
+	"github.com/matrix-org/dendrite/userapi"
+)
+
+var (
+	unixSocket = flag.String("unix-socket", "",
+		"EXPERIMENTAL(unstable): The HTTP listening unix socket for the server (disables http[s]-bind-address feature)",
+	)
+	unixSocketPermission = flag.Int("unix-socket-permission", 0755,
+		"EXPERIMENTAL(unstable): The HTTP listening unix socket permission for the server",
+	)
+	httpBindAddr  = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
+	httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
+	certFile      = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
+	keyFile       = flag.String("tls-key", "", "The PEM private key to use for TLS")
+)
+
+func main() {
+	cfg := setup.ParseFlags(true)
+	httpAddr := config.ServerAddress{}
+	httpsAddr := config.ServerAddress{}
+	if *unixSocket == "" {
+		http, err := config.HTTPAddress("http://" + *httpBindAddr)
+		if err != nil {
+			logrus.WithError(err).Fatalf("Failed to parse http address")
+		}
+		httpAddr = http
+		https, err := config.HTTPAddress("https://" + *httpsBindAddr)
+		if err != nil {
+			logrus.WithError(err).Fatalf("Failed to parse https address")
+		}
+		httpsAddr = https
+	} else {
+		httpAddr = config.UnixSocketAddress(*unixSocket, fs.FileMode(*unixSocketPermission))
+	}
+
+	options := []basepkg.BaseDendriteOptions{}
+
+	base := basepkg.NewBaseDendrite(cfg, options...)
+	defer base.Close() // nolint: errcheck
+
+	federation := base.CreateFederationClient()
+
+	rsAPI := roomserver.NewInternalAPI(base)
+
+	fsAPI := federationapi.NewInternalAPI(
+		base, federation, rsAPI, base.Caches, nil, false,
+	)
+
+	keyRing := fsAPI.KeyRing()
+
+	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
+
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this
+	// dependency. Other components also need updating after their dependencies are up.
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
+	rsAPI.SetAppserviceAPI(asAPI)
+	rsAPI.SetUserAPI(userAPI)
+
+	monolith := setup.Monolith{
+		Config:    base.Cfg,
+		Client:    base.CreateClient(),
+		FedClient: federation,
+		KeyRing:   keyRing,
+
+		AppserviceAPI: asAPI,
+		// always use the concrete impl here even in -http mode because adding public routes
+		// must be done on the concrete impl not an HTTP client else fedapi will call itself
+		FederationAPI: fsAPI,
+		RoomserverAPI: rsAPI,
+		UserAPI:       userAPI,
+	}
+	monolith.AddAllPublicRoutes(base)
+
+	if len(base.Cfg.MSCs.MSCs) > 0 {
+		if err := mscs.Enable(base, &monolith); err != nil {
+			logrus.WithError(err).Fatalf("Failed to enable MSCs")
+		}
+	}
+
+	// Expose the matrix APIs directly rather than putting them under a /api path.
+	go func() {
+		base.SetupAndServeHTTP(httpAddr, nil, nil)
+	}()
+	// Handle HTTPS if certificate and key are provided
+	if *unixSocket == "" && *certFile != "" && *keyFile != "" {
+		go func() {
+			base.SetupAndServeHTTP(httpsAddr, certFile, keyFile)
+		}()
+	}
+
+	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
+	base.WaitForShutdown()
+}

cmd/dendrite/main_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 // This is an instrumented main, used when running integration tests (sytest) with code coverage.
-// Compile: go test -c -race -cover -covermode=atomic -o monolith.debug -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server
+// Compile: go test -c -race -cover -covermode=atomic -o monolith.debug -coverpkg "github.com/matrix-org/..." ./cmd/dendrite
 // Run the monolith: ./monolith.debug -test.coverprofile=/somewhere/to/dump/integrationcover.out DEVEL --config dendrite.yaml
 // Generate HTML with coverage: go tool cover -html=/somewhere/where/there/is/integrationcover.out -o cover.html
 // Source: https://dzone.com/articles/measuring-integration-test-coverage-rate-in-pouchc

cmd/generate-config/main.go

@@ -18,7 +18,6 @@ func main() {
 	dbURI := flag.String("db", "", "The DB URI to use for all components (PostgreSQL only)")
 	dirPath := flag.String("dir", "./", "The folder to use for paths (like SQLite databases, media storage)")
 	normalise := flag.String("normalise", "", "Normalise an existing configuration file by adding new/missing options and defaults")
-	polylith := flag.Bool("polylith", false, "Generate a config that makes sense for polylith deployments")
 	flag.Parse()
 
 	var cfg *config.Dendrite
@@ -28,13 +27,13 @@ func main() {
 		}
 		cfg.Defaults(config.DefaultOpts{
 			Generate:       true,
-			Monolithic: !*polylith,
+			SingleDatabase: true,
 		})
 		if *serverName != "" {
 			cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
 		}
 		uri := config.DataSource(*dbURI)
-		if *polylith || uri.IsSQLite() || uri == "" {
+		if uri.IsSQLite() || uri == "" {
 			for name, db := range map[string]*config.DatabaseOptions{
 				"federationapi": &cfg.FederationAPI.Database,
 				"keyserver":     &cfg.KeyServer.Database,
@@ -54,6 +53,9 @@ func main() {
 		} else {
 			cfg.Global.DatabaseOptions.ConnectionString = uri
 		}
+		cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))
+		cfg.Global.JetStream.StoragePath = config.Path(*dirPath)
+		cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(*dirPath, "searchindex"))
 		cfg.Logging = []config.LogrusHook{
 			{
 				Type:  "file",
@@ -67,6 +69,7 @@ func main() {
 			cfg.AppServiceAPI.DisableTLSValidation = true
 			cfg.ClientAPI.RateLimiting.Enabled = false
 			cfg.FederationAPI.DisableTLSValidation = false
+			cfg.FederationAPI.DisableHTTPKeepalives = true
 			// don't hit matrix.org when running tests!!!
 			cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
 			cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))
@@ -92,7 +95,7 @@ func main() {
 		}
 	} else {
 		var err error
-		if cfg, err = config.Load(*normalise, !*polylith); err != nil {
+		if cfg, err = config.Load(*normalise); err != nil {
 			panic(err)
 		}
 	}

cmd/resolve-state/main.go

@@ -40,7 +40,7 @@ func main() {
 		Level: "error",
 	})
 	cfg.ClientAPI.RegistrationDisabled = true
-	base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics)
+	base := base.NewBaseDendrite(cfg, base.DisableMetrics)
 	args := flag.Args()
 
 	fmt.Println("Room version", *roomVersion)
@@ -62,9 +62,10 @@ func main() {
 		panic(err)
 	}
 
-	stateres := state.NewStateResolution(roomserverDB, &types.RoomInfo{
+	roomInfo := &types.RoomInfo{
 		RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion),
-	})
+	}
+	stateres := state.NewStateResolution(roomserverDB, roomInfo)
 
 	if *difference {
 		if len(snapshotNIDs) != 2 {
@@ -87,7 +88,7 @@ func main() {
 		}
 
 		var eventEntries []types.Event
-		eventEntries, err = roomserverDB.Events(ctx, eventNIDs)
+		eventEntries, err = roomserverDB.Events(ctx, roomInfo, eventNIDs)
 		if err != nil {
 			panic(err)
 		}
@@ -145,7 +146,7 @@ func main() {
 	}
 
 	fmt.Println("Fetching", len(eventNIDMap), "state events")
-	eventEntries, err := roomserverDB.Events(ctx, eventNIDs)
+	eventEntries, err := roomserverDB.Events(ctx, roomInfo, eventNIDs)
 	if err != nil {
 		panic(err)
 	}
@@ -165,7 +166,7 @@ func main() {
 	}
 
 	fmt.Println("Fetching", len(authEventIDs), "auth events")
-	authEventEntries, err := roomserverDB.EventsFromIDs(ctx, authEventIDs)
+	authEventEntries, err := roomserverDB.EventsFromIDs(ctx, roomInfo, authEventIDs)
 	if err != nil {
 		panic(err)
 	}

dendrite-sample.polylith.yaml

@@ -1,417 +0,0 @@
-# This is the Dendrite configuration file.
-#
-# The configuration is split up into sections - each Dendrite component has a
-# configuration section, in addition to the "global" section which applies to
-# all components.
-
-# The version of the configuration file.
-version: 2
-
-# Global Matrix configuration. This configuration applies to all components.
-global:
-  # The domain name of this homeserver.
-  server_name: localhost
-
-  # The path to the signing private key file, used to sign requests and events.
-  # Note that this is NOT the same private key as used for TLS! To generate a
-  # signing key, use "./bin/generate-keys --private-key matrix_key.pem".
-  private_key: matrix_key.pem
-
-  # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
-  # to old signing keys that were formerly in use on this domain name. These
-  # keys will not be used for federation request or event signing, but will be
-  # provided to any other homeserver that asks when trying to verify old events.
-  old_private_keys:
-  #  If the old private key file is available:
-  #  - private_key: old_matrix_key.pem
-  #    expired_at: 1601024554498
-  #  If only the public key (in base64 format) and key ID are known:
-  #  - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM=
-  #    key_id: ed25519:mykeyid
-  #    expired_at: 1601024554498
-
-  # How long a remote server can cache our server signing key before requesting it
-  # again. Increasing this number will reduce the number of requests made by other
-  # servers for our key but increases the period that a compromised key will be
-  # considered valid by other homeservers.
-  key_validity_period: 168h0m0s
-
-  # Configuration for in-memory caches. Caches can often improve performance by
-  # keeping frequently accessed items (like events, identifiers etc.) in memory
-  # rather than having to read them from the database.
-  cache:
-    # The estimated maximum size for the global cache in bytes, or in terabytes,
-    # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or
-    # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a
-    # memory limit for the entire process. A cache that is too small may ultimately
-    # provide little or no benefit.
-    max_size_estimated: 1gb
-
-    # The maximum amount of time that a cache entry can live for in memory before
-    # it will be evicted and/or refreshed from the database. Lower values result in
-    # easier admission of new cache entries but may also increase database load in
-    # comparison to higher values, so adjust conservatively. Higher values may make
-    # it harder for new items to make it into the cache, e.g. if new rooms suddenly
-    # become popular.
-    max_age: 1h
-
-  # The server name to delegate server-server communications to, with optional port
-  # e.g. localhost:443
-  well_known_server_name: ""
-
-  # The server name to delegate client-server communications to, with optional port
-  # e.g. localhost:443
-  well_known_client_name: ""
-
-  # Lists of domains that the server will trust as identity servers to verify third
-  # party identifiers such as phone numbers and email addresses.
-  trusted_third_party_id_servers:
-    - matrix.org
-    - vector.im
-
-  # Disables federation. Dendrite will not be able to communicate with other servers
-  # in the Matrix federation and the federation API will not be exposed.
-  disable_federation: false
-
-  # Configures the handling of presence events. Inbound controls whether we receive
-  # presence events from other servers, outbound controls whether we send presence
-  # events for our local users to other servers.
-  presence:
-    enable_inbound: false
-    enable_outbound: false
-
-  # Configures phone-home statistics reporting. These statistics contain the server
-  # name, number of active users and some information on your deployment config.
-  # We use this information to understand how Dendrite is being used in the wild.
-  report_stats:
-    enabled: false
-    endpoint: https://matrix.org/report-usage-stats/push
-
-  # Server notices allows server admins to send messages to all users on the server.
-  server_notices:
-    enabled: false
-    # The local part, display name and avatar URL (as a mxc:// URL) for the user that
-    # will send the server notices. These are visible to all users on the deployment.
-    local_part: "_server"
-    display_name: "Server Alerts"
-    avatar_url: ""
-    # The room name to be used when sending server notices. This room name will
-    # appear in user clients.
-    room_name: "Server Alerts"
-
-  # Configuration for NATS JetStream
-  jetstream:
-    # A list of NATS Server addresses to connect to. If none are specified, an
-    # internal NATS server will be started automatically when running Dendrite in
-    # monolith mode. For polylith deployments, it is required to specify the address
-    # of at least one NATS Server node.
-    addresses:
-      - hostname:4222
-
-    # Disable the validation of TLS certificates of NATS. This is
-    # not recommended in production since it may allow NATS traffic
-    # to be sent to an insecure endpoint.
-    disable_tls_validation: false
-
-    # The prefix to use for stream names for this homeserver - really only useful
-    # if you are running more than one Dendrite server on the same NATS deployment.
-    topic_prefix: Dendrite
-
-  # Configuration for Prometheus metric collection.
-  metrics:
-    enabled: false
-    basic_auth:
-      username: metrics
-      password: metrics
-
-  # Optional DNS cache. The DNS cache may reduce the load on DNS servers if there
-  # is no local caching resolver available for use.
-  dns_cache:
-    enabled: false
-    cache_size: 256
-    cache_lifetime: "5m" # 5 minutes; https://pkg.go.dev/time@master#ParseDuration
-
-# Configuration for the Appservice API.
-app_service_api:
-  internal_api:
-    listen: http://[::]:7777 # The listen address for incoming API requests
-    connect: http://app_service_api:7777 # The connect address for other components to use
-
-  # Disable the validation of TLS certificates of appservices. This is
-  # not recommended in production since it may allow appservice traffic
-  # to be sent to an insecure endpoint.
-  disable_tls_validation: false
-
-  # Appservice configuration files to load into this homeserver.
-  config_files:
-  #  - /path/to/appservice_registration.yaml
-
-# Configuration for the Client API.
-client_api:
-  internal_api:
-    listen: http://[::]:7771 # The listen address for incoming API requests
-    connect: http://client_api:7771 # The connect address for other components to use
-  external_api:
-    listen: http://[::]:8071
-
-  # Prevents new users from being able to register on this homeserver, except when
-  # using the registration shared secret below.
-  registration_disabled: true
-
-  # Prevents new guest accounts from being created. Guest registration is also
-  # disabled implicitly by setting 'registration_disabled' above.
-  guests_disabled: true
-
-  # If set, allows registration by anyone who knows the shared secret, regardless
-  # of whether registration is otherwise disabled.
-  registration_shared_secret: ""
-
-  # Whether to require reCAPTCHA for registration. If you have enabled registration
-  # then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used
-  # for coordinated spam attacks.
-  enable_registration_captcha: false
-
-  # Settings for ReCAPTCHA.
-  recaptcha_public_key: ""
-  recaptcha_private_key: ""
-  recaptcha_bypass_secret: ""
-
-  # To use hcaptcha.com instead of ReCAPTCHA, set the following parameters, otherwise just keep them empty.
-  # recaptcha_siteverify_api: "https://hcaptcha.com/siteverify"
-  # recaptcha_api_js_url: "https://js.hcaptcha.com/1/api.js"
-  # recaptcha_form_field: "h-captcha-response"
-  # recaptcha_sitekey_class: "h-captcha"
-
-
-  # TURN server information that this homeserver should send to clients.
-  turn:
-    turn_user_lifetime: "5m"
-    turn_uris:
-    #  - turn:turn.server.org?transport=udp
-    #  - turn:turn.server.org?transport=tcp
-    turn_shared_secret: ""
-    # If your TURN server requires static credentials, then you will need to enter
-    # them here instead of supplying a shared secret. Note that these credentials
-    # will be visible to clients!
-    # turn_username: ""
-    # turn_password: ""
-
-  # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
-  # number of "slots" have been taken by requests from a specific host. Each "slot"
-  # will be released after the cooloff time in milliseconds. Server administrators
-  # and appservice users are exempt from rate limiting by default.
-  rate_limiting:
-    enabled: true
-    threshold: 20
-    cooloff_ms: 500
-    exempt_user_ids:
-    #  - "@user:domain.com"
-
-# Configuration for the Federation API.
-federation_api:
-  internal_api:
-    listen: http://[::]:7772 # The listen address for incoming API requests
-    connect: http://federation_api:7772 # The connect address for other components to use
-  external_api:
-    listen: http://[::]:8072
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_federationapi?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # How many times we will try to resend a failed transaction to a specific server. The
-  # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. Once
-  # the max retries are exceeded, Dendrite will no longer try to send transactions to
-  # that server until it comes back to life and connects to us again.
-  send_max_retries: 16
-
-  # Disable the validation of TLS certificates of remote federated homeservers. Do not
-  # enable this option in production as it presents a security risk!
-  disable_tls_validation: false
-
-  # Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically
-  # keep HTTP connections open to remote hosts for 5 minutes as they can be reused much
-  # more quickly than opening new connections each time. Disabling keepalives will close
-  # HTTP connections immediately after a successful request but may result in more CPU and
-  # memory being used on TLS handshakes for each new connection instead.
-  disable_http_keepalives: false
-
-  # Perspective keyservers to use as a backup when direct key fetches fail. This may
-  # be required to satisfy key requests for servers that are no longer online when
-  # joining some rooms.
-  key_perspectives:
-    - server_name: matrix.org
-      keys:
-        - key_id: ed25519:auto
-          public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
-        - key_id: ed25519:a_RXGa
-          public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
-
-  # This option will control whether Dendrite will prefer to look up keys directly
-  # or whether it should try perspective servers first, using direct fetches as a
-  # last resort.
-  prefer_direct_fetch: false
-
-# Configuration for the Key Server (for end-to-end encryption).
-key_server:
-  internal_api:
-    listen: http://[::]:7779 # The listen address for incoming API requests
-    connect: http://key_server:7779 # The connect address for other components to use
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_keyserver?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Media API.
-media_api:
-  internal_api:
-    listen: http://[::]:7774 # The listen address for incoming API requests
-    connect: http://media_api:7774 # The connect address for other components to use
-  external_api:
-    listen: http://[::]:8074
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_mediaapi?sslmode=disable
-    max_open_conns: 5
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # Storage path for uploaded media. May be relative or absolute.
-  base_path: ./media_store
-
-  # The maximum allowed file size (in bytes) for media uploads to this homeserver
-  # (0 = unlimited). If using a reverse proxy, ensure it allows requests at least
-  #this large (e.g. the client_max_body_size setting in nginx).
-  max_file_size_bytes: 10485760
-
-  # Whether to dynamically generate thumbnails if needed.
-  dynamic_thumbnails: false
-
-  # The maximum number of simultaneous thumbnail generators to run.
-  max_thumbnail_generators: 10
-
-  # A list of thumbnail sizes to be generated for media content.
-  thumbnail_sizes:
-    - width: 32
-      height: 32
-      method: crop
-    - width: 96
-      height: 96
-      method: crop
-    - width: 640
-      height: 480
-      method: scale
-
-# Configuration for enabling experimental MSCs on this homeserver.
-mscs:
-  mscs:
-  #  - msc2836  # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
-  #  - msc2946  # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_mscs?sslmode=disable
-    max_open_conns: 5
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Room Server.
-room_server:
-  internal_api:
-    listen: http://[::]:7770 # The listen address for incoming API requests
-    connect: http://room_server:7770 # The connect address for other components to use
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_roomserver?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Sync API.
-sync_api:
-  internal_api:
-    listen: http://[::]:7773 # The listen address for incoming API requests
-    connect: http://sync_api:7773 # The connect address for other components to use
-  external_api:
-    listen: http://[::]:8073
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_syncapi?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # Configuration for the full-text search engine.
-  search:
-    # Whether or not search is enabled.
-    enabled: false
-
-    # The path where the search index will be created in.
-    index_path: "./searchindex"
-
-    # The language most likely to be used on the server - used when indexing, to
-    # ensure the returned results match expectations. A full list of possible languages
-    # can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
-    language: "en"
-
-  # This option controls which HTTP header to inspect to find the real remote IP
-  # address of the client. This is likely required if Dendrite is running behind
-  # a reverse proxy server.
-  # real_ip_header: X-Real-IP
-
-# Configuration for the User API.
-user_api:
-  internal_api:
-    listen: http://[::]:7781 # The listen address for incoming API requests
-    connect: http://user_api:7781 # The connect address for other components to use
-  account_database:
-    connection_string: postgresql://username:password@hostname/dendrite_userapi?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31
-  # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information.
-  # Setting this lower makes registration/login consume less CPU resources at the cost
-  # of security should the database be compromised. Setting this higher makes registration/login
-  # consume more CPU resources but makes it harder to brute force password hashes. This value
-  # can be lowered if performing tests or on embedded Dendrite instances (e.g WASM builds).
-  bcrypt_cost: 10
-
-  # The length of time that a token issued for a relying party from
-  # /_matrix/client/r0/user/{userId}/openid/request_token endpoint
-  # is considered to be valid in milliseconds.
-  # The default lifetime is 3600000ms (60 minutes).
-  # openid_token_lifetime_ms: 3600000
-
-  # Users who register on this homeserver will automatically be joined to the rooms listed under "auto_join_rooms" option.
-  # By default, any room aliases included in this list will be created as a publicly joinable room
-  # when the first user registers for the homeserver. If the room already exists,
-  # make certain it is a publicly joinable room, i.e. the join rule of the room must be set to 'public'.
-  # As Spaces are just rooms under the hood, Space aliases may also be used.
-  auto_join_rooms:
-  #  - "#main:matrix.org"
-
-# Configuration for Opentracing.
-# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
-# how this works and how to set it up.
-tracing:
-  enabled: false
-  jaeger:
-    serviceName: ""
-    disabled: false
-    rpc_metrics: false
-    tags: []
-    sampler: null
-    reporter: null
-    headers: null
-    baggage_restrictions: null
-    throttler: null
-
-# Logging configuration. The "std" logging type controls the logs being sent to
-# stdout. The "file" logging type controls logs being written to a log folder on
-# the disk. Supported log levels are "debug", "info", "warn", "error".
-logging:
-  - type: std
-    level: info
-  - type: file
-    level: info
-    params:
-      path: ./logs

dendrite-sample.yaml

@@ -38,7 +38,7 @@ global:
 
   # Global database connection pool, for PostgreSQL monolith deployments only. If
   # this section is populated then you can omit the "database" blocks in all other
-  # sections. For polylith deployments, or monolith deployments using SQLite databases,
+  # sections. For monolith deployments using SQLite databases,
   # you must configure the "database" block for each component instead.
   database:
     connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable
@@ -95,7 +95,7 @@ global:
   # We use this information to understand how Dendrite is being used in the wild.
   report_stats:
     enabled: false
-    endpoint: https://matrix.org/report-usage-stats/push
+    endpoint: https://panopticon.matrix.org/push
 
   # Server notices allows server admins to send messages to all users on the server.
   server_notices:
@@ -113,8 +113,7 @@ global:
   jetstream:
     # A list of NATS Server addresses to connect to. If none are specified, an
     # internal NATS server will be started automatically when running Dendrite in
-    # monolith mode. For polylith deployments, it is required to specify the address
+    # monolith mode.
-    # of at least one NATS Server node.
     addresses:
     # - localhost:4222
 

docs/FAQ.md

@@ -6,6 +6,12 @@ permalink: /faq
 
 # FAQ
 
+## Why does Dendrite exist?
+
+Dendrite aims to provide a matrix compatible server that has low resource usage compared to [Synapse](https://github.com/matrix-org/synapse).
+It also aims to provide more flexibility when scaling either up or down.
+Dendrite's code is also very easy to hack on which makes it suitable for experimenting with new matrix features such as peer-to-peer.
+
 ## Is Dendrite stable?
 
 Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead.
@@ -29,10 +35,9 @@ possible to migrate an existing Synapse deployment to Dendrite.
 
 No, Dendrite has a very different database schema to Synapse and the two are not interchangeable.
 
-## Should I run a monolith or a polylith deployment?
+## Can I configure which port Dendrite listens on?
 
-Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't
+Yes, use the cli flag `-http-bind-address`. 
-recommend.
 
 ## I've installed Dendrite but federation isn't working
 
@@ -42,6 +47,10 @@ Check the [Federation Tester](https://federationtester.matrix.org). You need at
 * A valid TLS certificate for that DNS name
 * Either DNS SRV records or well-known files
 
+## Whenever I try to connect from Element it says unable to connect to homeserver
+
+Check that your dendrite instance is running. Otherwise this is most likely due to a reverse proxy misconfiguration.
+
 ## Does Dendrite work with my favourite client?
 
 It should do, although we are aware of some minor issues:
@@ -49,6 +58,10 @@ It should do, although we are aware of some minor issues:
 * **Element Android**: registration does not work, but logging in with an existing account does
 * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
 
+## Is there a public instance of Dendrite I can try out?
+
+Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially support.
+
 ## Does Dendrite support Space Summaries?
 
 Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration:
@@ -84,14 +97,46 @@ Remember to add the config file(s) to the `app_service_api` section of the confi
 
 Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file.
 
+## How can I migrate a room in order to change the internal ID?
+
+This can be done by performing a room upgrade. Use the command `/upgraderoom <version>` in Element to do this.
+
+## How do I reset somebody's password on my server?
+
+Use the admin endpoint [resetpassword](https://matrix-org.github.io/dendrite/administration/adminapi#post-_dendriteadminresetpassworduserid)
+
 ## Should I use PostgreSQL or SQLite for my databases?
 
 Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users.
 
+## What data needs to be kept if transferring/backing up Dendrite?
+
+The list of files that need to be stored is:
+- matrix-key.pem
+- dendrite.yaml
+- the postgres or sqlite DB
+- the media store
+- the search index (although this can be regenerated)
+
+Note that this list may change / be out of date. We don't officially maintain instructions for migrations like this.
+
+## How can I prepare enough storage for media caches?
+
+This might be what you want: [matrix-media-repo](https://github.com/turt2live/matrix-media-repo)
+We don't officially support this or any other dedicated media storage solutions.
+
+## Is there an upgrade guide for Dendrite?
+
+Run a newer docker image. We don't officially support deployments other than Docker. 
+Most of the time you should be able to just 
+- stop 
+- replace binary
+- start
+
 ## Dendrite is using a lot of CPU
 
 Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know what you were doing when the
-CPU usage shot up, or file a GitHub issue. If you can take a [CPU profile](PROFILING.md) then that would
+CPU usage shot up, or file a GitHub issue. If you can take a [CPU profile](development/PROFILING.md) then that would
 be a huge help too, as that will help us to understand where the CPU time is going.
 
 ## Dendrite is using a lot of RAM
@@ -99,9 +144,13 @@ be a huge help too, as that will help us to understand where the CPU time is goi
 As above with CPU usage, some memory spikes are expected if Dendrite is doing particularly heavy work
 at a given instant. However, if it is using more RAM than you expect for a long time, that's probably
 not expected. Join `#dendrite-dev:matrix.org` and let us know what you were doing when the memory usage
-ballooned, or file a GitHub issue if you can. If you can take a [memory profile](PROFILING.md) then that
+ballooned, or file a GitHub issue if you can. If you can take a [memory profile](development/PROFILING.md) then that
 would be a huge help too, as that will help us to understand where the memory usage is happening.
 
+## Do I need to generate the self-signed certificate if I'm going to use a reverse proxy?
+
+No, if you already have a proper certificate from some provider, like Let's Encrypt, and use that on your reverse proxy, and the reverse proxy does TLS termination, then you’re good and can use HTTP to the dendrite process.
+
 ## Dendrite is running out of PostgreSQL database connections
 
 You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!

docs/Gemfile.lock

@@ -1,7 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.5)
+    activesupport (6.0.6.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -14,8 +14,8 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.23.6)
+    commonmarker (0.23.7)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.0)
     dnsruby (1.61.9)
       simpleidn (~> 0.1)
     em-websocket (0.5.3)
@@ -229,11 +229,11 @@ GEM
       jekyll (>= 3.5, < 5.0)
       jekyll-feed (~> 0.9)
       jekyll-seo-tag (~> 2.1)
-    minitest (5.15.0)
+    minitest (5.17.0)
     multipart-post (2.1.1)
-    nokogiri (1.13.9-arm64-darwin)
+    nokogiri (1.13.10-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.13.9-x86_64-linux)
+    nokogiri (1.13.10-x86_64-linux)
       racc (~> 1.4)
     octokit (4.22.0)
       faraday (>= 0.9)
@@ -241,7 +241,7 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.7)
-    racc (1.6.0)
+    racc (1.6.1)
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
@@ -265,13 +265,13 @@ GEM
     thread_safe (0.3.6)
     typhoeus (1.4.0)
       ethon (>= 0.9.0)
-    tzinfo (1.2.10)
+    tzinfo (1.2.11)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8.1)
     unicode-display_width (1.8.0)
-    zeitwerk (2.5.4)
+    zeitwerk (2.6.6)
 
 PLATFORMS
   arm64-darwin-21

docs/INSTALL.md

@@ -9,7 +9,5 @@ or alternatively, in the [installation](installation/) folder:
 3. [Preparing database storage](installation/3_database.md)
 4. [Generating signing keys](installation/4_signingkey.md)
 5. [Installing as a monolith](installation/5_install_monolith.md)
-6. [Installing as a polylith](installation/6_install_polylith.md)
+6. [Populate the configuration](installation/7_configuration.md)
-7. [Populate the configuration](installation/7_configuration.md)
+7. [Starting the monolith](installation/8_starting_monolith.md)
-8. [Starting the monolith](installation/8_starting_monolith.md)
-9. [Starting the polylith](installation/9_starting_polylith.md)

docs/administration/4_adminapi.md

@@ -22,12 +22,12 @@ curl --header "Authorization: Bearer <access_token>" -X <POST|GET|PUT> <Endpoint
 An `access_token` can be obtained through most Element-based matrix clients by going to `Settings` -> `Help & About` -> `Advanced` -> `Access Token`.
 Be aware that an `access_token` allows a client to perform actions as an user and should be kept **secret**.
 
-The user must be an administrator in the `account_accounts` table in order to use these endpoints.
+The user must be an administrator in the `userapi_accounts` table in order to use these endpoints.
 
-Existing user accounts can be set to administrative accounts by changing `account_type` to `3` in `account_accounts`
+Existing user accounts can be set to administrative accounts by changing `account_type` to `3` in `userapi_accounts`
 
 ```
-UPDATE account_accounts SET account_type = 3 WHERE localpart = '$localpart';
+UPDATE userapi_accounts SET account_type = 3 WHERE localpart = '$localpart';
 ```
 
 Where `$localpart` is the username only (e.g. `alice`).
@@ -38,13 +38,18 @@ This endpoint will instruct Dendrite to part all local users from the given `roo
 in the URL. It may take some time to complete. A JSON body will be returned containing
 the user IDs of all affected users.
 
+If the room has an alias set (e.g. is published), the room's ID will not be visible in the URL, but it can
+be found as the room's "internal ID" in Element Web (Settings -> Advanced)
+
 ## GET `/_dendrite/admin/evacuateUser/{userID}`
 
 This endpoint will instruct Dendrite to part the given local `userID` in the URL from
 all rooms which they are currently joined. A JSON body will be returned containing
 the room IDs of all affected rooms.
 
-## POST `/_dendrite/admin/resetPassword/{localpart}`
+## POST `/_dendrite/admin/resetPassword/{userID}`
+
+Reset the password of a local user.
 
 Request body format:
 
@@ -54,9 +59,6 @@ Request body format:
 }
 ```
 
-Reset the password of a local user. The `localpart` is the username only, i.e. if
-the full user ID is `@alice:domain.com` then the local part is `alice`.
-
 ## GET `/_dendrite/admin/fulltext/reindex`
 
 This endpoint instructs Dendrite to reindex all searchable events (`m.room.message`, `m.room.topic` and `m.room.name`). An empty JSON body will be returned immediately.

docs/development/CONTRIBUTING.md

@@ -9,6 +9,28 @@ permalink: /development/contributing
 Everyone is welcome to contribute to Dendrite! We aim to make it as easy as
 possible to get started.
 
+## Contribution types
+
+We are a small team maintaining a large project. As a result, we cannot merge every feature, even if it
+is bug-free and useful, because we then commit to maintaining it indefinitely. We will always accept:
+ - bug fixes
+ - security fixes (please responsibly disclose via security@matrix.org *before* creating pull requests)
+
+We will accept the following with caveats:
+ - documentation fixes, provided they do not add additional instructions which can end up going out-of-date,
+   e.g example configs, shell commands.
+ - performance fixes, provided they do not add significantly more maintenance burden.
+ - additional functionality on existing features, provided the functionality is small and maintainable.
+ - additional functionality that, in its absence, would impact the ecosystem e.g spam and abuse mitigations
+ - test-only changes, provided they help improve coverage or test tricky code.
+
+The following items are at risk of not being accepted:
+ - Configuration or CLI changes, particularly ones which increase the overall configuration surface.
+
+The following items are unlikely to be accepted into a main Dendrite release for now:
+ - New MSC implementations.
+ - New features which are not in the specification.
+
 ## Sign off
 
 We require that everyone who contributes to the project signs off their contributions
@@ -35,7 +57,7 @@ to do so for future contributions.
 
 ## Getting up and running
 
-See the [Installation](installation) section for information on how to build an
+See the [Installation](../installation) section for information on how to build an
 instance of Dendrite. You will likely need this in order to test your changes.
 
 ## Code style
@@ -75,7 +97,20 @@ comment. Please avoid doing this if you can.
 We also have unit tests which we run via:
 
 ```bash
-go test --race ./...
+DENDRITE_TEST_SKIP_NODB=1 go test --race ./...
+```
+
+This only runs SQLite database tests. If you wish to execute Postgres tests as well, you'll either need to 
+have Postgres installed locally (`createdb` will be used) or have a remote/containerized Postgres instance
+available.
+
+To configure the connection to a remote Postgres, you can use the following enviroment variables:
+
+```bash
+POSTGRES_USER=postgres
+POSTGERS_PASSWORD=yourPostgresPassword
+POSTGRES_HOST=localhost
+POSTGRES_DB=postgres # the superuser database to use
 ```
 
 In general, we like submissions that come with tests. Anything that proves that the
@@ -116,7 +151,7 @@ significant amount of CPU and RAM.
 
 Once the code builds, run [Sytest](https://github.com/matrix-org/sytest)
 according to the guide in
-[docs/sytest.md](https://github.com/matrix-org/dendrite/blob/main/docs/sytest.md#using-a-sytest-docker-image)
+[docs/development/sytest.md](https://github.com/matrix-org/dendrite/blob/main/docs/development/sytest.md#using-a-sytest-docker-image)
 so you can see whether something is being broken and whether there are newly
 passing tests.
 

docs/development/PROFILING.md

@@ -15,7 +15,7 @@ Dendrite contains an embedded profiler called `pprof`, which is a part of the st
 To enable the profiler, start Dendrite with the `PPROFLISTEN` environment variable. This variable specifies which address and port to listen on, e.g.
 
 ```
-PPROFLISTEN=localhost:65432 ./bin/dendrite-monolith-server ...
+PPROFLISTEN=localhost:65432 ./bin/dendrite ...
 ```
 
 If pprof has been enabled successfully, a log line at startup will show that pprof is listening:

docs/development/coverage.md

@@ -14,8 +14,8 @@ index 8f0e209c..ad057e52 100644
  
     $output->diag( "Starting monolith server" );
     my @command = (
--      $self->{bindir} . '/dendrite-monolith-server',
+-      $self->{bindir} . '/dendrite',
-+      $self->{bindir} . '/dendrite-monolith-server', '--test.coverprofile=' . $self->{hs_dir} . '/integrationcover.log', "DEVEL",
++      $self->{bindir} . '/dendrite', '--test.coverprofile=' . $self->{hs_dir} . '/integrationcover.log', "DEVEL",
        '--config', $self->{paths}{config},
        '--http-bind-address', $self->{bind_host} . ':' . $self->unsecure_port,
        '--https-bind-address', $self->{bind_host} . ':' . $self->secure_port,
@@ -27,9 +27,9 @@ index f009332b..7ea79869 100755
  echo >&2 "--- Building dendrite from source"
  cd /src
  mkdir -p $GOBIN
--go install -v ./cmd/dendrite-monolith-server
+-go install -v ./cmd/dendrite
-+# go install -v ./cmd/dendrite-monolith-server
++# go install -v ./cmd/dendrite
-+go test -c -cover -covermode=atomic -o $GOBIN/dendrite-monolith-server -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server
++go test -c -cover -covermode=atomic -o $GOBIN/dendrite -coverpkg "github.com/matrix-org/..." ./cmd/dendrite
  go install -v ./cmd/generate-keys
  cd -
  ```
@@ -57,22 +57,16 @@ github.com/matrix-org/util/unique.go:55:										UniqueStrings					100.0%
 total:															(statements)					53.7%
 ```
 The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in many different configurations,
-which will never be tested in a single test run (e.g sqlite or postgres, monolith or polylith). To get a more accurate value, additional processing is required
+which will never be tested in a single test run (e.g sqlite or postgres). To get a more accurate value, additional processing is required
 to remove packages which will never be tested and extension MSCs:
 ```bash
 # These commands are all similar but change which package paths are _removed_ from the output.
 
-# For Postgres (monolith)
+# For Postgres
 go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|sqlite|setup/mscs|api_trace' > coverage.txt
 
-# For Postgres (polylith)
+# For SQLite
-go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'sqlite|setup/mscs|api_trace' > coverage.txt
-
-# For SQLite (monolith)
 go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|postgres|setup/mscs|api_trace' > coverage.txt
-
-# For SQLite (polylith)
-go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'postgres|setup/mscs|api_trace' > coverage.txt
 ```
 
 A total value can then be calculated using:

docs/development/tracing/setup.md

@@ -46,10 +46,10 @@ tracing:
       param: 1
 ```
 
-then run the monolith server with `--api true` to use polylith components which do tracing spans:
+then run the monolith server:
 
 ```
-./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml --api true
+./dendrite --tls-cert server.crt --tls-key server.key --config dendrite.yaml
 ```
 
 ## Checking traces

docs/installation/10_starting_polylith.md

@@ -1,73 +0,0 @@
----
-title: Starting the polylith
-parent: Installation
-has_toc: true
-nav_order: 10
-permalink: /installation/start/polylith
----
-
-# Starting the polylith
-
-Once you have completed all of the preparation and installation steps,
-you can start your Dendrite polylith deployment by starting the various components
-using the `dendrite-polylith-multi` personalities.
-
-## Start the reverse proxy
-
-Ensure that your reverse proxy is started and is proxying the correct
-endpoints to the correct components. Software such as [NGINX](https://www.nginx.com) or
-[HAProxy](http://www.haproxy.org) can be used for this purpose. A [sample configuration
-for NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf)
-is provided.
-
-## Starting the components
-
-Each component must be started individually:
-
-### Client API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml clientapi
-```
-
-### Sync API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml syncapi
-```
-
-### Media API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml mediaapi
-```
-
-### Federation API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml federationapi
-```
-
-### Roomserver
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml roomserver
-```
-
-### Appservice API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml appservice
-```
-
-### User API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml userapi
-```
-
-### Key server
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml keyserver
-```

docs/installation/1_planning.md

@@ -16,12 +16,6 @@ Users can run Dendrite in one of two modes which dictate how these components ar
   server with generally low overhead. This mode dramatically simplifies deployment complexity and offers the
   best balance between performance and resource usage for low-to-mid volume deployments.
 
-* **Polylith mode** runs all components in isolated processes. Components communicate through an external NATS
-  server and HTTP APIs, which incur considerable overhead. While this mode allows for more granular control of
-  resources dedicated toward individual processes, given the additional communications overhead, it is only
-  necessary for very large deployments.
-
-Given our current state of development, **we recommend monolith mode** for all deployments.
 
 ## Databases
 
@@ -85,21 +79,15 @@ If using the PostgreSQL database engine, you should install PostgreSQL 12 or lat
 
 ### NATS Server
 
-Monolith deployments come with a built-in [NATS Server](https://github.com/nats-io/nats-server) and
+Dendrite comes with a built-in [NATS Server](https://github.com/nats-io/nats-server) and
-therefore do not need this to be manually installed. If you are planning a monolith installation, you
+therefore does not need this to be manually installed. If you are planning a monolith installation, you
 do not need to do anything.
 
-Polylith deployments, however, currently need a standalone NATS Server installation with JetStream
-enabled.
-
-To do so, follow the [NATS Server installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation) and then [start your NATS deployment](https://docs.nats.io/running-a-nats-service/introduction/running). JetStream must be enabled, either by passing the `-js` flag to `nats-server`,
-or by specifying the `store_dir` option in the the `jetstream` configuration.
 
 ### Reverse proxy
 
 A reverse proxy such as [Caddy](https://caddyserver.com), [NGINX](https://www.nginx.com) or
-[HAProxy](http://www.haproxy.org) is required for polylith deployments and is useful for monolith
+[HAProxy](http://www.haproxy.org) is useful for deployments. Configuring those is not covered in this documentation, although sample configurations
-deployments. Configuring those is not covered in this documentation, although sample configurations
 for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy) and
 [NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx) are provided.