Merge branch 'master' into master

.github/workflows/docker-hub.yml

@@ -0,0 +1,71 @@
+# Based on https://github.com/docker/build-push-action
+
+name: "Docker Hub"
+
+on:
+  release:
+    types: [published]
+
+env:
+  DOCKER_NAMESPACE: matrixdotorg
+  DOCKER_HUB_USER: dendritegithub
+  PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
+
+jobs:
+  Monolith:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get release tag
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build monolith image
+        id: docker_build_monolith
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./build/docker/Dockerfile.monolith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
+
+  Polylith:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get release tag
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build polylith image
+        id: docker_build_polylith
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./build/docker/Dockerfile.polylith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}

.golangci.yml

@@ -185,6 +185,7 @@ linters:
     - gocyclo
     - goimports # Does everything gofmt does
     - gosimple
+    - govet
     - ineffassign
     - megacheck
     - misspell # Check code comments, whereas misspell in CI checks *.md files

CHANGES.md

@@ -1,5 +1,117 @@
 # Changelog
 
+## Dendrite 0.3.6 (2021-01-18)
+
+### Features
+
+* Experimental support for MSC2946 (Spaces Summary) has been merged
+* Send-to-device messages have been refactored and now take advantage of having their own stream position, making delivery more reliable
+* Unstable features and MSCs are now listed in `/versions` (contributed by [sumitks866](https://github.com/sumitks866))
+* Well-known and DNS SRV record results for federated servers are now cached properly, improving outbound federation performance and reducing traffic
+
+### Fixes
+
+* Updating forward extremities will no longer result in so many unnecessary state snapshots, reducing on-going disk usage in the roomserver database
+* Pagination tokens for `/messages` have been fixed, which should improve the reliability of scrollback/pagination
+* Dendrite now avoids returning `null`s in fields of the `/sync` response, and omitting some fields altogether when not needed, which should fix sync issues with Element Android
+* Requests for user device lists now time out quicker, which prevents federated `/send` requests from also timing out in many cases
+* Empty push rules are no longer sent over and over again in `/sync`
+* An integer overflow in the device list updater which could result in panics on 32-bit platforms has been fixed (contributed by [Lesterpig](https://github.com/Lesterpig))
+* Event IDs are now logged properly in federation sender and sync API consumer errors
+
+## Dendrite 0.3.5 (2021-01-11)
+
+### Features
+
+* All `/sync` streams are now logically separate after a refactoring exercise
+
+### Fixes
+
+* Event references are now deeply checked properly when calculating forward extremities, reducing the amount of forward extremities in most cases, which improves RAM utilisation and reduces the work done by state resolution
+* Sync no longer sends incorrect `next_batch` tokens with old stream positions, reducing flashbacks of old messages in clients
+* The federation `/send` endpoint no longer uses the request context, which could result in some events failing to be persisted if the sending server gave up the HTTP connection
+* Appservices can now auth as users in their namespaces properly
+
+## Dendrite 0.3.4 (2020-12-18)
+
+### Features
+
+* The stream tokens for `/sync` have been refactored, giving PDUs, typing notifications, read receipts, invites and send-to-device messages their own respective stream positions, greatly improving the correctness of sync
+* A new roominfo cache has been added, which results in less database hits in the roomserver
+* Prometheus metrics have been added for sync requests, destination queues and client API event send perceived latency
+
+### Fixes
+
+* Event IDs are no longer recalculated so often in `/sync`, which reduces CPU usage
+* Sync requests are now woken up correctly for our own device list updates
+* The device list stream position is no longer lost, so unnecessary device updates no longer appear in every other sync
+* A crash on concurrent map read/writes has been fixed in the stream token code
+* The roomserver input API no longer starts more worker goroutines than needed
+* The roomserver no longer uses the request context for queued tasks which could lead to send requests failing to be processed
+* A new index has been added to the sync API current state table, which improves lookup performance significantly
+* The client API `/joined_rooms` endpoint no longer incorrectly returns `null` if there are 0 rooms joined
+* The roomserver will now query appservices when looking up a local room alias that isn't known
+* The check on registration for appservice-exclusive namespaces has been fixed
+
+## Dendrite 0.3.3 (2020-12-09)
+
+### Features
+
+* Federation sender should now use considerably less CPU cycles and RAM when sending events into large rooms
+* The roomserver now uses considerably less CPU cycles by not calculating event IDs so often
+* Experimental support for [MSC2836](https://github.com/matrix-org/matrix-doc/pull/2836) (threading) has been merged
+* Dendrite will no longer hold federation HTTP connections open unnecessarily, which should help to reduce ambient CPU/RAM usage and hold fewer long-term file descriptors
+
+### Fixes
+
+* A bug in the latest event updater has been fixed, which should prevent the roomserver from losing forward extremities in some rare cases
+* A panic has been fixed when federation is disabled (contributed by [kraem](https://github.com/kraem))
+* The response format of the `/joined_members` endpoint has been fixed (contributed by [alexkursell](https://github.com/alexkursell))
+
+## Dendrite 0.3.2 (2020-12-02)
+
+### Features
+
+* Federation can now be disabled with the `global.disable_federation` configuration option
+
+### Fixes
+
+* The `"since"` parameter is now checked more thoroughly in the sync API, which led to a bug that could cause forgotten rooms to reappear (contributed by [kaniini](https://github.com/kaniini))
+* The polylith now proxies signing key requests through the federation sender correctly
+* The code for checking if remote servers are allowed to see events now no longer wastes CPU time retrieving irrelevant state events
+
+## Dendrite 0.3.1 (2020-11-20)
+
+### Features
+
+* Memory optimisation by reference passing, significantly reducing the number of allocations and duplication in memory
+* A hook API has been added for experimental MSCs, with an early implementation of MSC2836
+* The last seen timestamp and IP address are now updated automatically when calling `/sync`
+* The last seen timestamp and IP address are now reported in `/_matrix/client/r0/devices` (contributed by [alexkursell](https://github.com/alexkursell))
+* An optional configuration option `sync_api.real_ip_header` has been added for specifying which HTTP header contains the real client IP address (for if Dendrite is running behind a reverse HTTP proxy)
+* Partial implementation of `/_matrix/client/r0/admin/whois` (contributed by [DavidSpenler](https://github.com/DavidSpenler))
+
+### Fixes
+
+* A concurrency bug has been fixed in the federation API that could cause Dendrite to crash
+* The error when registering a username with invalid characters has been corrected (contributed by [bodqhrohro](https://github.com/bodqhrohro))
+
+## Dendrite 0.3.0 (2020-11-16)
+
+### Features
+
+* Read receipts (both inbound and outbound) are now supported (contributed by [S7evinK](https://github.com/S7evinK))
+* Forgetting rooms is now supported (contributed by [S7evinK](https://github.com/S7evinK))
+* The `-version` command line flag has been added (contributed by [S7evinK](https://github.com/S7evinK))
+
+### Fixes
+
+* User accounts that contain the `=` character can now be registered
+* Backfilling should now work properly on rooms with world-readable history visibility (contributed by [MayeulC](https://github.com/MayeulC))
+* The `gjson` dependency has been updated for correct JSON integer ranges
+* Some more client event fields have been marked as omit-when-empty (contributed by [S7evinK](https://github.com/S7evinK))
+* The `build.sh` script has been updated to work properly on all POSIX platforms (contributed by [felix](https://github.com/felix))
+
 ## Dendrite 0.2.1 (2020-10-22)
 
 ### Fixes

README.md

@@ -8,7 +8,6 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ
    a [brand new Go test suite](https://github.com/matrix-org/complement).
  - Scalable: can run on multiple machines and eventually scale to massive homeserver deployments.
 
-
 As of October 2020, Dendrite has now entered **beta** which means:
 - Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database.
 - Dendrite has periodic semver releases. We intend to release new versions as we land significant features.
@@ -18,13 +17,13 @@ As of October 2020, Dendrite has now entered **beta** which means:
 This does not mean:
  - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
  - All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular,
-   read receipts, presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
+   presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
  - Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
 In the future, we will be able to scale up to gigantic servers (equivalent to matrix.org) via polylith mode. 
 
-Join us in:
+If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or join us in:
 
 - **[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org)** - General chat about the Dendrite project, for users and server admins alike
 - **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** - The place for developers, where all Dendrite development discussion happens
@@ -54,31 +53,32 @@ The following instructions are enough to get Dendrite started as a non-federatin
 ```bash
 $ git clone https://github.com/matrix-org/dendrite
 $ cd dendrite
+$ ./build.sh
 
-# generate self-signed certificate and an event signing key for federation
+# Generate a Matrix signing key for federation (required)
-$ go build ./cmd/generate-keys
+$ ./bin/generate-keys --private-key matrix_key.pem
-$ ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
 
-# Copy and modify the config file:
+# Generate a self-signed certificate (optional, but a valid TLS certificate is normally
-# you'll need to set a server name and paths to the keys at the very least, along with setting
+# needed for Matrix federation/clients to work properly!)
-# up the database filenames
+$ ./bin/generate-keys --tls-cert server.crt --tls-key server.key
+
+# Copy and modify the config file - you'll need to set a server name and paths to the keys
+# at the very least, along with setting up the database connection strings.
 $ cp dendrite-config.yaml dendrite.yaml
 
-# build and run the server
+# Build and run the server:
-$ go build ./cmd/dendrite-monolith-server
+$ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
-$ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
 ```
 
-Then point your favourite Matrix client at `http://localhost:8008`.
+Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
 
 ## Progress
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of October 2020 we're at around 57% CS API coverage and 81% Federation coverage, though check
+updates with CI. As of November 2020 we're at around 58% CS API coverage and 83% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably:
- - Receipts
  - Push
  - Search and Context
  - User Directory
@@ -98,6 +98,7 @@ This means Dendrite supports amongst others:
  - Redaction
  - Tagging
  - E2E keys and device lists
+ - Receipts
 
 
 ## Contributing

appservice/api/query.go

@@ -20,9 +20,9 @@ package api
 import (
 	"context"
 	"database/sql"
+	"errors"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -109,7 +109,7 @@ func RetrieveUserProfile(
 
 	// If no user exists, return
 	if !userResp.UserIDExists {
-		return nil, eventutil.ErrProfileNoExists
+		return nil, errors.New("no known profile for given user ID")
 	}
 
 	// Try to query the user from the local database again

appservice/appservice.go

@@ -28,10 +28,10 @@ import (
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/appservice/workers"
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/internal/setup/kafka"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/kafka"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
 )

appservice/consumers/roomserver.go

@@ -21,8 +21,8 @@ import (
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/Shopify/sarama"
@@ -88,7 +88,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 		return nil
 	}
 
-	events := []gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
+	events := []*gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
 	events = append(events, output.NewRoomEvent.AddStateEvents...)
 
 	// Send event to any relevant application services
@@ -102,14 +102,14 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 // application service.
 func (s *OutputRoomEventConsumer) filterRoomserverEvents(
 	ctx context.Context,
-	events []gomatrixserverlib.HeaderedEvent,
+	events []*gomatrixserverlib.HeaderedEvent,
 ) error {
 	for _, ws := range s.workerStates {
 		for _, event := range events {
 			// Check if this event is interesting to this application service
 			if s.appserviceIsInterestedInEvent(ctx, event, ws.AppService) {
 				// Queue this event to be sent off to the application service
-				if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, &event); err != nil {
+				if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, event); err != nil {
 					log.WithError(err).Warn("failed to insert incoming event into appservices database")
 				} else {
 					// Tell our worker to send out new messages by updating remaining message
@@ -125,7 +125,7 @@ func (s *OutputRoomEventConsumer) filterRoomserverEvents(
 
 // appserviceIsInterestedInEvent returns a boolean depending on whether a given
 // event falls within one of a given application service's namespaces.
-func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
+func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
 	// No reason to queue events if they'll never be sent to the application
 	// service
 	if appservice.URL == "" {

appservice/query/query.go

@@ -23,7 +23,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	opentracing "github.com/opentracing/opentracing-go"
 	log "github.com/sirupsen/logrus"
 )

appservice/storage/postgres/storage.go

@@ -21,8 +21,8 @@ import (
 
 	// Import postgres database driver
 	_ "github.com/lib/pq"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 

appservice/storage/sqlite3/storage.go

@@ -20,8 +20,8 @@ import (
 	"database/sql"
 
 	// Import SQLite database driver
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	_ "github.com/mattn/go-sqlite3"
 )

appservice/storage/storage.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage/postgres"
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)

appservice/storage/storage_wasm.go

@@ -18,7 +18,7 @@ import (
 	"fmt"
 
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {

appservice/types/types.go

@@ -15,7 +15,7 @@ package types
 import (
 	"sync"
 
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 const (

appservice/workers/transaction_scheduler.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	log "github.com/sirupsen/logrus"
 )
@@ -185,14 +185,14 @@ func createTransaction(
 		}
 	}
 
-	var ev []gomatrixserverlib.Event
+	var ev []*gomatrixserverlib.HeaderedEvent
-	for _, e := range events {
+	for i := range events {
-		ev = append(ev, e.Event)
+		ev = append(ev, &events[i])
 	}
 
 	// Create a transaction and store the events inside
 	transaction := gomatrixserverlib.ApplicationServiceTransaction{
-		Events: ev,
+		Events: gomatrixserverlib.HeaderedToClientEvents(ev, gomatrixserverlib.FormatAll),
 	}
 
 	transactionJSON, err = json.Marshal(transaction)

build-dendritejs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -eu
+#!/bin/sh -eu
 
 export GIT_COMMIT=$(git rev-list -1 HEAD) && \
 GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs

build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -eu
+#!/bin/sh -eu
 
 # Put installed packages into ./bin
 export GOBIN=$PWD/`dirname $0`/bin
@@ -7,7 +7,7 @@ if [ -d ".git" ]
 then
     export BUILD=`git rev-parse --short HEAD || ""`
     export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""`
-    if [[ $BRANCH == "master" ]]
+    if [ "$BRANCH" = master ]
     then
         export BRANCH=""
     fi
@@ -17,6 +17,6 @@ else
     export FLAGS=""
 fi
 
-go install -trimpath -ldflags "$FLAGS" -v $PWD/`dirname $0`/cmd/...
+CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
 
-GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o main.wasm ./cmd/dendritejs
+CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs

build/docker/Dockerfile

@@ -1,10 +0,0 @@
-FROM docker.io/golang:1.15-alpine AS builder
-
-RUN apk --update --no-cache add bash build-base
-
-WORKDIR /build
-
-COPY . /build
-
-RUN mkdir -p bin
-RUN sh ./build.sh

build/docker/Dockerfile.monolith

@@ -1,11 +1,20 @@
-FROM matrixdotorg/dendrite:latest AS base
+FROM docker.io/golang:1.15-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
+RUN go build -trimpath -o bin/ ./cmd/goose
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
 
-COPY --from=base /build/bin/dendrite-monolith-server /usr/bin
+COPY --from=base /build/bin/* /usr/bin
-COPY --from=base /build/bin/goose /usr/bin
-COPY --from=base /build/bin/create-account /usr/bin
-COPY --from=base /build/bin/generate-keys /usr/bin
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite

build/docker/Dockerfile.polylith

@@ -1,11 +1,20 @@
-FROM matrixdotorg/dendrite:latest AS base
+FROM docker.io/golang:1.15-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
+RUN go build -trimpath -o bin/ ./cmd/goose
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
 
-COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin
+COPY --from=base /build/bin/* /usr/bin
-COPY --from=base /build/bin/goose /usr/bin
-COPY --from=base /build/bin/create-account /usr/bin
-COPY --from=base /build/bin/generate-keys /usr/bin
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite

build/docker/README.md

@@ -2,6 +2,11 @@
 
 These are Docker images for Dendrite!
 
+They can be found on Docker Hub:
+
+- [matrixdotorg/dendrite-monolith](https://hub.docker.com/r/matrixdotorg/dendrite-monolith) for monolith deployments
+- [matrixdotorg/dendrite-polylith](https://hub.docker.com/r/matrixdotorg/dendrite-polylith) for polylith deployments
+
 ## Dockerfiles
 
 The `Dockerfile` builds the base image which contains all of the Dendrite

build/docker/config/dendrite-config.yaml

@@ -77,7 +77,7 @@ global:
     # Naffka database options. Not required when using Kafka.
     naffka_database:
       connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable
-      max_open_conns: 100
+      max_open_conns: 10
       max_idle_conns: 2
       conn_max_lifetime: -1
 
@@ -98,7 +98,7 @@ app_service_api:
     connect: http://appservice_api:7777
   database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -173,7 +173,7 @@ federation_sender:
     connect: http://federation_sender:7775
   database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -199,7 +199,7 @@ key_server:
     connect: http://key_server:7779
   database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_keyserver?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -212,7 +212,7 @@ media_api:
     listen: http://0.0.0.0:8074
   database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -248,7 +248,7 @@ room_server:
     connect: http://room_server:7770
   database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -259,7 +259,7 @@ signing_key_server:
     connect: http://signing_key_server:7780
   database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_signingkeyserver?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -288,7 +288,7 @@ sync_api:
       listen: http://0.0.0.0:8073
   database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -299,12 +299,12 @@ user_api:
     connect: http://user_api:7781
   account_database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_account?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
   device_database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_device?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 

build/docker/docker-compose.deps.yml

@@ -1,8 +1,9 @@
 version: "3.4"
 services:
+  # PostgreSQL is needed for both polylith and monolith modes.
   postgres:
     hostname: postgres
-    image: postgres:9.6
+    image: postgres:11
     restart: always
     volumes:
       - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
@@ -15,12 +16,14 @@ services:
     networks:
       - internal
 
+  # Zookeeper is only needed for polylith mode!
   zookeeper:
     hostname: zookeeper
     image: zookeeper
     networks:
       - internal
 
+  # Kafka is only needed for polylith mode!
   kafka:
     container_name: dendrite_kafka
     hostname: kafka
@@ -29,8 +32,6 @@ services:
       KAFKA_ADVERTISED_HOST_NAME: "kafka"
       KAFKA_DELETE_TOPIC_ENABLE: "true"
       KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
-    ports:
-      - 9092:9092
     depends_on:
       - zookeeper
     networks:

build/docker/docker-compose.monolith.yml

@@ -7,6 +7,9 @@ services:
       "--tls-cert=server.crt",
       "--tls-key=server.key"
     ]
+    ports:
+      - 8008:8008
+      - 8448:8448
     volumes:
       - ./config:/etc/dendrite
     networks:

build/docker/images-build.sh

@@ -6,7 +6,5 @@ TAG=${1:-latest}
 
 echo "Building tag '${TAG}'"
 
-docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
-
 docker build -t matrixdotorg/dendrite-monolith:${TAG}  -f build/docker/Dockerfile.monolith .
 docker build -t matrixdotorg/dendrite-polylith:${TAG}  -f build/docker/Dockerfile.polylith .

build/gobind/monolith.go

@@ -17,11 +17,11 @@ import (
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
@@ -112,22 +112,25 @@ func (m *DendriteMonolith) Start() {
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
-	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
-	keyAPI.SetUserAPI(userAPI)
 
 	rsAPI := roomserver.NewInternalAPI(
 		base, keyRing,
 	)
 
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing,
+	)
+
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
+	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
+	keyAPI.SetUserAPI(userAPI)
+
 	eduInputAPI := eduserver.NewInternalAPI(
 		base, cache.New(), userAPI,
 	)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	fsAPI := federationsender.NewInternalAPI(
+	rsAPI.SetAppserviceAPI(asAPI)
-		base, federation, rsAPI, keyRing,
-	)
 
 	ygg.SetSessionFunc(func(address string) {
 		req := &api.PerformServersAliveRequest{

build/scripts/find-lint.sh

@@ -8,7 +8,7 @@
 # - `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run,
 #   golangci-lint defaults this to NumCPU
 # - `GOGC` - how often to perform garbage collection during golangci-lint runs.
-#   Essentially a ratio of memory/speed. See https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint
+#   Essentially a ratio of memory/speed. See https://golangci-lint.run/usage/performance/#memory-usage
 #   for more info.
 
 
@@ -24,8 +24,6 @@ fi
 echo "Installing golangci-lint..."
 
 # Make a backup of go.{mod,sum} first
-# TODO: Once go 1.13 is out, use go get's -mod=readonly option
-# https://github.com/golang/go/issues/30667
 cp go.mod go.mod.bak && cp go.sum go.sum.bak
 go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1
 

clientapi/auth/password.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )

clientapi/auth/user_interactive.go

@@ -20,7 +20,7 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"

clientapi/auth/user_interactive_test.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"

clientapi/clientapi.go

@@ -22,11 +22,11 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/internal/setup/kafka"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/kafka"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"

clientapi/routing/account_data.go

@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
+	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
 
@@ -148,7 +149,8 @@ type fullyReadEvent struct {
 
 // SaveReadMarker implements POST /rooms/{roomId}/read_markers
 func SaveReadMarker(
-	req *http.Request, userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
+	req *http.Request,
+	userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI,
 	syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
 ) util.JSONResponse {
 	// Verify that the user is a member of this room
@@ -192,8 +194,10 @@ func SaveReadMarker(
 		return jsonerror.InternalServerError()
 	}
 
-	// TODO handle the read receipt that may be included in the read marker
+	// Handle the read receipt that may be included in the read marker
-	// See https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-rooms-roomid-read-markers
+	if r.Read != "" {
+		return SetReceipt(req, eduAPI, device, roomID, "m.read", r.Read)
+	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,

clientapi/routing/admin_whois.go

@@ -0,0 +1,88 @@
+// Copyright 2020 David Spenler
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+
+	"github.com/matrix-org/util"
+)
+
+type adminWhoisResponse struct {
+	UserID  string                `json:"user_id"`
+	Devices map[string]deviceInfo `json:"devices"`
+}
+
+type deviceInfo struct {
+	Sessions []sessionInfo `json:"sessions"`
+}
+
+type sessionInfo struct {
+	Connections []connectionInfo `json:"connections"`
+}
+
+type connectionInfo struct {
+	IP        string `json:"ip"`
+	LastSeen  int64  `json:"last_seen"`
+	UserAgent string `json:"user_agent"`
+}
+
+// GetAdminWhois implements GET /admin/whois/{userId}
+func GetAdminWhois(
+	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	userID string,
+) util.JSONResponse {
+	if userID != device.UserID {
+		// TODO: Still allow if user is admin
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("userID does not match the current user"),
+		}
+	}
+
+	var queryRes api.QueryDevicesResponse
+	err := userAPI.QueryDevices(req.Context(), &api.QueryDevicesRequest{
+		UserID: userID,
+	}, &queryRes)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("GetAdminWhois failed to query user devices")
+		return jsonerror.InternalServerError()
+	}
+
+	devices := make(map[string]deviceInfo)
+	for _, device := range queryRes.Devices {
+		connInfo := connectionInfo{
+			IP:        device.LastSeenIP,
+			LastSeen:  device.LastSeenTS,
+			UserAgent: device.UserAgent,
+		}
+		dev, ok := devices[device.ID]
+		if !ok {
+			dev.Sessions = []sessionInfo{{}}
+		}
+		dev.Sessions[0].Connections = append(dev.Sessions[0].Connections, connInfo)
+		devices[device.ID] = dev
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: adminWhoisResponse{
+			UserID:  userID,
+			Devices: devices,
+		},
+	}
+}

clientapi/routing/auth_fallback.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/util"
 )
 

clientapi/routing/createroom.go

@@ -28,8 +28,8 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -255,7 +255,7 @@ func createRoom(
 		historyVisibility = historyVisibilityShared
 	}
 
-	var builtEvents []gomatrixserverlib.HeaderedEvent
+	var builtEvents []*gomatrixserverlib.HeaderedEvent
 
 	// send events into the room in order of:
 	//  1- m.room.create
@@ -327,13 +327,13 @@ func createRoom(
 			return jsonerror.InternalServerError()
 		}
 
-		if err = gomatrixserverlib.Allowed(*ev, &authEvents); err != nil {
+		if err = gomatrixserverlib.Allowed(ev, &authEvents); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.Allowed failed")
 			return jsonerror.InternalServerError()
 		}
 
 		// Add the event to the list of auth events
-		builtEvents = append(builtEvents, (*ev).Headered(roomVersion))
+		builtEvents = append(builtEvents, ev.Headered(roomVersion))
 		err = authEvents.AddEvent(ev)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("authEvents.AddEvent failed")
@@ -397,7 +397,7 @@ func createRoom(
 				ev := event.Event
 				globalStrippedState = append(
 					globalStrippedState,
-					gomatrixserverlib.NewInviteV2StrippedState(&ev),
+					gomatrixserverlib.NewInviteV2StrippedState(ev),
 				)
 			}
 		}
@@ -415,7 +415,7 @@ func createRoom(
 			}
 			inviteStrippedState := append(
 				globalStrippedState,
-				gomatrixserverlib.NewInviteV2StrippedState(&inviteEvent.Event),
+				gomatrixserverlib.NewInviteV2StrippedState(inviteEvent.Event),
 			)
 			// Send the invite event to the roomserver.
 			err = roomserverAPI.SendInvite(
@@ -488,5 +488,5 @@ func buildEvent(
 	if err != nil {
 		return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err)
 	}
-	return &event, nil
+	return event, nil
 }

clientapi/routing/device.go

@@ -16,6 +16,7 @@ package routing
 
 import (
 	"io/ioutil"
+	"net"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
@@ -32,7 +33,7 @@ type deviceJSON struct {
 	DeviceID    string `json:"device_id"`
 	DisplayName string `json:"display_name"`
 	LastSeenIP  string `json:"last_seen_ip"`
-	LastSeenTS  uint64 `json:"last_seen_ts"`
+	LastSeenTS  int64  `json:"last_seen_ts"`
 }
 
 type devicesJSON struct {
@@ -79,6 +80,8 @@ func GetDeviceByID(
 		JSON: deviceJSON{
 			DeviceID:    targetDevice.ID,
 			DisplayName: targetDevice.DisplayName,
+			LastSeenIP:  stripIPPort(targetDevice.LastSeenIP),
+			LastSeenTS:  targetDevice.LastSeenTS,
 		},
 	}
 }
@@ -102,6 +105,8 @@ func GetDevicesByLocalpart(
 		res.Devices = append(res.Devices, deviceJSON{
 			DeviceID:    dev.ID,
 			DisplayName: dev.DisplayName,
+			LastSeenIP:  stripIPPort(dev.LastSeenIP),
+			LastSeenTS:  dev.LastSeenTS,
 		})
 	}
 
@@ -230,3 +235,20 @@ func DeleteDevices(
 		JSON: struct{}{},
 	}
 }
+
+// stripIPPort converts strings like "[::1]:12345" to "::1"
+func stripIPPort(addr string) string {
+	ip := net.ParseIP(addr)
+	if ip != nil {
+		return addr
+	}
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return ""
+	}
+	ip = net.ParseIP(host)
+	if ip != nil {
+		return host
+	}
+	return ""
+}

clientapi/routing/directory.go

@@ -21,8 +21,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/dendrite/internal/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"

clientapi/routing/directory_public.go

@@ -26,8 +26,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )

clientapi/routing/getevent.go

@@ -18,8 +18,8 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -32,7 +32,7 @@ type getEventRequest struct {
 	eventID        string
 	cfg            *config.ClientAPI
 	federation     *gomatrixserverlib.FederationClient
-	requestedEvent gomatrixserverlib.Event
+	requestedEvent *gomatrixserverlib.Event
 }
 
 // GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}

clientapi/routing/login.go

@@ -22,7 +22,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"

clientapi/routing/membership.go

@@ -25,10 +25,10 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -77,7 +77,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us
 	if err = roomserverAPI.SendEvents(
 		ctx, rsAPI,
 		api.KindNew,
-		[]gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)},
+		[]*gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)},
 		cfg.Matrix.ServerName,
 		nil,
 	); err != nil {
@@ -214,7 +214,7 @@ func SendInvite(
 
 	err = roomserverAPI.SendInvite(
 		req.Context(), rsAPI,
-		*event,
+		event,
 		nil, // ask the roomserver to draw up invite room state for us
 		cfg.Matrix.ServerName,
 		nil,
@@ -407,3 +407,47 @@ func checkMemberInRoom(ctx context.Context, rsAPI api.RoomserverInternalAPI, use
 	}
 	return nil
 }
+
+func SendForget(
+	req *http.Request, device *userapi.Device,
+	roomID string, rsAPI roomserverAPI.RoomserverInternalAPI,
+) util.JSONResponse {
+	ctx := req.Context()
+	logger := util.GetLogger(ctx).WithField("roomID", roomID).WithField("userID", device.UserID)
+	var membershipRes api.QueryMembershipForUserResponse
+	membershipReq := api.QueryMembershipForUserRequest{
+		RoomID: roomID,
+		UserID: device.UserID,
+	}
+	err := rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes)
+	if err != nil {
+		logger.WithError(err).Error("QueryMembershipForUser: could not query membership for user")
+		return jsonerror.InternalServerError()
+	}
+	if membershipRes.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Forbidden("user is still a member of the room"),
+		}
+	}
+	if !membershipRes.HasBeenInRoom {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Forbidden("user did not belong to room"),
+		}
+	}
+
+	request := api.PerformForgetRequest{
+		RoomID: roomID,
+		UserID: device.UserID,
+	}
+	response := api.PerformForgetResponse{}
+	if err := rsAPI.PerformForget(ctx, &request, &response); err != nil {
+		logger.WithError(err).Error("PerformForget: unable to forget room")
+		return jsonerror.InternalServerError()
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/memberships.go

@@ -19,8 +19,8 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -44,6 +44,13 @@ type joinedMember struct {
 	AvatarURL   string `json:"avatar_url"`
 }
 
+// The database stores 'displayname' without an underscore.
+// Deserialize into this and then change to the actual API response
+type databaseJoinedMember struct {
+	DisplayName string `json:"displayname"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
 // GetMemberships implements GET /rooms/{roomId}/members
 func GetMemberships(
 	req *http.Request, device *userapi.Device, roomID string, joinedOnly bool,
@@ -72,12 +79,12 @@ func GetMemberships(
 		var res getJoinedMembersResponse
 		res.Joined = make(map[string]joinedMember)
 		for _, ev := range queryRes.JoinEvents {
-			var content joinedMember
+			var content databaseJoinedMember
 			if err := json.Unmarshal(ev.Content, &content); err != nil {
 				util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
 				return jsonerror.InternalServerError()
 			}
-			res.Joined[ev.Sender] = content
+			res.Joined[ev.Sender] = joinedMember(content)
 		}
 		return util.JSONResponse{
 			Code: http.StatusOK,
@@ -104,6 +111,9 @@ func GetJoinedRooms(
 		util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
 		return jsonerror.InternalServerError()
 	}
+	if res.RoomIDs == nil {
+		res.RoomIDs = []string{}
+	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: getJoinedRoomsResponse{res.RoomIDs},

clientapi/routing/password.go

@@ -7,7 +7,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"

clientapi/routing/peekroom.go

@@ -77,3 +77,28 @@ func PeekRoomByIDOrAlias(
 		}{peekRes.RoomID},
 	}
 }
+
+func UnpeekRoomByID(
+	req *http.Request,
+	device *api.Device,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
+	accountDB accounts.Database,
+	roomID string,
+) util.JSONResponse {
+	unpeekReq := roomserverAPI.PerformUnpeekRequest{
+		RoomID:   roomID,
+		UserID:   device.UserID,
+		DeviceID: device.ID,
+	}
+	unpeekRes := roomserverAPI.PerformUnpeekResponse{}
+
+	rsAPI.PerformUnpeek(req.Context(), &unpeekReq, &unpeekRes)
+	if unpeekRes.Error != nil {
+		return unpeekRes.Error.JSONResponse()
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/profile.go

@@ -23,9 +23,9 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -346,14 +346,14 @@ func buildMembershipEvents(
 	roomIDs []string,
 	newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
 	evTime time.Time, rsAPI api.RoomserverInternalAPI,
-) ([]gomatrixserverlib.HeaderedEvent, error) {
+) ([]*gomatrixserverlib.HeaderedEvent, error) {
-	evs := []gomatrixserverlib.HeaderedEvent{}
+	evs := []*gomatrixserverlib.HeaderedEvent{}
 
 	for _, roomID := range roomIDs {
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
 		if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
-			return []gomatrixserverlib.HeaderedEvent{}, err
+			return nil, err
 		}
 
 		builder := gomatrixserverlib.EventBuilder{
@@ -379,7 +379,7 @@ func buildMembershipEvents(
 			return nil, err
 		}
 
-		evs = append(evs, (*event).Headered(verRes.RoomVersion))
+		evs = append(evs, event.Headered(verRes.RoomVersion))
 	}
 
 	return evs, nil

clientapi/routing/rate_limiting.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/util"
 )
 

clientapi/routing/receipt.go

@@ -0,0 +1,54 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/eduserver/api"
+
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+)
+
+func SetReceipt(req *http.Request, eduAPI api.EDUServerInputAPI, device *userapi.Device, roomId, receiptType, eventId string) util.JSONResponse {
+	timestamp := gomatrixserverlib.AsTimestamp(time.Now())
+	logrus.WithFields(logrus.Fields{
+		"roomId":      roomId,
+		"receiptType": receiptType,
+		"eventId":     eventId,
+		"userId":      device.UserID,
+		"timestamp":   timestamp,
+	}).Debug("Setting receipt")
+
+	// currently only m.read is accepted
+	if receiptType != "m.read" {
+		return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType))
+	}
+
+	if err := api.SendReceipt(req.Context(), eduAPI, device.UserID, roomId, eventId, receiptType, timestamp); err != nil {
+		return util.ErrorResponse(err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/redaction.go

@@ -21,10 +21,10 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -121,7 +121,7 @@ func SendRedaction(
 			JSON: jsonerror.NotFound("Room does not exist"),
 		}
 	}
-	if err = roomserverAPI.SendEvents(context.Background(), rsAPI, api.KindNew, []gomatrixserverlib.HeaderedEvent{*e}, cfg.Matrix.ServerName, nil); err != nil {
+	if err = roomserverAPI.SendEvents(context.Background(), rsAPI, api.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, cfg.Matrix.ServerName, nil); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/register.go

@@ -32,8 +32,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/setup/config"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@@ -113,7 +113,7 @@ var (
 	// TODO: Remove old sessions. Need to do so on a session-specific timeout.
 	// sessions stores the completed flow stages for all sessions. Referenced using their sessionID.
 	sessions           = newSessionsDict()
-	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-./]+$`)
+	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 )
 
 // registerRequest represents the submitted registration request.
@@ -209,7 +209,7 @@ func validateUsername(username string) *util.JSONResponse {
 	} else if !validUsernameRegex.MatchString(username) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./'"),
+			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
 		}
 	} else if username[0] == '_' { // Regex checks its not a zero length string
 		return &util.JSONResponse{
@@ -230,7 +230,7 @@ func validateApplicationServiceUsername(username string) *util.JSONResponse {
 	} else if !validUsernameRegex.MatchString(username) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./'"),
+			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
 		}
 	}
 	return nil
@@ -328,7 +328,22 @@ func UserIDIsWithinApplicationServiceNamespace(
 	userID string,
 	appservice *config.ApplicationService,
 ) bool {
+
+	var local, domain, err = gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		// Not a valid userID
+		return false
+	}
+
+	if domain != cfg.Matrix.ServerName {
+		return false
+	}
+
 	if appservice != nil {
+		if appservice.SenderLocalpart == local {
+			return true
+		}
+
 		// Loop through given application service's namespaces and see if any match
 		for _, namespace := range appservice.NamespaceMap["users"] {
 			// AS namespaces are checked for validity in config
@@ -341,6 +356,9 @@ func UserIDIsWithinApplicationServiceNamespace(
 
 	// Loop through all known application service's namespaces and see if any match
 	for _, knownAppService := range cfg.Derived.ApplicationServices {
+		if knownAppService.SenderLocalpart == local {
+			return true
+		}
 		for _, namespace := range knownAppService.NamespaceMap["users"] {
 			// AS namespaces are checked for validity in config
 			if namespace.RegexpObject.MatchString(userID) {
@@ -488,17 +506,6 @@ func Register(
 		return *resErr
 	}
 
-	// Make sure normal user isn't registering under an exclusive application
-	// service namespace. Skip this check if no app services are registered.
-	if r.Auth.Type != authtypes.LoginTypeApplicationService &&
-		len(cfg.Derived.ApplicationServices) != 0 &&
-		UsernameMatchesExclusiveNamespaces(cfg, r.Username) {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.ASExclusive("This username is reserved by an application service."),
-		}
-	}
-
 	logger := util.GetLogger(req.Context())
 	logger.WithFields(log.Fields{
 		"username":   r.Username,
@@ -581,11 +588,33 @@ func handleRegistrationFlow(
 	// TODO: Handle mapping registrationRequest parameters into session parameters
 
 	// TODO: email / msisdn auth types.
+	accessToken, accessTokenErr := auth.ExtractAccessToken(req)
+
+	// Appservices are special and are not affected by disabled
+	// registration or user exclusivity.
+	if r.Auth.Type == authtypes.LoginTypeApplicationService ||
+		(r.Auth.Type == "" && accessTokenErr == nil) {
+		return handleApplicationServiceRegistration(
+			accessToken, accessTokenErr, req, r, cfg, userAPI,
+		)
+	}
 
 	if cfg.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
 		return util.MessageResponse(http.StatusForbidden, "Registration has been disabled")
 	}
 
+	// Make sure normal user isn't registering under an exclusive application
+	// service namespace. Skip this check if no app services are registered.
+	// If an access token is provided, ignore this check this is an appservice
+	// request and we will validate in validateApplicationService
+	if len(cfg.Derived.ApplicationServices) != 0 &&
+		UsernameMatchesExclusiveNamespaces(cfg, r.Username) {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.ASExclusive("This username is reserved by an application service."),
+		}
+	}
+
 	switch r.Auth.Type {
 	case authtypes.LoginTypeRecaptcha:
 		// Check given captcha response
@@ -611,36 +640,15 @@ func handleRegistrationFlow(
 		// Add SharedSecret to the list of completed registration stages
 		AddCompletedSessionStage(sessionID, authtypes.LoginTypeSharedSecret)
 
-	case "":
-		// Extract the access token from the request, if there's one to extract
-		// (which we can know by checking whether the error is nil or not).
-		accessToken, err := auth.ExtractAccessToken(req)
-
-		// A missing auth type can mean either the registration is performed by
-		// an AS or the request is made as the first step of a registration
-		// using the User-Interactive Authentication API. This can be determined
-		// by whether the request contains an access token.
-		if err == nil {
-			return handleApplicationServiceRegistration(
-				accessToken, err, req, r, cfg, userAPI,
-			)
-		}
-
-	case authtypes.LoginTypeApplicationService:
-		// Extract the access token from the request.
-		accessToken, err := auth.ExtractAccessToken(req)
-		// Let the AS registration handler handle the process from here. We
-		// don't need a condition on that call since the registration is clearly
-		// stated as being AS-related.
-		return handleApplicationServiceRegistration(
-			accessToken, err, req, r, cfg, userAPI,
-		)
-
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
 		// Add Dummy to the list of completed registration stages
 		AddCompletedSessionStage(sessionID, authtypes.LoginTypeDummy)
 
+	case "":
+		// An empty auth type means that we want to fetch the available
+		// flows. It can also mean that we want to register as an appservice
+		// but that is handed above.
 	default:
 		return util.JSONResponse{
 			Code: http.StatusNotImplemented,

clientapi/routing/register_test.go

@@ -19,7 +19,7 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 var (

clientapi/routing/routing.go

@@ -28,11 +28,11 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -62,13 +62,19 @@ func Setup(
 	rateLimits := newRateLimits(&cfg.RateLimiting)
 	userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg)
 
+	unstableFeatures := make(map[string]bool)
+	for _, msc := range cfg.MSCs.MSCs {
+		unstableFeatures["org.matrix."+msc] = true
+	}
+
 	publicAPIMux.Handle("/versions",
 		httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusOK,
 				JSON: struct {
 					Versions         []string        `json:"versions"`
-				}{[]string{
+					UnstableFeatures map[string]bool `json:"unstable_features"`
+				}{Versions: []string{
 					"r0.0.1",
 					"r0.1.0",
 					"r0.2.0",
@@ -76,7 +82,7 @@ func Setup(
 					"r0.4.0",
 					"r0.5.0",
 					"r0.6.1",
-				}},
+				}, UnstableFeatures: unstableFeatures},
 			}
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -106,6 +112,9 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/peek/{roomIDOrAlias}",
 		httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			if r := rateLimits.rateLimit(req); r != nil {
+				return *r
+			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -148,6 +157,17 @@ func Setup(
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/unpeek",
+		httputil.MakeAuthAPI("unpeek", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return UnpeekRoomByID(
+				req, device, rsAPI, accountDB, vars["roomID"],
+			)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/ban",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -651,6 +671,16 @@ func Setup(
 		}),
 	).Methods(http.MethodGet)
 
+	r0mux.Handle("/admin/whois/{userID}",
+		httputil.MakeAuthAPI("admin_whois", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetAdminWhois(req, userAPI, device, vars["userID"])
+		}),
+	).Methods(http.MethodGet)
+
 	r0mux.Handle("/user_directory/search",
 		httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.rateLimit(req); r != nil {
@@ -705,7 +735,20 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SaveReadMarker(req, userAPI, rsAPI, syncProducer, device, vars["roomID"])
+			return SaveReadMarker(req, userAPI, rsAPI, eduAPI, syncProducer, device, vars["roomID"])
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/forget",
+		httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			if r := rateLimits.rateLimit(req); r != nil {
+				return *r
+			}
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendForget(req, device, vars["roomID"], rsAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -830,4 +873,17 @@ func Setup(
 			return ClaimKeys(req, keyAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			if r := rateLimits.rateLimit(req); r != nil {
+				return *r
+			}
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+
+			return SetReceipt(req, eduAPI, device, vars["roomId"], vars["receiptType"], vars["eventId"])
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
 }

clientapi/routing/sendevent.go

@@ -17,16 +17,18 @@ package routing
 import (
 	"net/http"
 	"sync"
+	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -40,6 +42,25 @@ var (
 	userRoomSendMutexes sync.Map // (roomID+userID) -> mutex. mutexes to ensure correct ordering of sendEvents
 )
 
+func init() {
+	prometheus.MustRegister(sendEventDuration)
+}
+
+var sendEventDuration = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Namespace: "dendrite",
+		Subsystem: "clientapi",
+		Name:      "sendevent_duration_millis",
+		Help:      "How long it takes to build and submit a new event from the client API to the roomserver",
+		Buckets: []float64{ // milliseconds
+			5, 10, 25, 50, 75, 100, 250, 500,
+			1000, 2000, 3000, 4000, 5000, 6000,
+			7000, 8000, 9000, 10000, 15000, 20000,
+		},
+	},
+	[]string{"action"},
+)
+
 // SendEvent implements:
 //   /rooms/{roomID}/send/{eventType}
 //   /rooms/{roomID}/send/{eventType}/{txnID}
@@ -75,10 +96,12 @@ func SendEvent(
 	mutex.(*sync.Mutex).Lock()
 	defer mutex.(*sync.Mutex).Unlock()
 
+	startedGeneratingEvent := time.Now()
 	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI)
 	if resErr != nil {
 		return *resErr
 	}
+	timeToGenerateEvent := time.Since(startedGeneratingEvent)
 
 	var txnAndSessionID *api.TransactionID
 	if txnID != nil {
@@ -90,10 +113,11 @@ func SendEvent(
 
 	// pass the new event to the roomserver and receive the correct event ID
 	// event ID in case of duplicate transaction is discarded
+	startedSubmittingEvent := time.Now()
 	if err := api.SendEvents(
 		req.Context(), rsAPI,
 		api.KindNew,
-		[]gomatrixserverlib.HeaderedEvent{
+		[]*gomatrixserverlib.HeaderedEvent{
 			e.Headered(verRes.RoomVersion),
 		},
 		cfg.Matrix.ServerName,
@@ -102,6 +126,7 @@ func SendEvent(
 		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
 		return jsonerror.InternalServerError()
 	}
+	timeToSubmitEvent := time.Since(startedSubmittingEvent)
 	util.GetLogger(req.Context()).WithFields(logrus.Fields{
 		"event_id":     e.EventID(),
 		"room_id":      roomID,
@@ -117,6 +142,11 @@ func SendEvent(
 		txnCache.AddTransaction(device.AccessToken, *txnID, &res)
 	}
 
+	// Take a note of how long it took to generate the event vs submit
+	// it to the roomserver.
+	sendEventDuration.With(prometheus.Labels{"action": "build"}).Observe(float64(timeToGenerateEvent.Milliseconds()))
+	sendEventDuration.With(prometheus.Labels{"action": "submit"}).Observe(float64(timeToSubmitEvent.Milliseconds()))
+
 	return res
 }
 
@@ -189,7 +219,7 @@ func generateSendEvent(
 	// check to see if this user can perform this operation
 	stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents))
 	for i := range queryRes.StateEvents {
-		stateEvents[i] = &queryRes.StateEvents[i].Event
+		stateEvents[i] = queryRes.StateEvents[i].Event
 	}
 	provider := gomatrixserverlib.NewAuthEvents(stateEvents)
 	if err = gomatrixserverlib.Allowed(e.Event, &provider); err != nil {
@@ -198,5 +228,5 @@ func generateSendEvent(
 			JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
 		}
 	}
-	return &e.Event, nil
+	return e.Event, nil
 }

clientapi/routing/state.go

@@ -267,7 +267,7 @@ func OnIncomingStateTypeRequest(
 		// to find the state event, if provided.
 		for _, ev := range stateRes.StateEvents {
 			if ev.Type() == evType && ev.StateKeyEquals(stateKey) {
-				event = &ev
+				event = ev
 				break
 			}
 		}
@@ -290,7 +290,7 @@ func OnIncomingStateTypeRequest(
 			return jsonerror.InternalServerError()
 		}
 		if len(stateAfterRes.StateEvents) > 0 {
-			event = &stateAfterRes.StateEvents[0]
+			event = stateAfterRes.StateEvents[0]
 		}
 	}
 
@@ -304,7 +304,7 @@ func OnIncomingStateTypeRequest(
 	}
 
 	stateEvent := stateEventInStateResp{
-		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(*event, gomatrixserverlib.FormatAll),
+		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(event, gomatrixserverlib.FormatAll),
 	}
 
 	var res interface{}

clientapi/routing/threepid.go

@@ -21,7 +21,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 

clientapi/routing/voip.go

@@ -23,7 +23,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/util"

clientapi/threepid/invites.go

@@ -25,9 +25,9 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -362,8 +362,8 @@ func emit3PIDInviteEvent(
 	return api.SendEvents(
 		ctx, rsAPI,
 		api.KindNew,
-		[]gomatrixserverlib.HeaderedEvent{
+		[]*gomatrixserverlib.HeaderedEvent{
-			(*event).Headered(queryRes.RoomVersion),
+			event.Headered(queryRes.RoomVersion),
 		},
 		cfg.Matrix.ServerName,
 		nil,

clientapi/threepid/threepid.go

@@ -24,7 +24,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken

cmd/create-account/main.go

@@ -20,24 +20,27 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
 )
 
 const usage = `Usage: %s
 
-Generate a new Matrix account for testing purposes.
+Creates a new user account on the homeserver.
+
+Example:
+
+  ./create-account --config dendrite.yaml --username alice --password foobarbaz
 
 Arguments:
 
 `
 
 var (
-	database      = flag.String("database", "", "The location of the account database.")
+	username = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
-	username      = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.")
+	password = flag.String("password", "", "The password to associate with the account (optional, account will be password-less if not specified)")
-	password      = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.")
-	serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.")
 )
 
 func main() {
@@ -45,36 +48,24 @@ func main() {
 		fmt.Fprintf(os.Stderr, usage, os.Args[0])
 		flag.PrintDefaults()
 	}
-
+	cfg := setup.ParseFlags(true)
-	flag.Parse()
 
 	if *username == "" {
 		flag.Usage()
-		fmt.Println("Missing --username")
 		os.Exit(1)
 	}
 
-	if *database == "" {
-		flag.Usage()
-		fmt.Println("Missing --database")
-		os.Exit(1)
-	}
-
-	serverName := gomatrixserverlib.ServerName(*serverNameStr)
-
 	accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{
-		ConnectionString: config.DataSource(*database),
+		ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString,
-	}, serverName)
+	}, cfg.Global.ServerName)
 	if err != nil {
-		fmt.Println(err.Error())
+		logrus.Fatalln("Failed to connect to the database:", err.Error())
-		os.Exit(1)
 	}
 
 	_, err = accountDB.CreateAccount(context.Background(), *username, *password, "")
 	if err != nil {
-		fmt.Println(err.Error())
+		logrus.Fatalln("Failed to create the account:", err.Error())
-		os.Exit(1)
 	}
 
-	fmt.Println("Created account")
+	logrus.Infoln("Created account", *username)
 }

cmd/create-room-events/main.go

@@ -123,7 +123,7 @@ func buildAndOutput() gomatrixserverlib.EventReference {
 }
 
 // Write an event to the output.
-func writeEvent(event gomatrixserverlib.Event) {
+func writeEvent(event *gomatrixserverlib.Event) {
 	encoder := json.NewEncoder(os.Stdout)
 	if *format == "InputRoomEvent" {
 		var ire api.InputRoomEvent

cmd/dendrite-demo-libp2p/main.go

@@ -31,11 +31,12 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/signingkeyserver"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -130,6 +131,8 @@ func main() {
 	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
 	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
 	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
+	cfg.MSCs.MSCs = []string{"msc2836"}
+	cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
@@ -158,6 +161,7 @@ func main() {
 		&base.Base, cache.New(), userAPI,
 	)
 	asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
+	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationsender.NewInternalAPI(
 		&base.Base, federation, rsAPI, keyRing,
 	)
@@ -190,6 +194,9 @@ func main() {
 		base.Base.PublicKeyAPIMux,
 		base.Base.PublicMediaAPIMux,
 	)
+	if err := mscs.Enable(&base.Base, &monolith); err != nil {
+		logrus.WithError(err).Fatalf("Failed to enable MSCs")
+	}
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.Base.InternalAPIMux)

cmd/dendrite-demo-libp2p/p2pdendrite.go

@@ -22,7 +22,7 @@ import (
 
 	pstore "github.com/libp2p/go-libp2p-core/peerstore"
 	record "github.com/libp2p/go-libp2p-record"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup"
 
 	"github.com/libp2p/go-libp2p"
 	circuit "github.com/libp2p/go-libp2p-circuit"
@@ -34,7 +34,7 @@ import (
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/matrix-org/gomatrixserverlib"
 
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // P2PDendrite is a Peer-to-Peer variant of BaseDendrite.

cmd/dendrite-demo-yggdrasil/main.go

@@ -34,11 +34,12 @@ import (
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -83,6 +84,8 @@ func main() {
 	cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
 	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
 	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
+	cfg.MSCs.MSCs = []string{"msc2836"}
+	cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
@@ -110,6 +113,7 @@ func main() {
 	)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationsender.NewInternalAPI(
 		base, federation, rsAPI, keyRing,
 	)
@@ -151,6 +155,9 @@ func main() {
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
 	)
+	if err := mscs.Enable(base, &monolith); err != nil {
+		logrus.WithError(err).Fatalf("Failed to enable MSCs")
+	}
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)

cmd/dendrite-demo-yggdrasil/yggconn/client.go

@@ -4,7 +4,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 

cmd/dendrite-monolith-server/main.go

@@ -22,11 +22,12 @@ import (
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/signingkeyserver"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/sirupsen/logrus"
@@ -125,6 +126,7 @@ func main() {
 		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
 		asAPI = base.AppserviceHTTPClient()
 	}
+	rsAPI.SetAppserviceAPI(asAPI)
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
@@ -148,6 +150,12 @@ func main() {
 		base.PublicMediaAPIMux,
 	)
 
+	if len(base.Cfg.MSCs.MSCs) > 0 {
+		if err := mscs.Enable(base, &monolith); err != nil {
+			logrus.WithError(err).Fatalf("Failed to enable MSCs")
+		}
+	}
+
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {
 		base.SetupAndServeHTTP(

cmd/dendrite-polylith-multi/main.go

@@ -20,8 +20,8 @@ import (
 	"strings"
 
 	"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/sirupsen/logrus"
 )
 

cmd/dendrite-polylith-multi/personalities/appservice.go

@@ -16,8 +16,8 @@ package personalities
 
 import (
 	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) {

cmd/dendrite-polylith-multi/personalities/clientapi.go

@@ -16,9 +16,9 @@ package personalities
 
 import (
 	"github.com/matrix-org/dendrite/clientapi"
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/internal/transactions"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {

cmd/dendrite-polylith-multi/personalities/eduserver.go

@@ -17,8 +17,8 @@ package personalities
 import (
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) {

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -16,8 +16,8 @@ package personalities
 
 import (
 	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {

cmd/dendrite-polylith-multi/personalities/federationsender.go

@@ -16,8 +16,8 @@ package personalities
 
 import (
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -15,13 +15,14 @@
 package personalities
 
 import (
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
+	fsAPI := base.FederationSenderHTTPClient()
+	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI)
 	intAPI.SetUserAPI(base.UserAPIClient())
 
 	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)

cmd/dendrite-polylith-multi/personalities/mediaapi.go

@@ -15,9 +15,9 @@
 package personalities
 
 import (
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/mediaapi"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -15,18 +15,20 @@
 package personalities
 
 import (
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	serverKeyAPI := base.SigningKeyServerHTTPClient()
 	keyRing := serverKeyAPI.KeyRing()
 
+	asAPI := base.AppserviceHTTPClient()
 	fsAPI := base.FederationSenderHTTPClient()
 	rsAPI := roomserver.NewInternalAPI(base, keyRing)
 	rsAPI.SetFederationSenderAPI(fsAPI)
+	rsAPI.SetAppserviceAPI(asAPI)
 	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
 
 	base.SetupAndServeHTTP(

cmd/dendrite-polylith-multi/personalities/signingkeyserver.go

@@ -15,8 +15,8 @@
 package personalities
 
 import (
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/signingkeyserver"
 )
 

cmd/dendrite-polylith-multi/personalities/syncapi.go

@@ -15,8 +15,8 @@
 package personalities
 
 import (
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi"
 )
 

cmd/dendrite-polylith-multi/personalities/userapi.go

@@ -15,8 +15,8 @@
 package personalities
 
 import (
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi"
 )
 

cmd/dendritejs/main.go

@@ -26,11 +26,11 @@ import (
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi"
 	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
 
@@ -207,6 +207,7 @@ func main() {
 	asQuery := appservice.NewInternalAPI(
 		base, userAPI, rsAPI,
 	)
+	rsAPI.SetAppserviceAPI(asQuery)
 	fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing)
 	rsAPI.SetFederationSenderAPI(fedSenderAPI)
 	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation)

cmd/generate-config/main.go

@@ -4,7 +4,7 @@ import (
 	"flag"
 	"fmt"
 
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup/config"
 	"gopkg.in/yaml.v2"
 )
 
@@ -63,6 +63,10 @@ func main() {
 	if *defaultsForCI {
 		cfg.ClientAPI.RateLimiting.Enabled = false
 		cfg.FederationSender.DisableTLSValidation = true
+		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+		cfg.Logging[0].Level = "trace"
+		// don't hit matrix.org when running tests!!!
+		cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{}
 	}
 
 	j, err := yaml.Marshal(cfg)

cmd/resolve-state/main.go

@@ -8,10 +8,10 @@ import (
 	"strconv"
 
 	"github.com/matrix-org/dendrite/internal/caching"
-	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -80,9 +80,9 @@ func main() {
 	}
 
 	authEventIDMap := make(map[string]struct{})
-	eventPtrs := make([]*gomatrixserverlib.Event, len(eventEntries))
+	events := make([]*gomatrixserverlib.Event, len(eventEntries))
 	for i := range eventEntries {
-		eventPtrs[i] = &eventEntries[i].Event
+		events[i] = eventEntries[i].Event
 		for _, authEventID := range eventEntries[i].AuthEventIDs() {
 			authEventIDMap[authEventID] = struct{}{}
 		}
@@ -99,18 +99,9 @@ func main() {
 		panic(err)
 	}
 
-	authEventPtrs := make([]*gomatrixserverlib.Event, len(authEventEntries))
+	authEvents := make([]*gomatrixserverlib.Event, len(authEventEntries))
 	for i := range authEventEntries {
-		authEventPtrs[i] = &authEventEntries[i].Event
+		authEvents[i] = authEventEntries[i].Event
-	}
-
-	events := make([]gomatrixserverlib.Event, len(eventEntries))
-	authEvents := make([]gomatrixserverlib.Event, len(authEventEntries))
-	for i, ptr := range eventPtrs {
-		events[i] = *ptr
-	}
-	for i, ptr := range authEventPtrs {
-		authEvents[i] = *ptr
 	}
 
 	fmt.Println("Resolving state")

cmd/roomserver-integration-tests/main.go

@@ -29,10 +29,10 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/internal/caching"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/inthttp"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 

cmd/syncserver-integration-tests/main.go

@@ -24,9 +24,9 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -103,7 +103,7 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string {
 	if err := json.Unmarshal([]byte(outputRoomEvent), &out); err != nil {
 		panic("failed to unmarshal output room event: " + err.Error())
 	}
-	clientEvs := gomatrixserverlib.ToClientEvents([]gomatrixserverlib.Event{
+	clientEvs := gomatrixserverlib.ToClientEvents([]*gomatrixserverlib.Event{
 		out.NewRoomEvent.Event.Event,
 	}, gomatrixserverlib.FormatSync)
 	b, err := json.Marshal(clientEvs[0])

dendrite-config.yaml

@@ -36,6 +36,8 @@ global:
   server_name: localhost
 
   # The path to the signing private key file, used to sign requests and events.
+  # Note that this is NOT the same private key as used for TLS! To generate a
+  # signing key, use "./bin/generate-keys --private-key matrix_key.pem".
   private_key: matrix_key.pem
 
   # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
@@ -58,6 +60,10 @@ global:
   - matrix.org
   - vector.im
 
+  # Disables federation. Dendrite will not be able to make any outbound HTTP requests
+  # to other servers and the federation API will not be exposed.
+  disable_federation: false
+
   # Configuration for Kafka/Naffka.
   kafka:
     # List of Kafka broker addresses to connect to. This is not needed if using
@@ -74,10 +80,16 @@ global:
     # Kafka.
     use_naffka: true
 
+    # The max size a Kafka message is allowed to use.
+    # You only need to change this value, if you encounter issues with too large messages.
+    # Must be less than/equal to "max.message.bytes" configured in Kafka.
+    # Defaults to 8388608 bytes.
+    # max_message_bytes: 8388608
+
     # Naffka database options. Not required when using Kafka.
     naffka_database:
       connection_string: file:naffka.db
-      max_open_conns: 100
+      max_open_conns: 10
       max_idle_conns: 2
       conn_max_lifetime: -1
 
@@ -98,7 +110,7 @@ app_service_api:
     connect: http://localhost:7777
   database:
     connection_string: file:appservice.db
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -173,7 +185,7 @@ federation_sender:
     connect: http://localhost:7775
   database:
     connection_string: file:federationsender.db
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -199,7 +211,7 @@ key_server:
     connect: http://localhost:7779
   database:
     connection_string: file:keyserver.db
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -212,7 +224,7 @@ media_api:
     listen: http://[::]:8074
   database:
     connection_string: file:mediaapi.db
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -241,6 +253,19 @@ media_api:
     height: 480
     method: scale
 
+# Configuration for experimental MSC's
+mscs:
+  # A list of enabled MSC's
+  # Currently valid values are:
+  # - msc2836    (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
+  # - msc2946    (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
+  mscs: []
+  database:
+    connection_string: file:mscs.db
+    max_open_conns: 10
+    max_idle_conns: 2
+    conn_max_lifetime: -1
+
 # Configuration for the Room Server.
 room_server:
   internal_api:
@@ -248,7 +273,7 @@ room_server:
     connect: http://localhost:7770
   database:
     connection_string: file:roomserver.db
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -259,7 +284,7 @@ signing_key_server:
     connect: http://localhost:7780
   database:
     connection_string: file:signingkeyserver.db
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
@@ -288,10 +313,15 @@ sync_api:
     listen: http://[::]:8073
   database:
     connection_string: file:syncapi.db
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
+  # This option controls which HTTP header to inspect to find the real remote IP
+  # address of the client. This is likely required if Dendrite is running behind
+  # a reverse proxy server.
+  # real_ip_header: X-Real-IP
+
 # Configuration for the User API.
 user_api:
   internal_api:
@@ -299,12 +329,12 @@ user_api:
     connect: http://localhost:7781
   account_database:
     connection_string: file:userapi_accounts.db
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
   device_database:
     connection_string: file:userapi_devices.db
-    max_open_conns: 100
+    max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 

docs/CODE_STYLE.md

@@ -2,13 +2,13 @@
 
 In addition to standard Go code style (`gofmt`, `goimports`), we use `golangci-lint`
 to run a number of linters, the exact list can be found under linters in [.golangci.yml](.golangci.yml).
-[Installation](https://github.com/golangci/golangci-lint#install) and [Editor
+[Installation](https://github.com/golangci/golangci-lint#install-golangci-lint) and [Editor
-Integration](https://github.com/golangci/golangci-lint#editor-integration) for
+Integration](https://golangci-lint.run/usage/integrations/#editor-integration) for
 it can be found in the readme of golangci-lint.
 
 For rare cases where a linter is giving a spurious warning, it can be disabled
 for that line or statement using a [comment
-directive](https://github.com/golangci/golangci-lint#nolint), e.g.  `var
+directive](https://golangci-lint.run/usage/false-positives/#nolint), e.g.  `var
 bad_name int //nolint:golint,unused`. This should be used sparingly and only
 when its clear that the lint warning is spurious.
 

docs/CONTRIBUTING.md

@@ -37,7 +37,7 @@ If a job fails, click the "details" button and you should be taken to the job's
 logs.
 
 ![Click the details button on the failing build
-step](docs/images/details-button-location.jpg)
+step](https://raw.githubusercontent.com/matrix-org/dendrite/master/docs/images/details-button-location.jpg)
 
 Scroll down to the failing step and you should see some log output. Scan the
 logs until you find what it's complaining about, fix it, submit a new commit,

docs/FAQ.md

@@ -0,0 +1,64 @@
+# Frequently Asked Questions
+
+### Is Dendrite stable?
+
+Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead.
+
+### Is Dendrite feature-complete?
+
+No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the readme at the root of the repository for more information.
+
+### Is there a migration path from Synapse to Dendrite?
+
+No, not at present. There will be in the future when Dendrite reaches version 1.0.
+
+### Should I run a monolith or a polylith deployment?
+
+Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines.
+
+### I've installed Dendrite but federation isn't working
+
+Check the [Federation Tester](https://federationtester.matrix.org). You need at least:
+
+* A valid DNS name
+* A valid TLS certificate for that DNS name
+* Either DNS SRV records or well-known files
+
+### Does Dendrite work with my favourite client?
+
+It should do, although we are aware of some minor issues:
+
+* **Element Android**: registration does not work, but logging in with an existing account does
+* **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
+
+### Does Dendrite support push notifications?
+
+No, not yet. This is a planned feature.
+
+### Does Dendrite support application services/bridges?
+
+Possibly - Dendrite does have some application service support but it is not well tested. Please let us know by raising a GitHub issue if you try it and run into problems.
+
+### Is it possible to prevent communication with the outside world?
+
+Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file. 
+
+### Should I use PostgreSQL or SQLite for my databases?
+
+Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users. 
+
+### Dendrite is using a lot of CPU
+
+Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue.
+
+### Dendrite is using a lot of RAM
+
+A lot of users report that Dendrite is using a lot of RAM, sometimes even gigabytes of it. This is usually due to Go's allocator behaviour, which tries to hold onto allocated memory until the operating system wants to reclaim it for something else. This can make the memory usage look significantly inflated in tools like `top`/`htop` when actually most of that memory is not really in use at all.
+
+If you want to prevent this behaviour so that the Go runtime releases memory normally, start Dendrite using the `GODEBUG=madvdontneed=1` environment variable. It is also expected that the allocator behaviour will be changed again in Go 1.16 so that it does not hold onto memory unnecessarily in this way.
+
+If you are running with `GODEBUG=madvdontneed=1` and still see hugely inflated memory usage then that's quite possibly a bug - please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue.
+
+### Dendrite is running out of PostgreSQL database connections
+
+You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!

docs/INSTALL.md

@@ -12,6 +12,8 @@ Dendrite can be run in one of two configurations:
    lightweight implementation called [Naffka](https://github.com/matrix-org/naffka). This
    will usually be the preferred model for low-volume, low-user or experimental deployments.
 
+For most deployments, it is **recommended to run in monolith mode with PostgreSQL databases**.
+
 Regardless of whether you are running in polylith or monolith mode, each Dendrite component that
 requires storage has its own database. Both Postgres and SQLite are supported and can be
 mixed-and-matched across components as needed in the configuration file.
@@ -30,23 +32,9 @@ If you want to run a polylith deployment, you also need:
 
 * Apache Kafka 0.10.2+
 
-## Building up a monolith deploment
+Please note that Kafka is **not required** for a monolith deployment.
 
-Start by cloning the code:
+## Building Dendrite
-
-```bash
-git clone https://github.com/matrix-org/dendrite
-cd dendrite
-```
-
-Then build it:
-
-```bash
-go build -o bin/dendrite-monolith-server ./cmd/dendrite-monolith-server
-go build -o bin/generate-keys ./cmd/generate-keys
-```
-
-## Building up a polylith deployment
 
 Start by cloning the code:
 
@@ -61,6 +49,8 @@ Then build it:
 ./build.sh
 ```
 
+## Install Kafka (polylith only)
+
 Install and start Kafka (c.f. [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh)):
 
 ```bash
@@ -90,15 +80,9 @@ brew services start kafka
 
 ## Configuration
 
-### SQLite database setup
+### PostgreSQL database setup
 
-Dendrite can use the built-in SQLite database engine for small setups.
+Assuming that PostgreSQL 9.6 (or later) is installed:
-The SQLite databases do not need to be pre-built - Dendrite will
-create them automatically at startup.
-
-### Postgres database setup
-
-Assuming that Postgres 9.6 (or later) is installed:
 
 * Create role, choosing a new password when prompted:
 
@@ -106,7 +90,23 @@ Assuming that Postgres 9.6 (or later) is installed:
   sudo -u postgres createuser -P dendrite
   ```
 
-* Create the component databases:
+At this point you have a choice on whether to run all of the Dendrite
+components from a single database, or for each component to have its
+own database. For most deployments, running from a single database will
+be sufficient, although you may wish to separate them if you plan to
+split out the databases across multiple machines in the future.
+
+On macOS, omit `sudo -u postgres` from the below commands.
+
+* If you want to run all Dendrite components from a single database:
+
+  ```bash
+    sudo -u postgres createdb -O dendrite dendrite
+  ```
+
+  ... in which case your connection string will look like `postgres://user:pass@database/dendrite`.
+
+* If you want to run each Dendrite component with its own database:
 
   ```bash
   for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do
@@ -114,22 +114,40 @@ Assuming that Postgres 9.6 (or later) is installed:
   done
   ```
 
-(On macOS, omit `sudo -u postgres` from the above commands.)
+  ... in which case your connection string will look like `postgres://user:pass@database/dendrite_componentname`.
+
+### SQLite database setup
+
+**WARNING:** SQLite is suitable for small experimental deployments only and should not be used in production - use PostgreSQL instead for any user-facing federating installation!
+
+Dendrite can use the built-in SQLite database engine for small setups.
+The SQLite databases do not need to be pre-built - Dendrite will
+create them automatically at startup.
 
 ### Server key generation
 
-Each Dendrite server requires unique server keys.
+Each Dendrite installation requires:
 
-In order for an instance to federate correctly, you should have a valid
+* A unique Matrix signing private key
-certificate issued by a trusted authority, and private key to match. If you
+* A valid and trusted TLS certificate and private key
-don't and just want to test locally, generate the self-signed SSL certificate
+
-for federation and the server signing key:
+To generate a Matrix signing private key:
 
 ```bash
-./bin/generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
+./bin/generate-keys --private-key matrix_key.pem
 ```
 
-If you have server keys from an older synapse instance, 
+**WARNING:** Make sure take a safe backup of this key! You will likely need it if you want to reinstall Dendrite, or
+any other Matrix homeserver, on the same domain name in the future. If you lose this key, you may have trouble joining
+federated rooms.
+
+For testing, you can generate a self-signed certificate and key, although this will not work for public federation:
+
+```bash
+./bin/generate-keys --tls-cert server.crt --tls-key server.key
+```
+
+If you have server keys from an older Synapse instance,
 [convert them](serverkeyformat.md#converting-synapse-keys) to Dendrite's PEM
 format and configure them as `old_private_keys` in your config.
 
@@ -140,9 +158,11 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th
 * The `server_name` entry to reflect the hostname of your Dendrite server
 * The `database` lines with an updated connection string based on your
   desired setup, e.g. replacing `database` with the name of the database:
-  * For Postgres: `postgres://dendrite:password@localhost/database`
+  * For Postgres: `postgres://dendrite:password@localhost/database`, e.g.
-  * For SQLite on disk: `file:component.db` or `file:///path/to/component.db`
+    * `postgres://dendrite:password@localhost/dendrite_userapi_account` to connect to PostgreSQL with SSL/TLS
-  * Postgres and SQLite can be mixed and matched.
+    * `postgres://dendrite:password@localhost/dendrite_userapi_account?sslmode=disable` to connect to PostgreSQL without SSL/TLS
+  * For SQLite on disk: `file:component.db` or `file:///path/to/component.db`, e.g. `file:userapi_account.db`
+  * Postgres and SQLite can be mixed and matched on different components as desired.
 * The `use_naffka` option if using Naffka in a monolith deployment
 
 There are other options which may be useful so review them all. In particular,
@@ -152,7 +172,7 @@ help to improve reliability considerably by allowing your homeserver to fetch
 public keys for dead homeservers from somewhere else.
 
 **WARNING:** Dendrite supports running all components from the same database in
-Postgres mode, but this is **NOT** a supported configuration with SQLite. When
+PostgreSQL mode, but this is **NOT** a supported configuration with SQLite. When
 using SQLite, all components **MUST** use their own database file.
 
 ## Starting a monolith server
@@ -164,8 +184,14 @@ Be sure to update the database username and password if needed.
 
 The monolith server can be started as shown below. By default it listens for
 HTTP connections on port 8008, so you can configure your Matrix client to use
-`http://localhost:8008` as the server. If you set `--tls-cert` and `--tls-key`
+`http://servername:8008` as the server:
-as shown below, it will also listen for HTTPS connections on port 8448.
+
+```bash
+./bin/dendrite-monolith-server
+```
+
+If you set `--tls-cert` and `--tls-key` as shown below, it will also listen
+for HTTPS connections on port 8448:
 
 ```bash
 ./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key
@@ -289,4 +315,3 @@ amongst other things.
 ```bash
 ./bin/dendrite-polylith-multi --config=dendrite.yaml userapi
 ```
-

docs/PROFILING.md

@@ -0,0 +1,89 @@
+# Profiling Dendrite
+
+If you are running into problems with Dendrite using excessive resources (e.g. CPU or RAM) then you can use the profiler to work out what is happening.
+
+Dendrite contains an embedded profiler called `pprof`, which is a part of the standard Go toolchain. 
+
+## Enable the profiler
+
+To enable the profiler, start Dendrite with the `PPROFLISTEN` environment variable. This variable specifies which address and port to listen on, e.g.
+
+```
+PPROFLISTEN=localhost:65432 ./bin/dendrite-monolith-server ...
+```
+
+If pprof has been enabled successfully, a log line at startup will show that pprof is listening:
+
+```
+WARN[2020-12-03T13:32:33.669405000Z] [/Users/neilalexander/Desktop/dendrite/internal/log.go:87] SetupPprof
+	 Starting pprof on localhost:65432
+```
+
+All examples from this point forward assume `PPROFLISTEN=localhost:65432` but you may need to adjust as necessary for your setup.
+
+## Profiling CPU usage
+
+To examine where CPU time is going, you can call the `profile` endpoint:
+
+```
+http://localhost:65432/debug/pprof/profile?seconds=30
+```
+
+The profile will run for the specified number of `seconds` and then will produce a result.
+
+### Examine a profile using the Go toolchain
+
+If you have Go installed and want to explore the profile, you can invoke `go tool pprof` to start the profile directly. The `-http=` parameter will instruct `go tool pprof` to start a web server providing a view of the captured profile:
+
+```
+go tool pprof -http=localhost:23456 http://localhost:65432/debug/pprof/profile?seconds=30
+```
+
+You can then visit `http://localhost:23456` in your web browser to see a visual representation of the profile. Particularly usefully, in the "View" menu, you can select "Flame Graph" to see a proportional interactive graph of CPU usage.
+
+### Download a profile to send to someone else
+
+If you don't have the Go tools installed but just want to capture the profile to send to someone else, you can instead use `curl` to download the profiler results:
+
+```
+curl -O http://localhost:65432/debug/pprof/profile?seconds=30
+```
+
+This will block for the specified number of seconds, capturing information about what Dendrite is doing, and then produces a `profile` file, which you can send onward.
+
+## Profiling memory usage
+
+To examine where memory usage is going, you can call the `heap` endpoint:
+
+```
+http://localhost:65432/debug/pprof/heap
+```
+
+The profile will return almost instantly.
+
+### Examine a profile using the Go toolchain
+
+If you have Go installed and want to explore the profile, you can invoke `go tool pprof` to start the profile directly. The `-http=` parameter will instruct `go tool pprof` to start a web server providing a view of the captured profile:
+
+```
+go tool pprof -http=localhost:23456 http://localhost:65432/debug/pprof/heap
+```
+
+You can then visit `http://localhost:23456` in your web browser to see a visual representation of the profile. The "Sample" menu lets you select between four different memory profiles:
+
+* `inuse_space`: Shows how much actual heap memory is allocated per function (this is generally the most useful profile when diagnosing high memory usage)
+* `inuse_objects`: Shows how many heap objects are allocated per function
+* `alloc_space`: Shows how much memory has been allocated per function (although that memory may have since been deallocated)
+* `alloc_objects`: Shows how many allocations have been made per function (although that memory may have since been deallocated)
+
+Also in the "View" menu, you can select "Flame Graph" to see a proportional interactive graph of the memory usage.
+
+### Download a profile to send to someone else
+
+If you don't have the Go tools installed but just want to capture the profile to send to someone else, you can instead use `curl` to download the profiler results:
+
+```
+curl -O http://localhost:65432/debug/pprof/heap
+```
+
+This will almost instantly produce a `heap` file, which you can send onward.

docs/hiawatha/monolith-sample.conf

@@ -0,0 +1,17 @@
+# Depending on which port is used for federation (.well-known/matrix/server or SRV record),
+# ensure there's a binding for that port in the configuration. Replace "FEDPORT" with port
+# number, (e.g. "8448"), and "IPV4" with your server's ipv4 address (separate binding for
+# each ip address, e.g. if you use both ipv4 and ipv6 addresses).
+
+Binding {
+        Port = FEDPORT
+        Interface = IPV4
+        TLScertFile = /path/to/fullchainandprivkey.pem
+}
+
+VirtualHost {
+        ...
+        ReverseProxy = /_matrix http://localhost:8008 600
+        ...
+
+}

docs/hiawatha/polylith-sample.conf

@@ -0,0 +1,28 @@
+# Depending on which port is used for federation (.well-known/matrix/server or SRV record),
+# ensure there's a binding for that port in the configuration. Replace "FEDPORT" with port
+# number, (e.g. "8448"), and "IPV4" with your server's ipv4 address (separate binding for
+# each ip address, e.g. if you use both ipv4 and ipv6 addresses).
+
+Binding {
+        Port = FEDPORT
+        Interface = IPV4
+        TLScertFile = /path/to/fullchainandprivkey.pem
+}
+
+
+VirtualHost {
+        ...
+        # route requests to:
+        # /_matrix/client/.*/sync
+        # /_matrix/client/.*/user/{userId}/filter
+        # /_matrix/client/.*/user/{userId}/filter/{filterID}
+        # /_matrix/client/.*/keys/changes
+        # /_matrix/client/.*/rooms/{roomId}/messages
+        # to sync_api
+        ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 600
+        ReverseProxy = /_matrix/client http://localhost:8071 600
+        ReverseProxy = /_matrix/federation http://localhost:8072 600
+        ReverseProxy = /_matrix/key http://localhost:8072 600
+        ReverseProxy = /_matrix/media http://localhost:8074 600
+        ...
+}

docs/nginx/monolith-sample.conf

@@ -1,5 +1,6 @@
 server {
-    listen 443 ssl;
+    listen 443 ssl; # IPv4
+    listen [::]:443; # IPv6
     server_name my.hostname.com;
 
     ssl_certificate /path/to/fullchain.pem;

docs/nginx/polylith-sample.conf

@@ -1,5 +1,6 @@
 server {
-    listen 443 ssl;
+    listen 443 ssl; # IPv4
+    listen [::]:443; # IPv6
     server_name my.hostname.com;
 
     ssl_certificate /path/to/fullchain.pem;

docs/systemd/monolith-example.service

@@ -5,6 +5,7 @@ After=network.target
 After=postgresql.service
 
 [Service]
+Environment=GODEBUG=madvdontneed=1
 RestartSec=2s
 Type=simple
 User=dendrite

docs/sytest.md

@@ -85,6 +85,7 @@ Set up the database:
 
 ```sh
 sudo -u postgres psql -c "CREATE USER dendrite PASSWORD 'itsasecret'"
+sudo -u postgres psql -c "ALTER USER dendrite CREATEDB"
 for i in dendrite0 dendrite1 sytest_template; do sudo -u postgres psql -c "CREATE DATABASE $i OWNER dendrite;"; done
 mkdir -p "server-0"
 cat > "server-0/database.yaml" << EOF

eduserver/api/input.go

@@ -59,6 +59,22 @@ type InputSendToDeviceEventRequest struct {
 // InputSendToDeviceEventResponse is a response to InputSendToDeviceEventRequest
 type InputSendToDeviceEventResponse struct{}
 
+type InputReceiptEvent struct {
+	UserID    string                      `json:"user_id"`
+	RoomID    string                      `json:"room_id"`
+	EventID   string                      `json:"event_id"`
+	Type      string                      `json:"type"`
+	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
+}
+
+// InputReceiptEventRequest is a request to EDUServerInputAPI
+type InputReceiptEventRequest struct {
+	InputReceiptEvent InputReceiptEvent `json:"input_receipt_event"`
+}
+
+// InputReceiptEventResponse is a response to InputReceiptEventRequest
+type InputReceiptEventResponse struct{}
+
 // EDUServerInputAPI is used to write events to the typing server.
 type EDUServerInputAPI interface {
 	InputTypingEvent(
@@ -72,4 +88,10 @@ type EDUServerInputAPI interface {
 		request *InputSendToDeviceEventRequest,
 		response *InputSendToDeviceEventResponse,
 	) error
+
+	InputReceiptEvent(
+		ctx context.Context,
+		request *InputReceiptEventRequest,
+		response *InputReceiptEventResponse,
+	) error
 }

eduserver/api/output.go

@@ -49,3 +49,39 @@ type OutputSendToDeviceEvent struct {
 	DeviceID string `json:"device_id"`
 	gomatrixserverlib.SendToDeviceEvent
 }
+
+type ReceiptEvent struct {
+	UserID    string                      `json:"user_id"`
+	RoomID    string                      `json:"room_id"`
+	EventID   string                      `json:"event_id"`
+	Type      string                      `json:"type"`
+	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
+}
+
+// OutputReceiptEvent is an entry in the receipt output kafka log
+type OutputReceiptEvent struct {
+	UserID    string                      `json:"user_id"`
+	RoomID    string                      `json:"room_id"`
+	EventID   string                      `json:"event_id"`
+	Type      string                      `json:"type"`
+	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
+}
+
+// Helper structs for receipts json creation
+type ReceiptMRead struct {
+	User map[string]ReceiptTS `json:"m.read"`
+}
+
+type ReceiptTS struct {
+	TS gomatrixserverlib.Timestamp `json:"ts"`
+}
+
+// FederationSender output
+type FederationReceiptMRead struct {
+	User map[string]FederationReceiptData `json:"m.read"`
+}
+
+type FederationReceiptData struct {
+	Data     ReceiptTS `json:"data"`
+	EventIDs []string  `json:"event_ids"`
+}

eduserver/api/wrapper.go

@@ -67,3 +67,22 @@ func SendToDevice(
 	response := InputSendToDeviceEventResponse{}
 	return eduAPI.InputSendToDeviceEvent(ctx, &request, &response)
 }
+
+// SendReceipt sends a receipt event to EDU Server
+func SendReceipt(
+	ctx context.Context,
+	eduAPI EDUServerInputAPI, userID, roomID, eventID, receiptType string,
+	timestamp gomatrixserverlib.Timestamp,
+) error {
+	request := InputReceiptEventRequest{
+		InputReceiptEvent: InputReceiptEvent{
+			UserID:    userID,
+			RoomID:    roomID,
+			EventID:   eventID,
+			Type:      receiptType,
+			Timestamp: timestamp,
+		},
+	}
+	response := InputReceiptEventResponse{}
+	return eduAPI.InputReceiptEvent(ctx, &request, &response)
+}

eduserver/cache/cache.go

@@ -113,19 +113,6 @@ func (t *EDUCache) AddTypingUser(
 	return t.GetLatestSyncPosition()
 }
 
-// AddSendToDeviceMessage increases the sync position for
-// send-to-device updates.
-// Returns the sync position before update, as the caller
-// will use this to record the current stream position
-// at the time that the send-to-device message was sent.
-func (t *EDUCache) AddSendToDeviceMessage() int64 {
-	t.Lock()
-	defer t.Unlock()
-	latestSyncPosition := t.latestSyncPosition
-	t.latestSyncPosition++
-	return latestSyncPosition
-}
-
 // addUser with mutex lock & replace the previous timer.
 // Returns the latest typing sync position after update.
 func (t *EDUCache) addUser(

eduserver/eduserver.go

@@ -22,9 +22,9 @@ import (
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/eduserver/input"
 	"github.com/matrix-org/dendrite/eduserver/inthttp"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/internal/setup/kafka"
+	"github.com/matrix-org/dendrite/setup/kafka"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
@@ -49,8 +49,9 @@ func NewInternalAPI(
 		Cache:                        eduCache,
 		UserAPI:                      userAPI,
 		Producer:                     producer,
-		OutputTypingEventTopic:       string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent)),
+		OutputTypingEventTopic:       cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent),
-		OutputSendToDeviceEventTopic: string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent)),
+		OutputSendToDeviceEventTopic: cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent),
+		OutputReceiptEventTopic:      cfg.Matrix.Kafka.TopicFor(config.TopicOutputReceiptEvent),
 		ServerName:                   cfg.Matrix.ServerName,
 	}
 }

eduserver/input/input.go

@@ -37,6 +37,8 @@ type EDUServerInputAPI struct {
 	OutputTypingEventTopic string
 	// The kafka topic to output new send to device events to.
 	OutputSendToDeviceEventTopic string
+	// The kafka topic to output new receipt events to
+	OutputReceiptEventTopic string
 	// kafka producer
 	Producer sarama.SyncProducer
 	// Internal user query API
@@ -173,3 +175,31 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e
 
 	return nil
 }
+
+// InputReceiptEvent implements api.EDUServerInputAPI
+// TODO: Intelligently batch requests sent by the same user (e.g wait a few milliseconds before emitting output events)
+func (t *EDUServerInputAPI) InputReceiptEvent(
+	ctx context.Context,
+	request *api.InputReceiptEventRequest,
+	response *api.InputReceiptEventResponse,
+) error {
+	logrus.WithFields(logrus.Fields{}).Infof("Producing to topic '%s'", t.OutputReceiptEventTopic)
+	output := &api.OutputReceiptEvent{
+		UserID:    request.InputReceiptEvent.UserID,
+		RoomID:    request.InputReceiptEvent.RoomID,
+		EventID:   request.InputReceiptEvent.EventID,
+		Type:      request.InputReceiptEvent.Type,
+		Timestamp: request.InputReceiptEvent.Timestamp,
+	}
+	js, err := json.Marshal(output)
+	if err != nil {
+		return err
+	}
+	m := &sarama.ProducerMessage{
+		Topic: t.OutputReceiptEventTopic,
+		Key:   sarama.StringEncoder(request.InputReceiptEvent.RoomID + ":" + request.InputReceiptEvent.UserID),
+		Value: sarama.ByteEncoder(js),
+	}
+	_, _, err = t.Producer.SendMessage(m)
+	return err
+}

eduserver/inthttp/client.go

@@ -14,6 +14,7 @@ import (
 const (
 	EDUServerInputTypingEventPath       = "/eduserver/input"
 	EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice"
+	EDUServerInputReceiptEventPath      = "/eduserver/receipt"
 )
 
 // NewEDUServerClient creates a EDUServerInputAPI implemented by talking to a HTTP POST API.
@@ -54,3 +55,16 @@ func (h *httpEDUServerInputAPI) InputSendToDeviceEvent(
 	apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
+
+// InputSendToDeviceEvent implements EDUServerInputAPI
+func (h *httpEDUServerInputAPI) InputReceiptEvent(
+	ctx context.Context,
+	request *api.InputReceiptEventRequest,
+	response *api.InputReceiptEventResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "InputReceiptEventPath")
+	defer span.Finish()
+
+	apiURL := h.eduServerURL + EDUServerInputReceiptEventPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

eduserver/inthttp/server.go

@@ -38,4 +38,17 @@ func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(EDUServerInputReceiptEventPath,
+		httputil.MakeInternalAPI("inputReceiptEvent", func(req *http.Request) util.JSONResponse {
+			var request api.InputReceiptEventRequest
+			var response api.InputReceiptEventResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := t.InputReceiptEvent(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }