Merge branch 'master' into develop

# Conflicts:
#	go.mod
#	go.sum

.github/workflows/wasm.yml

@@ -0,0 +1,49 @@
+name: WebAssembly
+
+on:
+    push:
+    pull_request:
+
+jobs:
+    test:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+
+            - name: Install Go
+              uses: actions/setup-go@v2
+              with:
+                  go-version: 1.16.5
+
+            - uses: actions/cache@v2
+              with:
+                  path: |
+                      ~/.cache/go-build
+                      ~/go/pkg/mod
+                  key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+                  restore-keys: |
+                      ${{ runner.os }}-go-
+
+            - name: Install Node
+              uses: actions/setup-node@v2
+              with:
+                  node-version: 14
+
+            - uses: actions/cache@v2
+              with:
+                  path: ~/.npm
+                  key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+                  restore-keys: |
+                      ${{ runner.os }}-node-
+
+            - name: Reconfigure Git to use HTTPS auth for repo packages
+              run: >
+                  git config --global url."https://github.com/".insteadOf
+                  ssh://git@github.com/
+
+            - name: Install test dependencies
+              working-directory: ./test/wasm
+              run: npm ci
+
+            - name: Test
+              run: ./test-dendritejs.sh

.gitignore

@@ -3,6 +3,9 @@
 # Hidden files
 .*
 
+# Allow GitHub config
+!.github
+
 # Downloads
 /.downloads
 
@@ -36,6 +39,7 @@ _testmain.go
 *.exe
 *.test
 *.prof
+*.wasm
 
 # Generated keys
 *.pem
@@ -53,3 +57,6 @@ dendrite.yaml
 
 # Generated code
 cmd/dendrite-demo-yggdrasil/embed/fs*.go
+
+# Test dependencies
+test/wasm/node_modules

CHANGES.md

@@ -1,5 +1,82 @@
 # Changelog
 
+## Dendrite 0.5.0 (2021-08-24)
+
+### Features
+
+* Support for serverside key backups has been added, allowing your E2EE keys to be backed up and to be restored after logging out or when logging in from a new device
+* Experimental support for cross-signing has been added, allowing verifying your own device keys and verifying other user's public keys
+* Dendrite can now send logs to a TCP syslog server by using the `syslog` logger type (contributed by [sambhavsaggi](https://github.com/sambhavsaggi))
+* Go 1.15 is now the minimum supported version for Dendrite
+
+### Fixes
+
+* Device keys are now cleaned up from the keyserver when the user API removes a device session
+* The `M_ROOM_IN_USE` error code is now returned when a room alias is already taken (contributed by [nivekuil](https://github.com/nivekuil))
+* A bug in the state storage migration has been fixed where room create events had incorrect state snapshots
+* A bug when deactivating accounts caused by only reading the deprecated username field has been fixed
+
+## Dendrite 0.4.1 (2021-07-26)
+
+### Features
+
+* Support for room version 7 has been added
+* Key notary support is now more complete, allowing Dendrite to be used as a notary server for looking up signing keys
+* State resolution v2 performance has been optimised further by caching the create event, power levels and join rules in memory instead of parsing them repeatedly
+* The media API now handles cases where the maximum file size is configured to be less than 0 for unlimited size
+* The `initial_state` in a `/createRoom` request is now respected when creating a room
+* Code paths for checking if servers are joined to rooms have been optimised significantly
+
+### Fixes
+
+* A bug resulting in `cannot xref null state block with snapshot` during the new state storage migration has been fixed
+* Invites are now retired correctly when rejecting an invite from a remote server which is no longer reachable
+* The DNS cache `cache_lifetime` option is now handled correctly (contributed by [S7evinK](https://github.com/S7evinK))
+* Invalid events in a room join response are now dropped correctly, rather than failing the entire join
+* The `prev_state` of an event will no longer be populated incorrectly to the state of the current event
+* Receiving an invite to an unsupported room version will now correctly return the `M_UNSUPPORTED_ROOM_VERSION` error code instead of `M_BAD_JSON` (contributed by [meenal06](https://github.com/meenal06))
+
+## Dendrite 0.4.0 (2021-07-12)
+
+### Features
+
+* All-new state storage in the roomserver, which dramatically reduces disk space utilisation
+  * State snapshots and blocks are now aggressively deduplicated and reused wherever possible, with state blocks being reduced by up to 15x and snapshot references being reduced up to 2x
+  * Dendrite will upgrade to the new state storage automatically on the first run after upgrade, although this may take some time depending on the size of the state storage
+* Appservice support has been improved significantly, with many bridges now working correctly with Dendrite
+  * Events are now correctly sent to appservices based on room memberships
+  * Aliases and namespaces are now handled correctly, calling the appservice to query for aliases as needed
+  * Appservice user registrations are no longer being subject to incorrect validation checks
+* Shared secret registration has now been implemented correctly
+* The roomserver input API implements a new queuing system to reduce backpressure across rooms
+* Checking if the local server is in a room has been optimised substantially, reducing CPU usage
+* State resolution v2 has been optimised further by improving the power level checks, reducing CPU usage
+* The federation API `/send` endpoint now deduplicates missing auth and prev events more aggressively to reduce memory usage
+* The federation API `/send` endpoint now uses workers to reduce backpressure across rooms
+* The bcrypt cost for password storage is now configurable with the `user_api.bcrypt_cost` option
+* The federation API will now use significantly less memory when calling `/get_missing_events`
+* MSC2946 Spaces endpoints have been updated to stable endpoint naming
+* The media API can now be configured without a maximum file size
+* A new `dendrite-upgrade-test` test has been added for verifying database schema upgrades across versions
+* Added Prometheus metrics for roomserver backpressure, excessive device list updates and federation API event processing summaries
+* Sentry support has been added for error reporting
+
+### Fixes
+
+* Removed the legacy `/v1` register endpoint. Dendrite only implements `/r0` of the CS API, and the legacy `/v1` endpoint had implementation errors which made it possible to bypass shared secret registration (thanks to Jakob Varmose Bentzen for reporting this)
+* Attempting to register an account that already exists now returns a sensible error code rather than a HTTP 500
+* Dendrite will no longer attempt to `/make_join` with itself if listed in the request `server_names`
+* `/sync` will no longer return immediately if there is nothing to sync, which happened particularly with new accounts, causing high CPU usage
+* Malicious media uploads can no longer exhaust all available memory (contributed by [S7evinK](https://github.com/S7evinK))
+* Selecting one-time keys from the database has been optimised (contributed by [S7evinK](https://github.com/S7evinK))
+* The return code when trying to fetch missing account data has been fixed (contributed by [adamgreig](https://github.com/adamgreig))
+* Dendrite will no longer attempt to use `/make_leave` over federation when rejecting a local invite
+* A panic has been fixed in `QueryMembershipsForRoom`
+* A panic on duplicate membership events has been fixed in the federation sender
+* A panic has been fixed in in `IsInterestedInRoomID` (contributed by [bodqhrohro](https://github.com/bodqhrohro))
+* A panic in the roomserver has been fixed when handling empty state sets
+* A panic in the federation API has been fixed when handling cached events
+
 ## Dendrite 0.3.11 (2021-03-02)
 
 ### Fixes

README.md

@@ -1,7 +1,7 @@
 # Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org)
 
 Dendrite is a second-generation Matrix homeserver written in Go.
-It intends to provide an **efficient**, **reliable** and **scalable** alternative to Synapse:
+It intends to provide an **efficient**, **reliable** and **scalable** alternative to [Synapse](https://github.com/matrix-org/synapse):
  - Efficient: A small memory footprint with better baseline performance than an out-of-the-box Synapse.
  - Reliable: Implements the Matrix specification as written, using the
    [same test suite](https://github.com/matrix-org/sytest) as Synapse as well as
@@ -31,7 +31,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 
 ## Requirements
 
-To build Dendrite, you will need Go 1.13 or later. 
+To build Dendrite, you will need Go 1.15 or later. 
 
 For a usable federating Dendrite deployment, you will also need:
 - A domain name (or subdomain) 

appservice/query/query.go

@@ -51,6 +51,10 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 		if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
 			// The full path to the rooms API, includes hs token
 			URL, err := url.Parse(appservice.URL + roomAliasExistsPath)
+			if err != nil {
+				return err
+			}
+
 			URL.Path += request.Alias
 			apiURL := URL.String() + "?access_token=" + appservice.HSToken
 
@@ -114,6 +118,9 @@ func (a *AppServiceQueryAPI) UserIDExists(
 		if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
 			// The full path to the rooms API, includes hs token
 			URL, err := url.Parse(appservice.URL + userIDExistsPath)
+			if err != nil {
+				return err
+			}
 			URL.Path += request.UserID
 			apiURL := URL.String() + "?access_token=" + appservice.HSToken
 

appservice/storage/sqlite3/storage.go

@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
-	_ "github.com/mattn/go-sqlite3"
 )
 
 // Database stores events intended to be later sent to application services

appservice/storage/sqlite3/txn_id_counter_table.go

@@ -32,14 +32,18 @@ INSERT OR IGNORE INTO appservice_counters (name, last_id) VALUES('txn_id', 1);
 `
 
 const selectTxnIDSQL = `
-  SELECT last_id FROM appservice_counters WHERE name='txn_id';
-  UPDATE appservice_counters SET last_id=last_id+1 WHERE name='txn_id';
+  SELECT last_id FROM appservice_counters WHERE name='txn_id'
+`
+
+const updateTxnIDSQL = `
+  UPDATE appservice_counters SET last_id=last_id+1 WHERE name='txn_id'
 `
 
 type txnStatements struct {
 	db              *sql.DB
 	writer          sqlutil.Writer
 	selectTxnIDStmt *sql.Stmt
+	updateTxnIDStmt *sql.Stmt
 }
 
 func (s *txnStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
@@ -54,6 +58,10 @@ func (s *txnStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
 		return
 	}
 
+	if s.updateTxnIDStmt, err = db.Prepare(updateTxnIDSQL); err != nil {
+		return
+	}
+
 	return
 }
 
@@ -63,6 +71,11 @@ func (s *txnStatements) selectTxnID(
 ) (txnID int, err error) {
 	err = s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
 		err := s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID)
+		if err != nil {
+			return err
+		}
+
+		_, err = s.updateTxnIDStmt.ExecContext(ctx)
 		return err
 	})
 	return

are-we-synapse-yet.list

@@ -17,6 +17,13 @@ reg POST /register rejects registration of usernames with '£'
 reg POST /register rejects registration of usernames with 'é'
 reg POST /register rejects registration of usernames with '\n'
 reg POST /register rejects registration of usernames with '''
+reg POST /register allows registration of usernames with 'q'
+reg POST /register allows registration of usernames with '3'
+reg POST /register allows registration of usernames with '.'
+reg POST /register allows registration of usernames with '_'
+reg POST /register allows registration of usernames with '='
+reg POST /register allows registration of usernames with '-'
+reg POST /register allows registration of usernames with '/'
 reg POST /r0/admin/register with shared secret 
 reg POST /r0/admin/register admin with shared secret 
 reg POST /r0/admin/register with shared secret downcases capitals 
@@ -76,7 +83,7 @@ rst GET /rooms/:room_id/state/m.room.topic gets topic
 rst GET /rooms/:room_id/state fetches entire room state 
 crm POST /createRoom with creation content
 ali PUT /directory/room/:room_alias creates alias
-nsp GET /rooms/:room_id/aliases lists aliases 
+ali GET /rooms/:room_id/aliases lists aliases 
 jon POST /rooms/:room_id/join can join a room
 jon POST /join/:room_alias can join a room
 jon POST /join/:room_id can join a room
@@ -95,6 +102,7 @@ typ Typing notifications don't leak (3 subtests)
 rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels
 rst PUT /rooms/:room_id/state/m.room.power_levels can set levels
 rst PUT power_levels should not explode if the old power levels were empty
+rst Users cannot set notifications powerlevel higher than their own (2 subtests)
 rst Both GET and PUT work
 rct POST /rooms/:room_id/receipt can create receipts 
 red POST /rooms/:room_id/read_markers can create read marker
@@ -175,7 +183,7 @@ ali Users with sufficient power-level can delete other's aliases
 ali Can delete canonical alias 
 ali Alias creators can delete alias with no ops 
 ali Alias creators can delete canonical alias with no ops 
-ali Only room members can list aliases of a room 
+msc Only room members can list aliases of a room 
 inv Can invite users to invite-only rooms 
 inv Uninvited users cannot join the room 
 inv Invited user can reject invite 
@@ -353,6 +361,7 @@ syn Syncing a new room with a large timeline limit isn't limited
 syn A full_state incremental update returns only recent timeline 
 syn A prev_batch token can be used in the v1 messages API 
 syn A next_batch token can be used in the v1 messages API 
+syn A prev_batch token from incremental sync can be used in the v1 messages API
 syn User sees their own presence in a sync 
 syn User is offline if they set_presence=offline in their sync
 syn User sees updates to presence from other users in the incremental sync. 
@@ -574,6 +583,7 @@ fqu Outbound federation can query profile data
 fqu Inbound federation can query profile data
 fqu Outbound federation can query room alias directory
 fqu Inbound federation can query room alias directory
+fsj Membership event with an invalid displayname in the send_join response should not cause room join to fail
 fsj Outbound federation can query v1 /send_join 
 fsj Outbound federation can query v2 /send_join
 fmj Outbound federation passes make_join failures through to the client 
@@ -596,7 +606,7 @@ fsj Inbound: send_join rejects invalid JSON for room version 6
 fed Outbound federation can send events
 fed Inbound federation can receive events 
 fed Inbound federation can receive redacted events 
-fed Ephemeral messages received from servers are correctly expired 
+msc Ephemeral messages received from servers are correctly expired 
 fed Events whose auth_events are in the wrong room do not mess up the room state 
 fed Inbound federation can return events 
 fed Inbound federation redacts events from erased users 
@@ -743,6 +753,10 @@ nsp Set group joinable and join it
 nsp Group is not joinable by default 
 nsp Group is joinable over federation 
 nsp Room is transitioned on local and remote groups upon room upgrade 
+nsp POST /_synapse/admin/v1/register with shared secret
+nsp POST /_synapse/admin/v1/register admin with shared secret
+nsp POST /_synapse/admin/v1/register with shared secret downcases capitals
+nsp POST /_synapse/admin/v1/register with shared secret disallows symbols
 3pd Can bind 3PID via home server 
 3pd Can bind and unbind 3PID via homeserver 
 3pd Can unbind 3PID via homeserver when bound out of band 
@@ -859,8 +873,14 @@ jso Invalid JSON special values
 inv Can invite users to invite-only rooms (2 subtests)
 plv setting 'm.room.name' respects room powerlevel (2 subtests)
 psh Messages that notify from another user increment notification_count
-psh Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count
+msc Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count
 dvk Can claim one time key using POST (2 subtests)
 fdk Can query remote device keys using POST (1 subtests)
 fdk Can claim remote one time key using POST (2 subtests)
-fmj Inbound /make_join rejects attempts to join rooms where all users have left
+fmj Inbound /make_join rejects attempts to join rooms where all users have left
+msc Local users can peek into world_readable rooms by room ID
+msc We can't peek into rooms with shared history_visibility
+msc We can't peek into rooms with invited history_visibility
+msc We can't peek into rooms with joined history_visibility
+msc Local users can peek by room alias
+msc Peeked rooms only turn up in the sync for the device who peeked them

are-we-synapse-yet.py

@@ -35,6 +35,7 @@ test_mappings = {
     "nsp": "Non-Spec API",
     "unk": "Unknown API (no group specified)",
     "app": "Application Services API",
+    "msc": "MSCs",
     "f": "Federation", # flag to mark test involves federation
 
     "federation_apis": {
@@ -223,6 +224,7 @@ def main(results_tap_path, verbose):
         },
         "nonspec": {
             "nsp": {},
+            "msc": {},
             "unk": {}
         },
     }
@@ -237,6 +239,8 @@ def main(results_tap_path, verbose):
                 summary["nonspec"]["unk"][name] = test_result["ok"]
             if group_id == "nsp":
                 summary["nonspec"]["nsp"][name] = test_result["ok"]
+            elif group_id == "msc":
+                summary["nonspec"]["msc"][name] = test_result["ok"]
             elif group_id == "app":
                 summary["appservice"]["app"][name] = test_result["ok"]
             elif group_id in test_mappings["federation_apis"]:

build-dendritejs.sh

@@ -1,4 +1,4 @@
 #!/bin/sh -eu
 
 export GIT_COMMIT=$(git rev-list -1 HEAD) && \
-GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs
+GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o bin/main.wasm ./cmd/dendritejs-pinecone

build.sh

@@ -21,4 +21,4 @@ mkdir -p bin
 
 CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
 
-CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs
+CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone

build/docker/Dockerfile.monolith

@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.15-alpine AS base
+FROM docker.io/golang:1.17-alpine AS base
 
 RUN apk --update --no-cache add bash build-base
 

build/docker/Dockerfile.polylith

@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.15-alpine AS base
+FROM docker.io/golang:1.17-alpine AS base
 
 RUN apk --update --no-cache add bash build-base
 

build/docker/README.md

@@ -28,8 +28,8 @@ There are three sample `docker-compose` files:
 The `docker-compose` files refer to the `/etc/dendrite` volume as where the
 runtime config should come from. The mounted folder must contain:
 
-- `dendrite.yaml` configuration file (based on the sample `dendrite-config.yaml`
-   in the `docker/config` folder in the [Dendrite repository](https://github.com/matrix-org/dendrite)
+- `dendrite.yaml` configuration file (based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml)
+   sample in the `build/docker/config` folder of this repository.)
 - `matrix_key.pem` server key, as generated using `cmd/generate-keys`
 - `server.crt` certificate file
 - `server.key` private key file for the above certificate
@@ -37,19 +37,20 @@ runtime config should come from. The mounted folder must contain:
 To generate keys:
 
 ```
-go run github.com/matrix-org/dendrite/cmd/generate-keys \
-  --private-key=matrix_key.pem \
-  --tls-cert=server.crt \
-  --tls-key=server.key
+docker run --rm --entrypoint="" \
+  -v $(pwd):/mnt \
+  matrixdotorg/dendrite-monolith:latest \
+  /usr/bin/generate-keys \
+  -private-key /mnt/matrix_key.pem \
+  -tls-cert /mnt/server.crt \
+  -tls-key /mnt/server.key
 ```
 
+The key files will now exist in your current working directory, and can be mounted into place.
+
 ## Starting Dendrite as a monolith deployment
 
-Create your config based on the `dendrite.yaml` configuration file in the `docker/config`
-folder in the [Dendrite repository](https://github.com/matrix-org/dendrite). Additionally,
-make the following changes to the configuration:
-
-- Enable Naffka: `use_naffka: true`
+Create your config based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml) configuration file in the `build/docker/config` folder of this repository. And rename the config file to `dendrite.yml` (and put it in your `config` directory).
 
 Once in place, start the PostgreSQL dependency:
 
@@ -65,8 +66,7 @@ docker-compose -f docker-compose.monolith.yml up
 
 ## Starting Dendrite as a polylith deployment
 
-Create your config based on the `dendrite.yaml` configuration file in the `docker/config`
-folder in the [Dendrite repository](https://github.com/matrix-org/dendrite).
+Create your config based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml) configuration file in the `build/docker/config` folder of this repository. And rename the config file to `dendrite.yml` (and put it in your `config` directory).
 
 Once in place, start all the dependencies:
 
@@ -82,10 +82,10 @@ docker-compose -f docker-compose.polylith.yml up
 
 ## Building the images
 
-The `docker/images-build.sh` script will build the base image, followed by
+The `build/docker/images-build.sh` script will build the base image, followed by
 all of the component images.
 
-The `docker/images-push.sh` script will push them to Docker Hub (subject
+The `build/docker/images-push.sh` script will push them to Docker Hub (subject
 to permissions).
 
 If you wish to build and push your own images, rename `matrixdotorg/dendrite` to

build/gobind-pinecone/build.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+TARGET=""
+
+while getopts "ai" option
+do
+    case "$option"
+    in
+    a) gomobile bind -v -target android -trimpath -ldflags="-s -w" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
+    i) gomobile bind -v -target ios -trimpath -ldflags="" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
+    *) echo "No target specified, specify -a or -i"; exit 1 ;;
+    esac
+done

build/gobind-pinecone/monolith.go

@@ -0,0 +1,429 @@
+package gobind
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/tls"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/userapi"
+	userapiAPI "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+
+	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
+	"github.com/matrix-org/pinecone/router"
+	pineconeRouter "github.com/matrix-org/pinecone/router"
+	pineconeSessions "github.com/matrix-org/pinecone/sessions"
+	"github.com/matrix-org/pinecone/types"
+
+	_ "golang.org/x/mobile/bind"
+)
+
+const (
+	PeerTypeRemote    = pineconeRouter.PeerTypeRemote
+	PeerTypeMulticast = pineconeRouter.PeerTypeMulticast
+	PeerTypeBluetooth = pineconeRouter.PeerTypeBluetooth
+)
+
+type DendriteMonolith struct {
+	logger            logrus.Logger
+	PineconeRouter    *pineconeRouter.Router
+	PineconeMulticast *pineconeMulticast.Multicast
+	PineconeQUIC      *pineconeSessions.Sessions
+	StorageDirectory  string
+	CacheDirectory    string
+	staticPeerURI     string
+	staticPeerMutex   sync.RWMutex
+	staticPeerAttempt chan struct{}
+	listener          net.Listener
+	httpServer        *http.Server
+	processContext    *process.ProcessContext
+	userAPI           userapiAPI.UserInternalAPI
+}
+
+func (m *DendriteMonolith) BaseURL() string {
+	return fmt.Sprintf("http://%s", m.listener.Addr().String())
+}
+
+func (m *DendriteMonolith) PeerCount(peertype int) int {
+	return m.PineconeRouter.PeerCount(peertype)
+}
+
+func (m *DendriteMonolith) SessionCount() int {
+	return len(m.PineconeQUIC.Sessions())
+}
+
+func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
+	if enabled {
+		m.PineconeMulticast.Start()
+	} else {
+		m.PineconeMulticast.Stop()
+		m.DisconnectType(pineconeRouter.PeerTypeMulticast)
+	}
+}
+
+func (m *DendriteMonolith) SetStaticPeer(uri string) {
+	m.staticPeerMutex.Lock()
+	m.staticPeerURI = uri
+	m.staticPeerMutex.Unlock()
+	m.DisconnectType(pineconeRouter.PeerTypeRemote)
+	if uri != "" {
+		go func() {
+			m.staticPeerAttempt <- struct{}{}
+		}()
+	}
+}
+
+func (m *DendriteMonolith) DisconnectType(peertype int) {
+	for _, p := range m.PineconeRouter.Peers() {
+		if peertype == p.PeerType {
+			_ = m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
+		}
+	}
+}
+
+func (m *DendriteMonolith) DisconnectZone(zone string) {
+	for _, p := range m.PineconeRouter.Peers() {
+		if zone == p.Zone {
+			_ = m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
+		}
+	}
+}
+
+func (m *DendriteMonolith) DisconnectPort(port int) error {
+	return m.PineconeRouter.Disconnect(types.SwitchPortID(port), nil)
+}
+
+func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) {
+	l, r := net.Pipe()
+	conduit := &Conduit{conn: r, port: 0}
+	go func() {
+		conduit.portMutex.Lock()
+		defer conduit.portMutex.Unlock()
+	loop:
+		for i := 1; i <= 10; i++ {
+			logrus.Errorf("Attempting authenticated connect (attempt %d)", i)
+			var err error
+			conduit.port, err = m.PineconeRouter.AuthenticatedConnect(l, zone, peertype)
+			switch err {
+			case io.ErrClosedPipe:
+				logrus.Errorf("Authenticated connect failed due to closed pipe (attempt %d)", i)
+				return
+			case io.EOF:
+				logrus.Errorf("Authenticated connect failed due to EOF (attempt %d)", i)
+				break loop
+			case nil:
+				logrus.Errorf("Authenticated connect succeeded, connected to port %d (attempt %d)", conduit.port, i)
+				return
+			default:
+				logrus.WithError(err).Errorf("Authenticated connect failed (attempt %d)", i)
+				time.Sleep(time.Second)
+			}
+		}
+		_ = l.Close()
+		_ = r.Close()
+	}()
+	return conduit, nil
+}
+
+func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, error) {
+	pubkey := m.PineconeRouter.PublicKey()
+	userID := userutil.MakeUserID(
+		localpart,
+		gomatrixserverlib.ServerName(hex.EncodeToString(pubkey[:])),
+	)
+	userReq := &userapiAPI.PerformAccountCreationRequest{
+		AccountType: userapiAPI.AccountTypeUser,
+		Localpart:   localpart,
+		Password:    password,
+	}
+	userRes := &userapiAPI.PerformAccountCreationResponse{}
+	if err := m.userAPI.PerformAccountCreation(context.Background(), userReq, userRes); err != nil {
+		return userID, fmt.Errorf("userAPI.PerformAccountCreation: %w", err)
+	}
+	return userID, nil
+}
+
+func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, error) {
+	accessTokenBytes := make([]byte, 16)
+	n, err := rand.Read(accessTokenBytes)
+	if err != nil {
+		return "", fmt.Errorf("rand.Read: %w", err)
+	}
+	loginReq := &userapiAPI.PerformDeviceCreationRequest{
+		Localpart:   localpart,
+		DeviceID:    &deviceID,
+		AccessToken: hex.EncodeToString(accessTokenBytes[:n]),
+	}
+	loginRes := &userapiAPI.PerformDeviceCreationResponse{}
+	if err := m.userAPI.PerformDeviceCreation(context.Background(), loginReq, loginRes); err != nil {
+		return "", fmt.Errorf("userAPI.PerformDeviceCreation: %w", err)
+	}
+	if !loginRes.DeviceCreated {
+		return "", fmt.Errorf("device was not created")
+	}
+	return loginRes.Device.AccessToken, nil
+}
+
+func (m *DendriteMonolith) staticPeerConnect() {
+	attempt := func() {
+		if m.PineconeRouter.PeerCount(router.PeerTypeRemote) == 0 {
+			m.staticPeerMutex.RLock()
+			uri := m.staticPeerURI
+			m.staticPeerMutex.RUnlock()
+			if uri == "" {
+				return
+			}
+			if err := conn.ConnectToPeer(m.PineconeRouter, uri); err != nil {
+				logrus.WithError(err).Error("Failed to connect to static peer")
+			}
+		}
+	}
+	for {
+		select {
+		case <-m.processContext.Context().Done():
+		case <-m.staticPeerAttempt:
+			attempt()
+		case <-time.After(time.Second * 5):
+			attempt()
+		}
+	}
+}
+
+// nolint:gocyclo
+func (m *DendriteMonolith) Start() {
+	var err error
+	var sk ed25519.PrivateKey
+	var pk ed25519.PublicKey
+	keyfile := fmt.Sprintf("%s/p2p.key", m.StorageDirectory)
+	if _, err = os.Stat(keyfile); os.IsNotExist(err) {
+		if pk, sk, err = ed25519.GenerateKey(nil); err != nil {
+			panic(err)
+		}
+		if err = ioutil.WriteFile(keyfile, sk, 0644); err != nil {
+			panic(err)
+		}
+	} else if err == nil {
+		if sk, err = ioutil.ReadFile(keyfile); err != nil {
+			panic(err)
+		}
+		if len(sk) != ed25519.PrivateKeySize {
+			panic("the private key is not long enough")
+		}
+		pk = sk.Public().(ed25519.PublicKey)
+	}
+
+	m.listener, err = net.Listen("tcp", "localhost:65432")
+	if err != nil {
+		panic(err)
+	}
+
+	m.logger = logrus.Logger{
+		Out: BindLogger{},
+	}
+	m.logger.SetOutput(BindLogger{})
+	logrus.SetOutput(BindLogger{})
+
+	logger := log.New(os.Stdout, "PINECONE: ", 0)
+	m.PineconeRouter = pineconeRouter.NewRouter(logger, "dendrite", sk, pk, nil)
+	m.PineconeQUIC = pineconeSessions.NewSessions(logger, m.PineconeRouter)
+	m.PineconeMulticast = pineconeMulticast.NewMulticast(logger, m.PineconeRouter)
+
+	prefix := hex.EncodeToString(pk)
+	cfg := &config.Dendrite{}
+	cfg.Defaults()
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.PrivateKey = sk
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-naffka.db", m.StorageDirectory, prefix))
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-account.db", m.StorageDirectory, prefix))
+	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-device.db", m.StorageDirectory, prefix))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-mediaapi.db", m.CacheDirectory, prefix))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-syncapi.db", m.StorageDirectory, prefix))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix))
+	cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-signingkeyserver.db", m.StorageDirectory, prefix))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix))
+	cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix))
+	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-appservice.db", m.StorageDirectory, prefix))
+	cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
+	cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
+	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+	if err := cfg.Derive(); err != nil {
+		panic(err)
+	}
+
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	federation := conn.CreateFederationClient(base, m.PineconeQUIC)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+
+	rsAPI := roomserver.NewInternalAPI(
+		base, keyRing,
+	)
+
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing, true,
+	)
+
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
+	m.userAPI = userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
+	keyAPI.SetUserAPI(m.userAPI)
+
+	eduInputAPI := eduserver.NewInternalAPI(
+		base, cache.New(), m.userAPI,
+	)
+
+	asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI)
+
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this dependency
+	rsAPI.SetFederationSenderAPI(fsAPI)
+
+	monolith := setup.Monolith{
+		Config:    base.Cfg,
+		AccountDB: accountDB,
+		Client:    conn.CreateClient(base, m.PineconeQUIC),
+		FedClient: federation,
+		KeyRing:   keyRing,
+
+		AppserviceAPI:          asAPI,
+		EDUInternalAPI:         eduInputAPI,
+		FederationSenderAPI:    fsAPI,
+		RoomserverAPI:          rsAPI,
+		UserAPI:                m.userAPI,
+		KeyAPI:                 keyAPI,
+		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation),
+	}
+	monolith.AddAllPublicRoutes(
+		base.ProcessContext,
+		base.PublicClientAPIMux,
+		base.PublicFederationAPIMux,
+		base.PublicKeyAPIMux,
+		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
+	)
+
+	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+
+	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
+	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+
+	pHTTP := m.PineconeQUIC.HTTP()
+	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
+	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
+
+	// Build both ends of a HTTP multiplex.
+	h2s := &http2.Server{}
+	m.httpServer = &http.Server{
+		Addr:         ":0",
+		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+		ReadTimeout:  10 * time.Second,
+		WriteTimeout: 10 * time.Second,
+		IdleTimeout:  30 * time.Second,
+		BaseContext: func(_ net.Listener) context.Context {
+			return context.Background()
+		},
+		Handler: h2c.NewHandler(pMux, h2s),
+	}
+
+	m.processContext = base.ProcessContext
+
+	m.staticPeerAttempt = make(chan struct{}, 1)
+	go m.staticPeerConnect()
+
+	go func() {
+		m.logger.Info("Listening on ", cfg.Global.ServerName)
+		m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC))
+	}()
+	go func() {
+		logrus.Info("Listening on ", m.listener.Addr())
+		logrus.Fatal(http.Serve(m.listener, httpRouter))
+	}()
+	go func() {
+		logrus.Info("Sending wake-up message to known nodes")
+		req := &api.PerformBroadcastEDURequest{}
+		res := &api.PerformBroadcastEDUResponse{}
+		if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil {
+			logrus.WithError(err).Error("Failed to send wake-up message to known nodes")
+		}
+	}()
+}
+
+func (m *DendriteMonolith) Stop() {
+	_ = m.listener.Close()
+	m.PineconeMulticast.Stop()
+	_ = m.PineconeQUIC.Close()
+	m.processContext.ShutdownDendrite()
+	_ = m.PineconeRouter.Close()
+}
+
+type Conduit struct {
+	conn      net.Conn
+	port      types.SwitchPortID
+	portMutex sync.Mutex
+}
+
+func (c *Conduit) Port() int {
+	c.portMutex.Lock()
+	defer c.portMutex.Unlock()
+	return int(c.port)
+}
+
+func (c *Conduit) Read(b []byte) (int, error) {
+	return c.conn.Read(b)
+}
+
+func (c *Conduit) ReadCopy() ([]byte, error) {
+	var buf [65535 * 2]byte
+	n, err := c.conn.Read(buf[:])
+	if err != nil {
+		return nil, err
+	}
+	return buf[:n], nil
+}
+
+func (c *Conduit) Write(b []byte) (int, error) {
+	return c.conn.Write(b)
+}
+
+func (c *Conduit) Close() error {
+	return c.conn.Close()
+}

build/gobind-yggdrasil/build.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+#!/bin/sh
+
+TARGET=""
+
+while getopts "ai" option
+do
+    case "$option"
+    in
+    a) TARGET="android";;
+    i) TARGET="ios";;
+    esac
+done
+
+if [[ $TARGET = "" ]];
+then
+    echo "No target specified, specify -a or -i"
+    exit 1
+fi
+
+gomobile bind -v \
+    -target $TARGET \
+    -ldflags "-X github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=dendrite" \
+    github.com/matrix-org/dendrite/build/gobind-pinecone

build/gobind-yggdrasil/monolith.go

@@ -25,6 +25,8 @@ import (
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
+
+	_ "golang.org/x/mobile/bind"
 )
 
 type DendriteMonolith struct {
@@ -43,10 +45,6 @@ func (m *DendriteMonolith) PeerCount() int {
 	return m.YggdrasilNode.PeerCount()
 }
 
-func (m *DendriteMonolith) SessionCount() int {
-	return m.YggdrasilNode.SessionCount()
-}
-
 func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
 	m.YggdrasilNode.SetMulticastEnabled(enabled)
 }
@@ -76,7 +74,7 @@ func (m *DendriteMonolith) Start() {
 		panic(err)
 	}
 
-	ygg, err := yggconn.Setup("dendrite", m.StorageDirectory)
+	ygg, err := yggconn.Setup("dendrite", m.StorageDirectory, "")
 	if err != nil {
 		panic(err)
 	}
@@ -85,7 +83,7 @@ func (m *DendriteMonolith) Start() {
 	cfg := &config.Dendrite{}
 	cfg.Defaults()
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Global.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.Kafka.UseNaffka = true
 	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-naffka.db", m.StorageDirectory))
@@ -118,10 +116,10 @@ func (m *DendriteMonolith) Start() {
 	)
 
 	fsAPI := federationsender.NewInternalAPI(
-		base, federation, rsAPI, keyRing,
+		base, federation, rsAPI, keyRing, true,
 	)
 
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
@@ -132,18 +130,6 @@ func (m *DendriteMonolith) Start() {
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 
-	ygg.SetSessionFunc(func(address string) {
-		req := &api.PerformServersAliveRequest{
-			Servers: []gomatrixserverlib.ServerName{
-				gomatrixserverlib.ServerName(address),
-			},
-		}
-		res := &api.PerformServersAliveResponse{}
-		if err := fsAPI.PerformServersAlive(context.TODO(), req, res); err != nil {
-			logrus.WithError(err).Error("Failed to send wake-up message to newly connected node")
-		}
-	})
-
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
 	rsAPI.SetFederationSenderAPI(fsAPI)
@@ -171,6 +157,7 @@ func (m *DendriteMonolith) Start() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 
 	httpRouter := mux.NewRouter()

build/gobind-yggdrasil/platform_ios.go

@@ -0,0 +1,25 @@
+// +build ios
+
+package gobind
+
+/*
+#cgo CFLAGS: -x objective-c
+#cgo LDFLAGS: -framework Foundation
+#import <Foundation/Foundation.h>
+void Log(const char *text) {
+  NSString *nss = [NSString stringWithUTF8String:text];
+  NSLog(@"%@", nss);
+}
+*/
+import "C"
+import "unsafe"
+
+type BindLogger struct {
+}
+
+func (nsl BindLogger) Write(p []byte) (n int, err error) {
+	p = append(p, 0)
+	cstr := (*C.char)(unsafe.Pointer(&p[0]))
+	C.Log(cstr)
+	return len(p), nil
+}

build/gobind-yggdrasil/platform_other.go

@@ -0,0 +1,12 @@
+// +build !ios
+
+package gobind
+
+import "log"
+
+type BindLogger struct{}
+
+func (nsl BindLogger) Write(p []byte) (n int, err error) {
+	log.Println(string(p))
+	return len(p), nil
+}

build/gobind/build.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-gomobile bind -v \
-    -ldflags "-X github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=dendrite" \
-    -target ios \
-    github.com/matrix-org/dendrite/build/gobind

build/scripts/find-lint.sh

@@ -25,7 +25,7 @@ echo "Installing golangci-lint..."
 
 # Make a backup of go.{mod,sum} first
 cp go.mod go.mod.bak && cp go.sum go.sum.bak
-go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1
+go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.41.1
 
 # Run linting
 echo "Looking for lint..."

clientapi/auth/password.go

@@ -52,7 +52,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 	if username == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.BadJSON("'user' must be supplied."),
+			JSON: jsonerror.BadJSON("A username must be supplied."),
 		}
 	}
 	localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName)
@@ -68,7 +68,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		// but that would leak the existence of the user.
 		return nil, &util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"),
+			JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."),
 		}
 	}
 	return &r.Login, nil

clientapi/auth/user_interactive.go

@@ -78,7 +78,7 @@ func (r *Login) Username() string {
 	if r.Identifier.Type == "m.id.user" {
 		return r.Identifier.User
 	}
-	// deprecated but without it Riot iOS won't log in
+	// deprecated but without it Element iOS won't log in
 	return r.User
 }
 
@@ -220,7 +220,7 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 	if !ok {
 		return nil, &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("unknown auth.type: " + authType),
+			JSON: jsonerror.BadJSON("Unknown auth.type: " + authType),
 		}
 	}
 
@@ -231,7 +231,7 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 		if !u.IsSingleStageFlow(authType) {
 			return nil, &util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: jsonerror.Unknown("missing or unknown auth.session"),
+				JSON: jsonerror.Unknown("The auth.session is missing or unknown."),
 			}
 		}
 	}

clientapi/clientapi.go

@@ -35,6 +35,7 @@ import (
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
 func AddPublicRoutes(
 	router *mux.Router,
+	synapseAdminRouter *mux.Router,
 	cfg *config.ClientAPI,
 	accountsDB accounts.Database,
 	federation *gomatrixserverlib.FederationClient,
@@ -56,7 +57,7 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		router, cfg, eduInputAPI, rsAPI, asAPI,
+		router, synapseAdminRouter, cfg, eduInputAPI, rsAPI, asAPI,
 		accountsDB, userAPI, federation,
 		syncProducer, transactionsCache, fsAPI, keyAPI, extRoomsProvider, mscCfg,
 	)

clientapi/jsonerror/jsonerror.go

@@ -111,6 +111,12 @@ func UserInUse(msg string) *MatrixError {
 	return &MatrixError{"M_USER_IN_USE", msg}
 }
 
+// RoomInUse is an error returned when the client tries to make a room
+// that already exists
+func RoomInUse(msg string) *MatrixError {
+	return &MatrixError{"M_ROOM_IN_USE", msg}
+}
+
 // ASExclusive is an error returned when an application service tries to
 // register an username that is outside of its registered namespace, or if a
 // user attempts to register a username or room alias within an exclusive
@@ -125,6 +131,24 @@ func GuestAccessForbidden(msg string) *MatrixError {
 	return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg}
 }
 
+// InvalidSignature is an error which is returned when the client tries
+// to upload invalid signatures.
+func InvalidSignature(msg string) *MatrixError {
+	return &MatrixError{"M_INVALID_SIGNATURE", msg}
+}
+
+// InvalidParam is an error that is returned when a parameter was invalid,
+// traditionally with cross-signing.
+func InvalidParam(msg string) *MatrixError {
+	return &MatrixError{"M_INVALID_PARAM", msg}
+}
+
+// MissingParam is an error that is returned when a parameter was incorrect,
+// traditionally with cross-signing.
+func MissingParam(msg string) *MatrixError {
+	return &MatrixError{"M_MISSING_PARAM", msg}
+}
+
 type IncompatibleRoomVersionError struct {
 	RoomVersion string `json:"room_version"`
 	Error       string `json:"error"`

clientapi/routing/account_data.go

@@ -69,7 +69,7 @@ func GetAccountData(
 
 	return util.JSONResponse{
 		Code: http.StatusNotFound,
-		JSON: jsonerror.Forbidden("data not found"),
+		JSON: jsonerror.NotFound("data not found"),
 	}
 }
 

clientapi/routing/aliases.go

@@ -0,0 +1,96 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/util"
+)
+
+// GetAliases implements GET /_matrix/client/r0/rooms/{roomId}/aliases
+func GetAliases(
+	req *http.Request, rsAPI api.RoomserverInternalAPI, device *userapi.Device, roomID string,
+) util.JSONResponse {
+	stateTuple := gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomHistoryVisibility,
+		StateKey:  "",
+	}
+	stateReq := &api.QueryCurrentStateRequest{
+		RoomID:      roomID,
+		StateTuples: []gomatrixserverlib.StateKeyTuple{stateTuple},
+	}
+	stateRes := &api.QueryCurrentStateResponse{}
+	if err := rsAPI.QueryCurrentState(req.Context(), stateReq, stateRes); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryCurrentState failed")
+		return util.ErrorResponse(fmt.Errorf("rsAPI.QueryCurrentState: %w", err))
+	}
+
+	visibility := "invite"
+	if historyVisEvent, ok := stateRes.StateEvents[stateTuple]; ok {
+		var err error
+		visibility, err = historyVisEvent.HistoryVisibility()
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("historyVisEvent.HistoryVisibility failed")
+			return util.ErrorResponse(fmt.Errorf("historyVisEvent.HistoryVisibility: %w", err))
+		}
+	}
+	if visibility != gomatrixserverlib.WorldReadable {
+		queryReq := api.QueryMembershipForUserRequest{
+			RoomID: roomID,
+			UserID: device.UserID,
+		}
+		var queryRes api.QueryMembershipForUserResponse
+		if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed")
+			return jsonerror.InternalServerError()
+		}
+		if !queryRes.IsInRoom {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden("You aren't a member of this room."),
+			}
+		}
+	}
+
+	aliasesReq := api.GetAliasesForRoomIDRequest{
+		RoomID: roomID,
+	}
+	aliasesRes := api.GetAliasesForRoomIDResponse{}
+	if err := rsAPI.GetAliasesForRoomID(req.Context(), &aliasesReq, &aliasesRes); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.GetAliasesForRoomID failed")
+		return util.ErrorResponse(fmt.Errorf("rsAPI.GetAliasesForRoomID: %w", err))
+	}
+
+	response := struct {
+		Aliases []string `json:"aliases"`
+	}{
+		Aliases: aliasesRes.Aliases,
+	}
+	if response.Aliases == nil {
+		response.Aliases = []string{} // pleases sytest
+	}
+
+	return util.JSONResponse{
+		Code: 200,
+		JSON: response,
+	}
+}

clientapi/routing/createroom.go

@@ -43,7 +43,7 @@ type createRoomRequest struct {
 	Visibility                string                        `json:"visibility"`
 	Topic                     string                        `json:"topic"`
 	Preset                    string                        `json:"preset"`
-	CreationContent           map[string]interface{}        `json:"creation_content"`
+	CreationContent           json.RawMessage               `json:"creation_content"`
 	InitialState              []fledglingEvent              `json:"initial_state"`
 	RoomAliasName             string                        `json:"room_alias_name"`
 	GuestCanJoin              bool                          `json:"guest_can_join"`
@@ -177,11 +177,6 @@ func createRoom(
 
 	// Clobber keys: creator, room_version
 
-	if r.CreationContent == nil {
-		r.CreationContent = make(map[string]interface{}, 2)
-	}
-
-	r.CreationContent["creator"] = userID
 	roomVersion := roomserverVersion.DefaultRoomVersion()
 	if r.RoomVersion != "" {
 		candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion)
@@ -194,7 +189,6 @@ func createRoom(
 		}
 		roomVersion = candidateVersion
 	}
-	r.CreationContent["room_version"] = roomVersion
 
 	// TODO: visibility/presets/raw initial state
 	// TODO: Create room alias association
@@ -203,7 +197,7 @@ func createRoom(
 	logger.WithFields(log.Fields{
 		"userID":      userID,
 		"roomID":      roomID,
-		"roomVersion": r.CreationContent["room_version"],
+		"roomVersion": roomVersion,
 	}).Info("Creating new room")
 
 	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
@@ -212,6 +206,109 @@ func createRoom(
 		return jsonerror.InternalServerError()
 	}
 
+	createContent := map[string]interface{}{}
+	if len(r.CreationContent) > 0 {
+		if err = json.Unmarshal(r.CreationContent, &createContent); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal for creation_content failed")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("invalid create content"),
+			}
+		}
+	}
+	createContent["creator"] = userID
+	createContent["room_version"] = roomVersion
+	powerLevelContent := eventutil.InitialPowerLevelsContent(userID)
+	joinRuleContent := gomatrixserverlib.JoinRuleContent{
+		JoinRule: gomatrixserverlib.Invite,
+	}
+	historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{
+		HistoryVisibility: historyVisibilityShared,
+	}
+
+	if r.PowerLevelContentOverride != nil {
+		// Merge powerLevelContentOverride fields by unmarshalling it atop the defaults
+		err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal for power_level_content_override failed")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("malformed power_level_content_override"),
+			}
+		}
+	}
+
+	switch r.Preset {
+	case presetPrivateChat:
+		joinRuleContent.JoinRule = gomatrixserverlib.Invite
+		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
+	case presetTrustedPrivateChat:
+		joinRuleContent.JoinRule = gomatrixserverlib.Invite
+		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
+		// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
+	case presetPublicChat:
+		joinRuleContent.JoinRule = gomatrixserverlib.Public
+		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
+	}
+
+	createEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomCreate,
+		Content: createContent,
+	}
+	powerLevelEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomPowerLevels,
+		Content: powerLevelContent,
+	}
+	joinRuleEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomJoinRules,
+		Content: joinRuleContent,
+	}
+	historyVisibilityEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomHistoryVisibility,
+		Content: historyVisibilityContent,
+	}
+	membershipEvent := fledglingEvent{
+		Type:     gomatrixserverlib.MRoomMember,
+		StateKey: userID,
+		Content: gomatrixserverlib.MemberContent{
+			Membership:  gomatrixserverlib.Join,
+			DisplayName: profile.DisplayName,
+			AvatarURL:   profile.AvatarURL,
+		},
+	}
+
+	var nameEvent *fledglingEvent
+	var topicEvent *fledglingEvent
+	var guestAccessEvent *fledglingEvent
+	var aliasEvent *fledglingEvent
+
+	if r.Name != "" {
+		nameEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomName,
+			Content: eventutil.NameContent{
+				Name: r.Name,
+			},
+		}
+	}
+
+	if r.Topic != "" {
+		topicEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomTopic,
+			Content: eventutil.TopicContent{
+				Topic: r.Topic,
+			},
+		}
+	}
+
+	if r.GuestCanJoin {
+		guestAccessEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomGuestAccess,
+			Content: eventutil.GuestAccessContent{
+				GuestAccess: "can_join",
+			},
+		}
+	}
+
 	var roomAlias string
 	if r.RoomAliasName != "" {
 		roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName)
@@ -228,46 +325,51 @@ func createRoom(
 			return jsonerror.InternalServerError()
 		}
 		if aliasResp.RoomID != "" {
-			return util.MessageResponse(400, "Alias already exists")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.RoomInUse("Room ID already exists."),
+			}
+		}
+
+		aliasEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomCanonicalAlias,
+			Content: eventutil.CanonicalAlias{
+				Alias: roomAlias,
+			},
 		}
 	}
 
-	membershipContent := gomatrixserverlib.MemberContent{
-		Membership:  gomatrixserverlib.Join,
-		DisplayName: profile.DisplayName,
-		AvatarURL:   profile.AvatarURL,
-	}
+	var initialStateEvents []fledglingEvent
+	for i := range r.InitialState {
+		if r.InitialState[i].StateKey != "" {
+			initialStateEvents = append(initialStateEvents, r.InitialState[i])
+			continue
+		}
 
-	var joinRules, historyVisibility string
-	switch r.Preset {
-	case presetPrivateChat:
-		joinRules = gomatrixserverlib.Invite
-		historyVisibility = historyVisibilityShared
-	case presetTrustedPrivateChat:
-		joinRules = gomatrixserverlib.Invite
-		historyVisibility = historyVisibilityShared
-		// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
-	case presetPublicChat:
-		joinRules = gomatrixserverlib.Public
-		historyVisibility = historyVisibilityShared
-	default:
-		// Default room rules, r.Preset was previously checked for valid values so
-		// only a request with no preset should end up here.
-		joinRules = gomatrixserverlib.Invite
-		historyVisibility = historyVisibilityShared
-	}
+		switch r.InitialState[i].Type {
+		case gomatrixserverlib.MRoomCreate:
+			continue
 
-	var builtEvents []*gomatrixserverlib.HeaderedEvent
+		case gomatrixserverlib.MRoomPowerLevels:
+			powerLevelEvent = r.InitialState[i]
 
-	powerLevelContent := eventutil.InitialPowerLevelsContent(userID)
-	if r.PowerLevelContentOverride != nil {
-		// Merge powerLevelContentOverride fields by unmarshalling it atop the defaults
-		err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent)
-		if err != nil {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.BadJSON("malformed power_level_content_override"),
-			}
+		case gomatrixserverlib.MRoomJoinRules:
+			joinRuleEvent = r.InitialState[i]
+
+		case gomatrixserverlib.MRoomHistoryVisibility:
+			historyVisibilityEvent = r.InitialState[i]
+
+		case gomatrixserverlib.MRoomGuestAccess:
+			guestAccessEvent = &r.InitialState[i]
+
+		case gomatrixserverlib.MRoomName:
+			nameEvent = &r.InitialState[i]
+
+		case gomatrixserverlib.MRoomTopic:
+			topicEvent = &r.InitialState[i]
+
+		default:
+			initialStateEvents = append(initialStateEvents, r.InitialState[i])
 		}
 	}
 
@@ -284,37 +386,33 @@ func createRoom(
 	//  10- m.room.topic (opt)
 	//  11- invite events (opt) - with is_direct flag if applicable TODO
 	//  12- 3pid invite events (opt) TODO
-	//  13- m.room.aliases event for HS (if alias specified) TODO
 	// This differs from Synapse slightly. Synapse would vary the ordering of 3-7
 	// depending on if those events were in "initial_state" or not. This made it
 	// harder to reason about, hence sticking to a strict static ordering.
 	// TODO: Synapse has txn/token ID on each event. Do we need to do this here?
 	eventsToMake := []fledglingEvent{
-		{"m.room.create", "", r.CreationContent},
-		{"m.room.member", userID, membershipContent},
-		{"m.room.power_levels", "", powerLevelContent},
-		{"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}},
-		{"m.room.history_visibility", "", eventutil.HistoryVisibilityContent{HistoryVisibility: historyVisibility}},
+		createEvent, membershipEvent, powerLevelEvent, joinRuleEvent, historyVisibilityEvent,
 	}
-	if roomAlias != "" {
+	if guestAccessEvent != nil {
+		eventsToMake = append(eventsToMake, *guestAccessEvent)
+	}
+	eventsToMake = append(eventsToMake, initialStateEvents...)
+	if nameEvent != nil {
+		eventsToMake = append(eventsToMake, *nameEvent)
+	}
+	if topicEvent != nil {
+		eventsToMake = append(eventsToMake, *topicEvent)
+	}
+	if aliasEvent != nil {
 		// TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room.
 		// This means we might fail creating the alias but say the canonical alias is something that doesn't exist.
-		// m.room.aliases is handled when we call roomserver.SetRoomAlias
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", eventutil.CanonicalAlias{Alias: roomAlias}})
-	}
-	if r.GuestCanJoin {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", eventutil.GuestAccessContent{GuestAccess: "can_join"}})
-	}
-	eventsToMake = append(eventsToMake, r.InitialState...)
-	if r.Name != "" {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", eventutil.NameContent{Name: r.Name}})
-	}
-	if r.Topic != "" {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", eventutil.TopicContent{Topic: r.Topic}})
+		eventsToMake = append(eventsToMake, *aliasEvent)
 	}
+
 	// TODO: invite events
 	// TODO: 3pid invite events
 
+	var builtEvents []*gomatrixserverlib.HeaderedEvent
 	authEvents := gomatrixserverlib.NewAuthEvents(nil)
 	for i, e := range eventsToMake {
 		depth := i + 1 // depth starts at 1
@@ -389,7 +487,10 @@ func createRoom(
 		}
 
 		if aliasResp.AliasExists {
-			return util.MessageResponse(400, "Alias already exists")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.RoomInUse("Room alias already exists."),
+			}
 		}
 	}
 
@@ -403,7 +504,7 @@ func createRoom(
 				fallthrough
 			case gomatrixserverlib.MRoomCanonicalAlias:
 				fallthrough
-			case "m.room.encryption": // TODO: move this to gmsl
+			case gomatrixserverlib.MRoomEncryption:
 				fallthrough
 			case gomatrixserverlib.MRoomMember:
 				fallthrough

clientapi/routing/deactivate.go

@@ -33,7 +33,7 @@ func Deactivate(
 		return *errRes
 	}
 
-	localpart, _, err := gomatrixserverlib.SplitID('@', login.User)
+	localpart, _, err := gomatrixserverlib.SplitID('@', login.Username())
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
 		return jsonerror.InternalServerError()

clientapi/routing/directory.go

@@ -113,13 +113,12 @@ func DirectoryRoom(
 }
 
 // SetLocalAlias implements PUT /directory/room/{roomAlias}
-// TODO: Check if the user has the power level to set an alias
 func SetLocalAlias(
 	req *http.Request,
 	device *api.Device,
 	alias string,
 	cfg *config.ClientAPI,
-	aliasAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', alias)
 	if err != nil {
@@ -172,7 +171,7 @@ func SetLocalAlias(
 		Alias:  alias,
 	}
 	var queryRes roomserverAPI.SetRoomAliasResponse
-	if err := aliasAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
+	if err := rsAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
 		return jsonerror.InternalServerError()
 	}
@@ -195,43 +194,32 @@ func RemoveLocalAlias(
 	req *http.Request,
 	device *api.Device,
 	alias string,
-	aliasAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
-
-	creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{
-		Alias: alias,
-	}
-	var creatorQueryRes roomserverAPI.GetCreatorIDForAliasResponse
-	if err := aliasAPI.GetCreatorIDForAlias(req.Context(), &creatorQueryReq, &creatorQueryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.GetCreatorIDForAlias failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if creatorQueryRes.UserID == "" {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Alias does not exist"),
-		}
-	}
-
-	if creatorQueryRes.UserID != device.UserID {
-		// TODO: Still allow deletion if user is admin
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("You do not have permission to delete this alias"),
-		}
-	}
-
 	queryReq := roomserverAPI.RemoveRoomAliasRequest{
 		Alias:  alias,
 		UserID: device.UserID,
 	}
 	var queryRes roomserverAPI.RemoveRoomAliasResponse
-	if err := aliasAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
+	if err := rsAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
 		return jsonerror.InternalServerError()
 	}
 
+	if !queryRes.Found {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The alias does not exist."),
+		}
+	}
+
+	if !queryRes.Removed {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You do not have permission to remove this alias."),
+		}
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: struct{}{},
@@ -294,9 +282,9 @@ func SetVisibility(
 		return jsonerror.InternalServerError()
 	}
 
-	// NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event
+	// NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event
 	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
-	if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) {
+	if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomCanonicalAlias, true) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),

clientapi/routing/key_backup.go

@@ -0,0 +1,291 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+)
+
+type keyBackupVersion struct {
+	Algorithm string          `json:"algorithm"`
+	AuthData  json.RawMessage `json:"auth_data"`
+}
+
+type keyBackupVersionCreateResponse struct {
+	Version string `json:"version"`
+}
+
+type keyBackupVersionResponse struct {
+	Algorithm string          `json:"algorithm"`
+	AuthData  json.RawMessage `json:"auth_data"`
+	Count     int64           `json:"count"`
+	ETag      string          `json:"etag"`
+	Version   string          `json:"version"`
+}
+
+type keyBackupSessionRequest struct {
+	Rooms map[string]struct {
+		Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+	} `json:"rooms"`
+}
+
+type keyBackupSessionResponse struct {
+	Count int64  `json:"count"`
+	ETag  string `json:"etag"`
+}
+
+// Create a new key backup. Request must contain a `keyBackupVersion`. Returns a `keyBackupVersionCreateResponse`.
+// Implements  POST /_matrix/client/r0/room_keys/version
+func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device) util.JSONResponse {
+	var kb keyBackupVersion
+	resErr := httputil.UnmarshalJSONRequest(req, &kb)
+	if resErr != nil {
+		return *resErr
+	}
+	var performKeyBackupResp userapi.PerformKeyBackupResponse
+	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+		UserID:    device.UserID,
+		Version:   "",
+		AuthData:  kb.AuthData,
+		Algorithm: kb.Algorithm,
+	}, &performKeyBackupResp)
+	if performKeyBackupResp.Error != "" {
+		if performKeyBackupResp.BadInput {
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.InvalidArgumentValue(performKeyBackupResp.Error),
+			}
+		}
+		return util.ErrorResponse(fmt.Errorf("PerformKeyBackup: %s", performKeyBackupResp.Error))
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: keyBackupVersionCreateResponse{
+			Version: performKeyBackupResp.Version,
+		},
+	}
+}
+
+// KeyBackupVersion returns the key backup version specified. If `version` is empty, the latest `keyBackupVersionResponse` is returned.
+// Implements GET  /_matrix/client/r0/room_keys/version and GET /_matrix/client/r0/room_keys/version/{version}
+func KeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
+	var queryResp userapi.QueryKeyBackupResponse
+	userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
+		UserID:  device.UserID,
+		Version: version,
+	}, &queryResp)
+	if queryResp.Error != "" {
+		return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error))
+	}
+	if !queryResp.Exists {
+		return util.JSONResponse{
+			Code: 404,
+			JSON: jsonerror.NotFound("version not found"),
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: keyBackupVersionResponse{
+			Algorithm: queryResp.Algorithm,
+			AuthData:  queryResp.AuthData,
+			Count:     queryResp.Count,
+			ETag:      queryResp.ETag,
+			Version:   queryResp.Version,
+		},
+	}
+}
+
+// Modify the auth data of a key backup. Version must not be empty. Request must contain a `keyBackupVersion`
+// Implements PUT  /_matrix/client/r0/room_keys/version/{version}
+func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
+	var kb keyBackupVersion
+	resErr := httputil.UnmarshalJSONRequest(req, &kb)
+	if resErr != nil {
+		return *resErr
+	}
+	var performKeyBackupResp userapi.PerformKeyBackupResponse
+	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+		UserID:    device.UserID,
+		Version:   version,
+		AuthData:  kb.AuthData,
+		Algorithm: kb.Algorithm,
+	}, &performKeyBackupResp)
+	if performKeyBackupResp.Error != "" {
+		if performKeyBackupResp.BadInput {
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.InvalidArgumentValue(performKeyBackupResp.Error),
+			}
+		}
+		return util.ErrorResponse(fmt.Errorf("PerformKeyBackup: %s", performKeyBackupResp.Error))
+	}
+	if !performKeyBackupResp.Exists {
+		return util.JSONResponse{
+			Code: 404,
+			JSON: jsonerror.NotFound("backup version not found"),
+		}
+	}
+	// Unclear what the 200 body should be
+	return util.JSONResponse{
+		Code: 200,
+		JSON: keyBackupVersionCreateResponse{
+			Version: performKeyBackupResp.Version,
+		},
+	}
+}
+
+// Delete a version of key backup. Version must not be empty. If the key backup was previously deleted, will return 200 OK.
+// Implements DELETE  /_matrix/client/r0/room_keys/version/{version}
+func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
+	var performKeyBackupResp userapi.PerformKeyBackupResponse
+	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+		UserID:       device.UserID,
+		Version:      version,
+		DeleteBackup: true,
+	}, &performKeyBackupResp)
+	if performKeyBackupResp.Error != "" {
+		if performKeyBackupResp.BadInput {
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.InvalidArgumentValue(performKeyBackupResp.Error),
+			}
+		}
+		return util.ErrorResponse(fmt.Errorf("PerformKeyBackup: %s", performKeyBackupResp.Error))
+	}
+	if !performKeyBackupResp.Exists {
+		return util.JSONResponse{
+			Code: 404,
+			JSON: jsonerror.NotFound("backup version not found"),
+		}
+	}
+	// Unclear what the 200 body should be
+	return util.JSONResponse{
+		Code: 200,
+		JSON: keyBackupVersionCreateResponse{
+			Version: performKeyBackupResp.Version,
+		},
+	}
+}
+
+// Upload a bunch of session keys for a given `version`.
+func UploadBackupKeys(
+	req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string, keys *keyBackupSessionRequest,
+) util.JSONResponse {
+	var performKeyBackupResp userapi.PerformKeyBackupResponse
+	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+		UserID:  device.UserID,
+		Version: version,
+		Keys:    *keys,
+	}, &performKeyBackupResp)
+	if performKeyBackupResp.Error != "" {
+		if performKeyBackupResp.BadInput {
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.InvalidArgumentValue(performKeyBackupResp.Error),
+			}
+		}
+		return util.ErrorResponse(fmt.Errorf("PerformKeyBackup: %s", performKeyBackupResp.Error))
+	}
+	if !performKeyBackupResp.Exists {
+		return util.JSONResponse{
+			Code: 404,
+			JSON: jsonerror.NotFound("backup version not found"),
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: keyBackupSessionResponse{
+			Count: performKeyBackupResp.KeyCount,
+			ETag:  performKeyBackupResp.KeyETag,
+		},
+	}
+}
+
+// Get keys from a given backup version. Response returned varies depending on if roomID and sessionID are set.
+func GetBackupKeys(
+	req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version, roomID, sessionID string,
+) util.JSONResponse {
+	var queryResp userapi.QueryKeyBackupResponse
+	userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
+		UserID:           device.UserID,
+		Version:          version,
+		ReturnKeys:       true,
+		KeysForRoomID:    roomID,
+		KeysForSessionID: sessionID,
+	}, &queryResp)
+	if queryResp.Error != "" {
+		return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error))
+	}
+	if !queryResp.Exists {
+		return util.JSONResponse{
+			Code: 404,
+			JSON: jsonerror.NotFound("version not found"),
+		}
+	}
+	if sessionID != "" {
+		// return the key itself if it was found
+		roomData, ok := queryResp.Keys[roomID]
+		if ok {
+			key, ok := roomData[sessionID]
+			if ok {
+				return util.JSONResponse{
+					Code: 200,
+					JSON: key,
+				}
+			}
+		}
+	} else if roomID != "" {
+		roomData, ok := queryResp.Keys[roomID]
+		if ok {
+			// wrap response in "sessions"
+			return util.JSONResponse{
+				Code: 200,
+				JSON: struct {
+					Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+				}{
+					Sessions: roomData,
+				},
+			}
+		}
+	} else {
+		// response is the same as the upload request
+		var resp keyBackupSessionRequest
+		resp.Rooms = make(map[string]struct {
+			Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+		})
+		for roomID, roomData := range queryResp.Keys {
+			resp.Rooms[roomID] = struct {
+				Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+			}{
+				Sessions: roomData,
+			}
+		}
+		return util.JSONResponse{
+			Code: 200,
+			JSON: resp,
+		}
+	}
+	return util.JSONResponse{
+		Code: 404,
+		JSON: jsonerror.NotFound("keys not found"),
+	}
+}

clientapi/routing/key_crosssigning.go

@@ -0,0 +1,149 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/util"
+)
+
+type crossSigningRequest struct {
+	api.PerformUploadDeviceKeysRequest
+	Auth newPasswordAuth `json:"auth"`
+}
+
+func UploadCrossSigningDeviceKeys(
+	req *http.Request, userInteractiveAuth *auth.UserInteractive,
+	keyserverAPI api.KeyInternalAPI, device *userapi.Device,
+	accountDB accounts.Database, cfg *config.ClientAPI,
+) util.JSONResponse {
+	uploadReq := &crossSigningRequest{}
+	uploadRes := &api.PerformUploadDeviceKeysResponse{}
+
+	resErr := httputil.UnmarshalJSONRequest(req, &uploadReq)
+	if resErr != nil {
+		return *resErr
+	}
+	sessionID := uploadReq.Auth.Session
+	if sessionID == "" {
+		sessionID = util.RandomString(sessionIDLength)
+	}
+	if uploadReq.Auth.Type != authtypes.LoginTypePassword {
+		return util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: newUserInteractiveResponse(
+				sessionID,
+				[]authtypes.Flow{
+					{
+						Stages: []authtypes.LoginType{authtypes.LoginTypePassword},
+					},
+				},
+				nil,
+			),
+		}
+	}
+	typePassword := auth.LoginTypePassword{
+		GetAccountByPassword: accountDB.GetAccountByPassword,
+		Config:               cfg,
+	}
+	if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {
+		return *authErr
+	}
+	AddCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
+
+	uploadReq.UserID = device.UserID
+	keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes)
+
+	if err := uploadRes.Error; err != nil {
+		switch {
+		case err.IsInvalidSignature:
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidSignature(err.Error()),
+			}
+		case err.IsMissingParam:
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.MissingParam(err.Error()),
+			}
+		case err.IsInvalidParam:
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidParam(err.Error()),
+			}
+		default:
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.Unknown(err.Error()),
+			}
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse {
+	uploadReq := &api.PerformUploadDeviceSignaturesRequest{}
+	uploadRes := &api.PerformUploadDeviceSignaturesResponse{}
+
+	if err := httputil.UnmarshalJSONRequest(req, &uploadReq.Signatures); err != nil {
+		return *err
+	}
+
+	uploadReq.UserID = device.UserID
+	keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes)
+
+	if err := uploadRes.Error; err != nil {
+		switch {
+		case err.IsInvalidSignature:
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidSignature(err.Error()),
+			}
+		case err.IsMissingParam:
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.MissingParam(err.Error()),
+			}
+		case err.IsInvalidParam:
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidParam(err.Error()),
+			}
+		default:
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.Unknown(err.Error()),
+			}
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/keys.go

@@ -100,7 +100,7 @@ func (r *queryKeysRequest) GetTimeout() time.Duration {
 	return time.Duration(r.Timeout) * time.Millisecond
 }
 
-func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI) util.JSONResponse {
+func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse {
 	var r queryKeysRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
@@ -108,6 +108,7 @@ func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI) util.JSONResponse {
 	}
 	queryRes := api.QueryKeysResponse{}
 	keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{
+		UserID:        device.UserID,
 		UserToDevices: r.DeviceKeys,
 		Timeout:       r.GetTimeout(),
 		// TODO: Token?
@@ -115,8 +116,11 @@ func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI) util.JSONResponse {
 	return util.JSONResponse{
 		Code: 200,
 		JSON: map[string]interface{}{
-			"device_keys": queryRes.DeviceKeys,
-			"failures":    queryRes.Failures,
+			"device_keys":       queryRes.DeviceKeys,
+			"master_keys":       queryRes.MasterKeys,
+			"self_signing_keys": queryRes.SelfSigningKeys,
+			"user_signing_keys": queryRes.UserSigningKeys,
+			"failures":          queryRes.Failures,
 		},
 	}
 }

clientapi/routing/membership.go

@@ -47,6 +47,37 @@ func SendBan(
 	if reqErr != nil {
 		return *reqErr
 	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+
+	plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomPowerLevels,
+		StateKey:  "",
+	})
+	if plEvent == nil {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Forbidden("You don't have permission to ban this user, no power_levels event in this room."),
+		}
+	}
+	pl, err := plEvent.PowerLevels()
+	if err != nil {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Forbidden("You don't have permission to ban this user, the power_levels event for this room is malformed so auth checks cannot be performed."),
+		}
+	}
+	allowedToBan := pl.UserLevel(device.UserID) >= pl.Ban
+	if !allowedToBan {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Forbidden("You don't have permission to ban this user, power level too low."),
+		}
+	}
+
 	return sendMembership(req.Context(), accountDB, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
 }
 

clientapi/routing/register.go

@@ -17,10 +17,7 @@ package routing
 
 import (
 	"context"
-	"crypto/hmac"
-	"crypto/sha1"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -594,7 +591,6 @@ func handleRegistrationFlow(
 	accessToken string,
 	accessTokenErr error,
 ) util.JSONResponse {
-	// TODO: Shared secret registration (create new user scripts)
 	// TODO: Enable registration config flag
 	// TODO: Guest account upgrading
 
@@ -643,20 +639,6 @@ func handleRegistrationFlow(
 		// Add Recaptcha to the list of completed registration stages
 		AddCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
 
-	case authtypes.LoginTypeSharedSecret:
-		// Check shared secret against config
-		valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Auth.Mac)
-
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("isValidMacLogin failed")
-			return jsonerror.InternalServerError()
-		} else if !valid {
-			return util.MessageResponse(http.StatusForbidden, "HMAC incorrect")
-		}
-
-		// Add SharedSecret to the list of completed registration stages
-		AddCompletedSessionStage(sessionID, authtypes.LoginTypeSharedSecret)
-
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
 		// Add Dummy to the list of completed registration stages
@@ -849,49 +831,6 @@ func completeRegistration(
 	}
 }
 
-// Used for shared secret registration.
-// Checks if the username, password and isAdmin flag matches the given mac.
-func isValidMacLogin(
-	cfg *config.ClientAPI,
-	username, password string,
-	isAdmin bool,
-	givenMac []byte,
-) (bool, error) {
-	sharedSecret := cfg.RegistrationSharedSecret
-
-	// Check that shared secret registration isn't disabled.
-	if cfg.RegistrationSharedSecret == "" {
-		return false, errors.New("Shared secret registration is disabled")
-	}
-
-	// Double check that username/password don't contain the HMAC delimiters. We should have
-	// already checked this.
-	if strings.Contains(username, "\x00") {
-		return false, errors.New("Username contains invalid character")
-	}
-	if strings.Contains(password, "\x00") {
-		return false, errors.New("Password contains invalid character")
-	}
-	if sharedSecret == "" {
-		return false, errors.New("Shared secret registration is disabled")
-	}
-
-	adminString := "notadmin"
-	if isAdmin {
-		adminString = "admin"
-	}
-	joined := strings.Join([]string{username, password, adminString}, "\x00")
-
-	mac := hmac.New(sha1.New, []byte(sharedSecret))
-	_, err := mac.Write([]byte(joined))
-	if err != nil {
-		return false, err
-	}
-	expectedMAC := mac.Sum(nil)
-
-	return hmac.Equal(givenMac, expectedMAC), nil
-}
-
 // checkFlows checks a single completed flow against another required one. If
 // one contains at least all of the stages that the other does, checkFlows
 // returns true.
@@ -995,3 +934,34 @@ func RegisterAvailable(
 		},
 	}
 }
+
+func handleSharedSecretRegistration(userAPI userapi.UserInternalAPI, sr *SharedSecretRegistration, req *http.Request) util.JSONResponse {
+	ssrr, err := NewSharedSecretRegistrationRequest(req.Body)
+	if err != nil {
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON(fmt.Sprintf("malformed json: %s", err)),
+		}
+	}
+	valid, err := sr.IsValidMacLogin(ssrr.Nonce, ssrr.User, ssrr.Password, ssrr.Admin, ssrr.MacBytes)
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	if !valid {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Forbidden("bad mac"),
+		}
+	}
+	// downcase capitals
+	ssrr.User = strings.ToLower(ssrr.User)
+
+	if resErr := validateUsername(ssrr.User); resErr != nil {
+		return *resErr
+	}
+	if resErr := validatePassword(ssrr.Password); resErr != nil {
+		return *resErr
+	}
+	deviceID := "shared_secret_registration"
+	return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), false, &ssrr.User, &deviceID)
+}

clientapi/routing/register_secret.go

@@ -0,0 +1,99 @@
+package routing
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/util"
+	cache "github.com/patrickmn/go-cache"
+)
+
+type SharedSecretRegistrationRequest struct {
+	User     string `json:"username"`
+	Password string `json:"password"`
+	Nonce    string `json:"nonce"`
+	MacBytes []byte
+	MacStr   string `json:"mac"`
+	Admin    bool   `json:"admin"`
+}
+
+func NewSharedSecretRegistrationRequest(reader io.ReadCloser) (*SharedSecretRegistrationRequest, error) {
+	defer internal.CloseAndLogIfError(context.Background(), reader, "NewSharedSecretRegistrationRequest: failed to close request body")
+	var ssrr SharedSecretRegistrationRequest
+	err := json.NewDecoder(reader).Decode(&ssrr)
+	if err != nil {
+		return nil, err
+	}
+	ssrr.MacBytes, err = hex.DecodeString(ssrr.MacStr)
+	return &ssrr, err
+}
+
+type SharedSecretRegistration struct {
+	sharedSecret string
+	nonces       *cache.Cache
+}
+
+func NewSharedSecretRegistration(sharedSecret string) *SharedSecretRegistration {
+	return &SharedSecretRegistration{
+		sharedSecret: sharedSecret,
+		// nonces live for 5mins, purge every 10mins
+		nonces: cache.New(5*time.Minute, 10*time.Minute),
+	}
+}
+
+func (r *SharedSecretRegistration) GenerateNonce() string {
+	nonce := util.RandomString(16)
+	r.nonces.Set(nonce, true, cache.DefaultExpiration)
+	return nonce
+}
+
+func (r *SharedSecretRegistration) validNonce(nonce string) bool {
+	_, exists := r.nonces.Get(nonce)
+	return exists
+}
+
+func (r *SharedSecretRegistration) IsValidMacLogin(
+	nonce, username, password string,
+	isAdmin bool,
+	givenMac []byte,
+) (bool, error) {
+	// Check that shared secret registration isn't disabled.
+	if r.sharedSecret == "" {
+		return false, errors.New("Shared secret registration is disabled")
+	}
+	if !r.validNonce(nonce) {
+		return false, fmt.Errorf("Incorrect or expired nonce: %s", nonce)
+	}
+
+	// Check that username/password don't contain the HMAC delimiters.
+	if strings.Contains(username, "\x00") {
+		return false, errors.New("Username contains invalid character")
+	}
+	if strings.Contains(password, "\x00") {
+		return false, errors.New("Password contains invalid character")
+	}
+
+	adminString := "notadmin"
+	if isAdmin {
+		adminString = "admin"
+	}
+	joined := strings.Join([]string{nonce, username, password, adminString}, "\x00")
+
+	mac := hmac.New(sha1.New, []byte(r.sharedSecret))
+	_, err := mac.Write([]byte(joined))
+	if err != nil {
+		return false, err
+	}
+	expectedMAC := mac.Sum(nil)
+
+	return hmac.Equal(givenMac, expectedMAC), nil
+}

clientapi/routing/register_secret_test.go

@@ -0,0 +1,43 @@
+package routing
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/patrickmn/go-cache"
+)
+
+func TestSharedSecretRegister(t *testing.T) {
+	// these values have come from a local synapse instance to ensure compatibility
+	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`)
+	sharedSecret := "dendritetest"
+
+	req, err := NewSharedSecretRegistrationRequest(ioutil.NopCloser(bytes.NewBuffer(jsonStr)))
+	if err != nil {
+		t.Fatalf("failed to read request: %s", err)
+	}
+
+	r := NewSharedSecretRegistration(sharedSecret)
+
+	// force the nonce to be known
+	r.nonces.Set(req.Nonce, true, cache.DefaultExpiration)
+
+	valid, err := r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes)
+	if err != nil {
+		t.Fatalf("failed to check for valid mac: %s", err)
+	}
+	if !valid {
+		t.Errorf("mac login failed, wanted success")
+	}
+
+	// modify the mac so it fails
+	req.MacBytes[0] = 0xff
+	valid, err = r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes)
+	if err != nil {
+		t.Fatalf("failed to check for valid mac: %s", err)
+	}
+	if valid {
+		t.Errorf("mac login succeeded, wanted failure")
+	}
+}

clientapi/routing/routing.go

@@ -37,6 +37,7 @@ import (
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
@@ -46,7 +47,7 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	publicAPIMux *mux.Router, cfg *config.ClientAPI,
+	publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI,
 	eduAPI eduServerAPI.EDUServerInputAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
@@ -63,7 +64,9 @@ func Setup(
 	rateLimits := newRateLimits(&cfg.RateLimiting)
 	userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg)
 
-	unstableFeatures := make(map[string]bool)
+	unstableFeatures := map[string]bool{
+		"org.matrix.e2e_cross_signing": true,
+	}
 	for _, msc := range cfg.MSCs.MSCs {
 		unstableFeatures["org.matrix."+msc] = true
 	}
@@ -88,6 +91,32 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	if cfg.RegistrationSharedSecret != "" {
+		logrus.Info("Enabling shared secret registration at /_synapse/admin/v1/register")
+		sr := NewSharedSecretRegistration(cfg.RegistrationSharedSecret)
+		synapseAdminRouter.Handle("/admin/v1/register",
+			httputil.MakeExternalAPI("shared_secret_registration", func(req *http.Request) util.JSONResponse {
+				if req.Method == http.MethodGet {
+					return util.JSONResponse{
+						Code: 200,
+						JSON: struct {
+							Nonce string `json:"nonce"`
+						}{
+							Nonce: sr.GenerateNonce(),
+						},
+					}
+				}
+				if req.Method == http.MethodPost {
+					return handleSharedSecretRegistration(userAPI, sr, req)
+				}
+				return util.JSONResponse{
+					Code: http.StatusMethodNotAllowed,
+					JSON: jsonerror.NotFound("unknown method"),
+				}
+			}),
+		).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
+	}
+
 	r0mux := publicAPIMux.PathPrefix("/r0").Subrouter()
 	unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter()
 
@@ -248,16 +277,21 @@ func Setup(
 		return OnIncomingStateRequest(req.Context(), device, rsAPI, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
+	r0mux.Handle("/rooms/{roomID}/aliases", httputil.MakeAuthAPI("aliases", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return GetAliases(req, rsAPI, device, vars["roomID"])
+	})).Methods(http.MethodGet, http.MethodOptions)
+
 	r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
 		// If there's a trailing slash, remove it
-		eventType := vars["type"]
-		if strings.HasSuffix(eventType, "/") {
-			eventType = eventType[:len(eventType)-1]
-		}
+		eventType := strings.TrimSuffix(vars["type"], "/")
 		eventFormat := req.URL.Query().Get("format") == "event"
 		return OnIncomingStateTypeRequest(req.Context(), device, rsAPI, vars["roomID"], eventType, "", eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
@@ -278,11 +312,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			emptyString := ""
-			eventType := vars["eventType"]
-			// If there's a trailing slash, remove it
-			if strings.HasSuffix(eventType, "/") {
-				eventType = eventType[:len(eventType)-1]
-			}
+			eventType := strings.TrimSuffix(vars["eventType"], "/")
 			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, nil)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -861,6 +891,192 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	// Key Backup Versions (Metadata)
+
+	getBackupKeysVersion := httputil.MakeAuthAPI("get_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return KeyBackupVersion(req, userAPI, device, vars["version"])
+	})
+
+	getLatestBackupKeysVersion := httputil.MakeAuthAPI("get_latest_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		return KeyBackupVersion(req, userAPI, device, "")
+	})
+
+	putBackupKeysVersion := httputil.MakeAuthAPI("put_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return ModifyKeyBackupVersionAuthData(req, userAPI, device, vars["version"])
+	})
+
+	deleteBackupKeysVersion := httputil.MakeAuthAPI("delete_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return DeleteKeyBackupVersion(req, userAPI, device, vars["version"])
+	})
+
+	postNewBackupKeysVersion := httputil.MakeAuthAPI("post_new_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		return CreateKeyBackupVersion(req, userAPI, device)
+	})
+
+	r0mux.Handle("/room_keys/version/{version}", getBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
+	r0mux.Handle("/room_keys/version", getLatestBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
+	r0mux.Handle("/room_keys/version/{version}", putBackupKeysVersion).Methods(http.MethodPut)
+	r0mux.Handle("/room_keys/version/{version}", deleteBackupKeysVersion).Methods(http.MethodDelete)
+	r0mux.Handle("/room_keys/version", postNewBackupKeysVersion).Methods(http.MethodPost, http.MethodOptions)
+
+	unstableMux.Handle("/room_keys/version/{version}", getBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
+	unstableMux.Handle("/room_keys/version", getLatestBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
+	unstableMux.Handle("/room_keys/version/{version}", putBackupKeysVersion).Methods(http.MethodPut)
+	unstableMux.Handle("/room_keys/version/{version}", deleteBackupKeysVersion).Methods(http.MethodDelete)
+	unstableMux.Handle("/room_keys/version", postNewBackupKeysVersion).Methods(http.MethodPost, http.MethodOptions)
+
+	// Inserting E2E Backup Keys
+
+	// Bulk room and session
+	putBackupKeys := httputil.MakeAuthAPI("put_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		version := req.URL.Query().Get("version")
+		if version == "" {
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.InvalidArgumentValue("version must be specified"),
+			}
+		}
+		var reqBody keyBackupSessionRequest
+		resErr := clientutil.UnmarshalJSONRequest(req, &reqBody)
+		if resErr != nil {
+			return *resErr
+		}
+		return UploadBackupKeys(req, userAPI, device, version, &reqBody)
+	})
+
+	// Single room bulk session
+	putBackupKeysRoom := httputil.MakeAuthAPI("put_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		version := req.URL.Query().Get("version")
+		if version == "" {
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.InvalidArgumentValue("version must be specified"),
+			}
+		}
+		roomID := vars["roomID"]
+		var reqBody keyBackupSessionRequest
+		reqBody.Rooms = make(map[string]struct {
+			Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+		})
+		reqBody.Rooms[roomID] = struct {
+			Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+		}{
+			Sessions: map[string]userapi.KeyBackupSession{},
+		}
+		body := reqBody.Rooms[roomID]
+		resErr := clientutil.UnmarshalJSONRequest(req, &body)
+		if resErr != nil {
+			return *resErr
+		}
+		reqBody.Rooms[roomID] = body
+		return UploadBackupKeys(req, userAPI, device, version, &reqBody)
+	})
+
+	// Single room, single session
+	putBackupKeysRoomSession := httputil.MakeAuthAPI("put_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		version := req.URL.Query().Get("version")
+		if version == "" {
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.InvalidArgumentValue("version must be specified"),
+			}
+		}
+		var reqBody userapi.KeyBackupSession
+		resErr := clientutil.UnmarshalJSONRequest(req, &reqBody)
+		if resErr != nil {
+			return *resErr
+		}
+		roomID := vars["roomID"]
+		sessionID := vars["sessionID"]
+		var keyReq keyBackupSessionRequest
+		keyReq.Rooms = make(map[string]struct {
+			Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+		})
+		keyReq.Rooms[roomID] = struct {
+			Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+		}{
+			Sessions: make(map[string]userapi.KeyBackupSession),
+		}
+		keyReq.Rooms[roomID].Sessions[sessionID] = reqBody
+		return UploadBackupKeys(req, userAPI, device, version, &keyReq)
+	})
+
+	r0mux.Handle("/room_keys/keys", putBackupKeys).Methods(http.MethodPut)
+	r0mux.Handle("/room_keys/keys/{roomID}", putBackupKeysRoom).Methods(http.MethodPut)
+	r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", putBackupKeysRoomSession).Methods(http.MethodPut)
+
+	unstableMux.Handle("/room_keys/keys", putBackupKeys).Methods(http.MethodPut)
+	unstableMux.Handle("/room_keys/keys/{roomID}", putBackupKeysRoom).Methods(http.MethodPut)
+	unstableMux.Handle("/room_keys/keys/{roomID}/{sessionID}", putBackupKeysRoomSession).Methods(http.MethodPut)
+
+	// Querying E2E Backup Keys
+
+	getBackupKeys := httputil.MakeAuthAPI("get_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), "", "")
+	})
+
+	getBackupKeysRoom := httputil.MakeAuthAPI("get_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], "")
+	})
+
+	getBackupKeysRoomSession := httputil.MakeAuthAPI("get_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], vars["sessionID"])
+	})
+
+	r0mux.Handle("/room_keys/keys", getBackupKeys).Methods(http.MethodGet, http.MethodOptions)
+	r0mux.Handle("/room_keys/keys/{roomID}", getBackupKeysRoom).Methods(http.MethodGet, http.MethodOptions)
+	r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", getBackupKeysRoomSession).Methods(http.MethodGet, http.MethodOptions)
+
+	unstableMux.Handle("/room_keys/keys", getBackupKeys).Methods(http.MethodGet, http.MethodOptions)
+	unstableMux.Handle("/room_keys/keys/{roomID}", getBackupKeysRoom).Methods(http.MethodGet, http.MethodOptions)
+	unstableMux.Handle("/room_keys/keys/{roomID}/{sessionID}", getBackupKeysRoomSession).Methods(http.MethodGet, http.MethodOptions)
+
+	// Deleting E2E Backup Keys
+
+	// Cross-signing device keys
+
+	postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, keyAPI, device, accountDB, cfg)
+	})
+
+	postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		return UploadCrossSigningDeviceSignatures(req, keyAPI, device)
+	})
+
+	r0mux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/keys/signatures/upload", postDeviceSigningSignatures).Methods(http.MethodPost, http.MethodOptions)
+
+	unstableMux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions)
+	unstableMux.Handle("/keys/signatures/upload", postDeviceSigningSignatures).Methods(http.MethodPost, http.MethodOptions)
+
 	// Supplying a device ID is deprecated.
 	r0mux.Handle("/keys/upload/{deviceID}",
 		httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -874,7 +1090,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/keys/query",
 		httputil.MakeAuthAPI("keys_query", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return QueryKeys(req, keyAPI)
+			return QueryKeys(req, keyAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/keys/claim",

cmd/create-account/main.go

@@ -18,13 +18,18 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"os"
+	"strings"
+	"syscall"
 
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/bcrypt"
+	"golang.org/x/term"
 )
 
 const usage = `Usage: %s
@@ -33,7 +38,15 @@ Creates a new user account on the homeserver.
 
 Example:
 
-  ./create-account --config dendrite.yaml --username alice --password foobarbaz
+	# provide password by parameter
+  	%s --config dendrite.yaml -username alice -password foobarbaz
+	# use password from file
+  	%s --config dendrite.yaml -username alice -passwordfile my.pass
+	# ask user to provide password
+	%s --config dendrite.yaml -username alice -ask-pass
+	# read password from stdin
+	%s --config dendrite.yaml -username alice -passwordstdin < my.pass
+	cat my.pass | %s --config dendrite.yaml -username alice -passwordstdin
 
 Arguments:
 
@@ -42,11 +55,15 @@ Arguments:
 var (
 	username = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
 	password = flag.String("password", "", "The password to associate with the account (optional, account will be password-less if not specified)")
+	pwdFile  = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
+	pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin")
+	askPass  = flag.Bool("ask-pass", false, "Ask for the password to use")
 )
 
 func main() {
+	name := os.Args[0]
 	flag.Usage = func() {
-		fmt.Fprintf(os.Stderr, usage, os.Args[0])
+		_, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name)
 		flag.PrintDefaults()
 	}
 	cfg := setup.ParseFlags(true)
@@ -56,6 +73,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	pass := getPassword(password, pwdFile, pwdStdin, askPass, os.Stdin)
+
 	accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{
 		ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString,
 	}, cfg.Global.ServerName, bcrypt.DefaultCost, cfg.UserAPI.OpenIDTokenLifetimeMS)
@@ -63,10 +82,61 @@ func main() {
 		logrus.Fatalln("Failed to connect to the database:", err.Error())
 	}
 
-	_, err = accountDB.CreateAccount(context.Background(), *username, *password, "")
+	_, err = accountDB.CreateAccount(context.Background(), *username, pass, "")
 	if err != nil {
 		logrus.Fatalln("Failed to create the account:", err.Error())
 	}
 
 	logrus.Infoln("Created account", *username)
 }
+
+func getPassword(password, pwdFile *string, pwdStdin, askPass *bool, r io.Reader) string {
+	// no password option set, use empty password
+	if password == nil && pwdFile == nil && pwdStdin == nil && askPass == nil {
+		return ""
+	}
+	// password defined as parameter
+	if password != nil && *password != "" {
+		return *password
+	}
+
+	// read password from file
+	if pwdFile != nil && *pwdFile != "" {
+		pw, err := ioutil.ReadFile(*pwdFile)
+		if err != nil {
+			logrus.Fatalln("Unable to read password from file:", err)
+		}
+		return strings.TrimSpace(string(pw))
+	}
+
+	// read password from stdin
+	if pwdStdin != nil && *pwdStdin {
+		data, err := ioutil.ReadAll(r)
+		if err != nil {
+			logrus.Fatalln("Unable to read password from stdin:", err)
+		}
+		return strings.TrimSpace(string(data))
+	}
+
+	// ask the user to provide the password
+	if *askPass {
+		fmt.Print("Enter Password: ")
+		bytePassword, err := term.ReadPassword(syscall.Stdin)
+		if err != nil {
+			logrus.Fatalln("Unable to read password:", err)
+		}
+		fmt.Println()
+		fmt.Print("Confirm Password: ")
+		bytePassword2, err := term.ReadPassword(syscall.Stdin)
+		if err != nil {
+			logrus.Fatalln("Unable to read password:", err)
+		}
+		fmt.Println()
+		if strings.TrimSpace(string(bytePassword)) != strings.TrimSpace(string(bytePassword2)) {
+			logrus.Fatalln("Entered passwords don't match")
+		}
+		return strings.TrimSpace(string(bytePassword))
+	}
+
+	return ""
+}

cmd/create-account/main_test.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+func Test_getPassword(t *testing.T) {
+	type args struct {
+		password *string
+		pwdFile  *string
+		pwdStdin *bool
+		askPass  *bool
+		reader   io.Reader
+	}
+
+	pass := "mySecretPass"
+	passwordFile := "testdata/my.pass"
+	passwordStdin := true
+	reader := &bytes.Buffer{}
+	_, err := reader.WriteString(pass)
+	if err != nil {
+		t.Errorf("unable to write to buffer: %+v", err)
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "no password defined",
+			args: args{},
+			want: "",
+		},
+		{
+			name: "password defined",
+			args: args{password: &pass},
+			want: pass,
+		},
+		{
+			name: "pwdFile defined",
+			args: args{pwdFile: &passwordFile},
+			want: pass,
+		},
+		{
+			name: "read pass from stdin defined",
+			args: args{
+				pwdStdin: &passwordStdin,
+				reader:   reader,
+			},
+			want: pass,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getPassword(tt.args.password, tt.args.pwdFile, tt.args.pwdStdin, tt.args.askPass, tt.args.reader); got != tt.want {
+				t.Errorf("getPassword() = '%v', want '%v'", got, tt.want)
+			}
+		})
+	}
+}

cmd/create-account/testdata/my.pass

@@ -0,0 +1 @@
+mySecretPass

cmd/create-room-events/main.go

@@ -1,147 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Generate a list of matrix room events for load testing.
-// Writes the events to stdout by default.
-package main
-
-import (
-	"encoding/base64"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"golang.org/x/crypto/ed25519"
-)
-
-const usage = `Usage: %s
-
-Generate a list of matrix room events for load testing.
-Writes the events to stdout separated by new lines
-
-Arguments:
-
-`
-
-var (
-	serverName       = flag.String("server-name", "localhost", "The name of the matrix server to generate events for")
-	keyID            = flag.String("key-id", "ed25519:auto", "The ID of the key used to sign the events")
-	privateKeyString = flag.String("private-key", defaultKey, "Base64 encoded private key to sign events with")
-	roomID           = flag.String("room-id", "!roomid:$SERVER_NAME", "The room ID to generate events in")
-	userID           = flag.String("user-id", "@userid:$SERVER_NAME", "The user ID to use as the event sender")
-	messageCount     = flag.Int("message-count", 10, "The number of m.room.messsage events to generate")
-	format           = flag.String("Format", "InputRoomEvent", "The output format to use for the messages: InputRoomEvent or Event")
-	ver              = flag.String("version", string(gomatrixserverlib.RoomVersionV1), "Room version to generate events as")
-)
-
-// By default we use a private key of 0.
-const defaultKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
-
-var privateKey ed25519.PrivateKey
-var emptyString = ""
-var now time.Time
-var b gomatrixserverlib.EventBuilder
-var eventID int
-
-func main() {
-	flag.Usage = func() {
-		fmt.Fprintf(os.Stderr, usage, os.Args[0])
-		flag.PrintDefaults()
-	}
-
-	flag.Parse()
-	*userID = strings.Replace(*userID, "$SERVER_NAME", *serverName, 1)
-	*roomID = strings.Replace(*roomID, "$SERVER_NAME", *serverName, 1)
-
-	// Decode the ed25519 private key.
-	privateKeyBytes, err := base64.RawStdEncoding.DecodeString(*privateKeyString)
-	if err != nil {
-		panic(err)
-	}
-	privateKey = ed25519.PrivateKey(privateKeyBytes)
-
-	// Build a m.room.create event.
-	b.Sender = *userID
-	b.RoomID = *roomID
-	b.Type = "m.room.create"
-	b.StateKey = &emptyString
-	b.SetContent(map[string]string{"creator": *userID}) // nolint: errcheck
-	create := buildAndOutput()
-
-	// Build a m.room.member event.
-	b.Type = "m.room.member"
-	b.StateKey = userID
-	b.SetContent(map[string]string{"membership": gomatrixserverlib.Join}) // nolint: errcheck
-	b.AuthEvents = []gomatrixserverlib.EventReference{create}
-	member := buildAndOutput()
-
-	// Build a number of m.room.message events.
-	b.Type = "m.room.message"
-	b.StateKey = nil
-	b.SetContent(map[string]string{"body": "Test Message"}) // nolint: errcheck
-	b.AuthEvents = []gomatrixserverlib.EventReference{create, member}
-	for i := 0; i < *messageCount; i++ {
-		buildAndOutput()
-	}
-}
-
-// Build an event and write the event to the output.
-func buildAndOutput() gomatrixserverlib.EventReference {
-	eventID++
-	now = time.Unix(0, 0)
-	name := gomatrixserverlib.ServerName(*serverName)
-	key := gomatrixserverlib.KeyID(*keyID)
-
-	event, err := b.Build(
-		now, name, key, privateKey,
-		gomatrixserverlib.RoomVersion(*ver),
-	)
-	if err != nil {
-		panic(err)
-	}
-	writeEvent(event)
-	reference := event.EventReference()
-	b.PrevEvents = []gomatrixserverlib.EventReference{reference}
-	b.Depth++
-	return reference
-}
-
-// Write an event to the output.
-func writeEvent(event *gomatrixserverlib.Event) {
-	encoder := json.NewEncoder(os.Stdout)
-	if *format == "InputRoomEvent" {
-		var ire api.InputRoomEvent
-		ire.Kind = api.KindNew
-		ire.Event = event.Headered(gomatrixserverlib.RoomVersion(*ver))
-		authEventIDs := []string{}
-		for _, ref := range b.AuthEvents.([]gomatrixserverlib.EventReference) {
-			authEventIDs = append(authEventIDs, ref.EventID)
-		}
-		ire.AuthEventIDs = authEventIDs
-		if err := encoder.Encode(ire); err != nil {
-			panic(err)
-		}
-	} else if *format == "Event" {
-		if err := encoder.Encode(event); err != nil {
-			panic(err)
-		}
-	} else {
-		panic(fmt.Errorf("Format %q is not valid, must be %q or %q", *format, "InputRoomEvent", "Event"))
-	}
-}

cmd/dendrite-demo-libp2p/main.go

@@ -44,6 +44,8 @@ import (
 	"github.com/matrix-org/dendrite/eduserver/cache"
 
 	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 func createKeyDB(
@@ -145,7 +147,7 @@ func main() {
 
 	accountDB := base.Base.CreateAccountsDB()
 	federation := createFederationClient(base)
-	keyAPI := keyserver.NewInternalAPI(&base.Base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(&base.Base, &base.Base.Cfg.KeyServer, federation)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
@@ -166,7 +168,7 @@ func main() {
 	asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationsender.NewInternalAPI(
-		&base.Base, federation, rsAPI, keyRing,
+		&base.Base, federation, rsAPI, keyRing, true,
 	)
 	rsAPI.SetFederationSenderAPI(fsAPI)
 	provider := newPublicRoomsProvider(base.LibP2PPubsub, rsAPI)
@@ -197,6 +199,7 @@ func main() {
 		base.Base.PublicFederationAPIMux,
 		base.Base.PublicKeyAPIMux,
 		base.Base.PublicMediaAPIMux,
+		base.Base.SynapseAdminMux,
 	)
 	if err := mscs.Enable(&base.Base, &monolith); err != nil {
 		logrus.WithError(err).Fatalf("Failed to enable MSCs")

cmd/dendrite-demo-pinecone/conn/client.go

@@ -0,0 +1,89 @@
+package conn
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/gomatrixserverlib"
+	"nhooyr.io/websocket"
+
+	pineconeRouter "github.com/matrix-org/pinecone/router"
+	pineconeSessions "github.com/matrix-org/pinecone/sessions"
+)
+
+func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error {
+	var parent net.Conn
+	if strings.HasPrefix(peer, "ws://") || strings.HasPrefix(peer, "wss://") {
+		ctx := context.Background()
+		c, _, err := websocket.Dial(ctx, peer, nil)
+		if err != nil {
+			return fmt.Errorf("websocket.DefaultDialer.Dial: %w", err)
+		}
+		parent = websocket.NetConn(ctx, c, websocket.MessageBinary)
+	} else {
+		var err error
+		parent, err = net.Dial("tcp", peer)
+		if err != nil {
+			return fmt.Errorf("net.Dial: %w", err)
+		}
+	}
+	if parent == nil {
+		return fmt.Errorf("failed to wrap connection")
+	}
+	_, err := pRouter.AuthenticatedConnect(parent, "static", pineconeRouter.PeerTypeRemote)
+	return err
+}
+
+type RoundTripper struct {
+	inner *http.Transport
+}
+
+func (y *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.URL.Scheme = "http"
+	return y.inner.RoundTrip(req)
+}
+
+func createTransport(s *pineconeSessions.Sessions) *http.Transport {
+	tr := &http.Transport{
+		DisableKeepAlives: false,
+		Dial:              s.Dial,
+		DialContext:       s.DialContext,
+		DialTLS:           s.DialTLS,
+		DialTLSContext:    s.DialTLSContext,
+	}
+	tr.RegisterProtocol(
+		"matrix", &RoundTripper{
+			inner: &http.Transport{
+				DisableKeepAlives: false,
+				Dial:              s.Dial,
+				DialContext:       s.DialContext,
+				DialTLS:           s.DialTLS,
+				DialTLSContext:    s.DialTLSContext,
+			},
+		},
+	)
+	return tr
+}
+
+func CreateClient(
+	base *setup.BaseDendrite, s *pineconeSessions.Sessions,
+) *gomatrixserverlib.Client {
+	return gomatrixserverlib.NewClient(
+		gomatrixserverlib.WithTransport(createTransport(s)),
+	)
+}
+
+func CreateFederationClient(
+	base *setup.BaseDendrite, s *pineconeSessions.Sessions,
+) *gomatrixserverlib.FederationClient {
+	return gomatrixserverlib.NewFederationClient(
+		base.Cfg.Global.ServerName,
+		base.Cfg.Global.KeyID,
+		base.Cfg.Global.PrivateKey,
+		gomatrixserverlib.WithTransport(createTransport(s)),
+	)
+}

cmd/dendrite-demo-pinecone/conn/ws.go

@@ -0,0 +1,81 @@
+package conn
+
+import (
+	"io"
+	"net"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+func WrapWebSocketConn(c *websocket.Conn) *WebSocketConn {
+	return &WebSocketConn{c: c}
+}
+
+type WebSocketConn struct {
+	r io.Reader
+	c *websocket.Conn
+}
+
+func (c *WebSocketConn) Write(p []byte) (int, error) {
+	err := c.c.WriteMessage(websocket.BinaryMessage, p)
+	if err != nil {
+		return 0, err
+	}
+	return len(p), nil
+}
+
+func (c *WebSocketConn) Read(p []byte) (int, error) {
+	for {
+		if c.r == nil {
+			// Advance to next message.
+			var err error
+			_, c.r, err = c.c.NextReader()
+			if err != nil {
+				return 0, err
+			}
+		}
+		n, err := c.r.Read(p)
+		if err == io.EOF {
+			// At end of message.
+			c.r = nil
+			if n > 0 {
+				return n, nil
+			} else {
+				// No data read, continue to next message.
+				continue
+			}
+		}
+		return n, err
+	}
+}
+
+func (c *WebSocketConn) Close() error {
+	return c.c.Close()
+}
+
+func (c *WebSocketConn) LocalAddr() net.Addr {
+	return c.c.LocalAddr()
+}
+
+func (c *WebSocketConn) RemoteAddr() net.Addr {
+	return c.c.RemoteAddr()
+}
+
+func (c *WebSocketConn) SetDeadline(t time.Time) error {
+	if err := c.SetReadDeadline(t); err != nil {
+		return err
+	}
+	if err := c.SetWriteDeadline(t); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *WebSocketConn) SetReadDeadline(t time.Time) error {
+	return c.c.SetReadDeadline(t)
+}
+
+func (c *WebSocketConn) SetWriteDeadline(t time.Time) error {
+	return c.c.SetWriteDeadline(t)
+}

cmd/dendrite-demo-pinecone/embed/embed_elementweb.go

@@ -1,4 +1,4 @@
-// +build riotweb
+// +build elementweb
 
 package embed
 
@@ -12,8 +12,8 @@ import (
 	"github.com/tidwall/sjson"
 )
 
-// From within the Riot Web directory:
-// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_riotweb.go -private -pkg embed .
+// From within the Element Web directory:
+// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_elementweb.go -private -pkg embed .
 
 var cssFile = regexp.MustCompile("\\.css$")
 var jsFile = regexp.MustCompile("\\.js$")
@@ -36,12 +36,12 @@ func (h mimeFixingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func Embed(rootMux *mux.Router, listenPort int, serverName string) {
-	url := fmt.Sprintf("http://localhost:%d", listenPort)
 	embeddedFS := _escFS(false)
 	embeddedServ := mimeFixingHandler{http.FileServer(embeddedFS)}
 
 	rootMux.NotFoundHandler = embeddedServ
-	rootMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) {
+	rootMux.HandleFunc("/config.json", func(w http.ResponseWriter, r *http.Request) {
+		url := fmt.Sprintf("http://%s:%d", r.Header("Host"), listenPort)
 		configFile, err := embeddedFS.Open("/config.sample.json")
 		if err != nil {
 			w.WriteHeader(500)
@@ -68,7 +68,7 @@ func Embed(rootMux *mux.Router, listenPort int, serverName string) {
 		}
 		js, _ := sjson.SetBytes(buf, "default_server_config.m\\.homeserver.base_url", url)
 		js, _ = sjson.SetBytes(js, "default_server_config.m\\.homeserver.server_name", serverName)
-		js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Riot %s", serverName))
+		js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Element %s", serverName))
 		js, _ = sjson.SetBytes(js, "disable_guests", true)
 		js, _ = sjson.SetBytes(js, "disable_3pid_login", true)
 		js, _ = sjson.DeleteBytes(js, "welcomeUserId")
@@ -76,7 +76,7 @@ func Embed(rootMux *mux.Router, listenPort int, serverName string) {
 	})
 
 	fmt.Println("*-------------------------------*")
-	fmt.Println("| This build includes Riot Web! |")
+	fmt.Println("| This build includes Element Web! |")
 	fmt.Println("*-------------------------------*")
 	fmt.Println("Point your browser to:", url)
 	fmt.Println()

cmd/dendrite-demo-pinecone/embed/embed_other.go

@@ -0,0 +1,9 @@
+// +build !elementweb
+
+package embed
+
+import "github.com/gorilla/mux"
+
+func Embed(_ *mux.Router, _ int, _ string) {
+
+}

cmd/dendrite-demo-pinecone/main.go

@@ -0,0 +1,282 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/tls"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/gorilla/websocket"
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/embed"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
+	"github.com/matrix-org/pinecone/router"
+	pineconeRouter "github.com/matrix-org/pinecone/router"
+	pineconeSessions "github.com/matrix-org/pinecone/sessions"
+
+	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+var (
+	instanceName   = flag.String("name", "dendrite-p2p-pinecone", "the name of this P2P demo instance")
+	instancePort   = flag.Int("port", 8008, "the port that the client API will listen on")
+	instancePeer   = flag.String("peer", "", "the static Pinecone peer to connect to")
+	instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to")
+)
+
+// nolint:gocyclo
+func main() {
+	flag.Parse()
+	internal.SetupPprof()
+
+	var pk ed25519.PublicKey
+	var sk ed25519.PrivateKey
+
+	keyfile := *instanceName + ".key"
+	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
+		if pk, sk, err = ed25519.GenerateKey(nil); err != nil {
+			panic(err)
+		}
+		if err = ioutil.WriteFile(keyfile, sk, 0644); err != nil {
+			panic(err)
+		}
+	} else if err == nil {
+		if sk, err = ioutil.ReadFile(keyfile); err != nil {
+			panic(err)
+		}
+		if len(sk) != ed25519.PrivateKeySize {
+			panic("the private key is not long enough")
+		}
+		pk = sk.Public().(ed25519.PublicKey)
+	}
+
+	logger := log.New(os.Stdout, "", 0)
+	pRouter := pineconeRouter.NewRouter(logger, "dendrite", sk, pk, nil)
+
+	go func() {
+		listener, err := net.Listen("tcp", *instanceListen)
+		if err != nil {
+			panic(err)
+		}
+
+		fmt.Println("Listening on", listener.Addr())
+
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				logrus.WithError(err).Error("listener.Accept failed")
+				continue
+			}
+
+			port, err := pRouter.AuthenticatedConnect(conn, "", pineconeRouter.PeerTypeRemote)
+			if err != nil {
+				logrus.WithError(err).Error("pSwitch.AuthenticatedConnect failed")
+				continue
+			}
+
+			fmt.Println("Inbound connection", conn.RemoteAddr(), "is connected to port", port)
+		}
+	}()
+
+	pQUIC := pineconeSessions.NewSessions(logger, pRouter)
+	pMulticast := pineconeMulticast.NewMulticast(logger, pRouter)
+	pMulticast.Start()
+
+	connectToStaticPeer := func() {
+		attempt := func() {
+			if pRouter.PeerCount(router.PeerTypeRemote) == 0 {
+				uri := *instancePeer
+				if uri == "" {
+					return
+				}
+				if err := conn.ConnectToPeer(pRouter, uri); err != nil {
+					logrus.WithError(err).Error("Failed to connect to static peer")
+				}
+			}
+		}
+		for {
+			attempt()
+			time.Sleep(time.Second * 5)
+		}
+	}
+
+	cfg := &config.Dendrite{}
+	cfg.Defaults()
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.PrivateKey = sk
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+	cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-signingkeyserver.db", *instanceName))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
+	cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
+	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
+	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+	if err := cfg.Derive(); err != nil {
+		panic(err)
+	}
+
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	federation := conn.CreateFederationClient(base, pQUIC)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+
+	rsComponent := roomserver.NewInternalAPI(
+		base, keyRing,
+	)
+	rsAPI := rsComponent
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing, true,
+	)
+
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
+	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
+	keyAPI.SetUserAPI(userAPI)
+
+	eduInputAPI := eduserver.NewInternalAPI(
+		base, cache.New(), userAPI,
+	)
+
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+
+	rsComponent.SetFederationSenderAPI(fsAPI)
+
+	monolith := setup.Monolith{
+		Config:    base.Cfg,
+		AccountDB: accountDB,
+		Client:    conn.CreateClient(base, pQUIC),
+		FedClient: federation,
+		KeyRing:   keyRing,
+
+		AppserviceAPI:          asAPI,
+		EDUInternalAPI:         eduInputAPI,
+		FederationSenderAPI:    fsAPI,
+		RoomserverAPI:          rsAPI,
+		UserAPI:                userAPI,
+		KeyAPI:                 keyAPI,
+		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation),
+	}
+	monolith.AddAllPublicRoutes(
+		base.ProcessContext,
+		base.PublicClientAPIMux,
+		base.PublicFederationAPIMux,
+		base.PublicKeyAPIMux,
+		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
+	)
+
+	wsUpgrader := websocket.Upgrader{
+		CheckOrigin: func(_ *http.Request) bool {
+			return true
+		},
+	}
+	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
+		c, err := wsUpgrader.Upgrade(w, r, nil)
+		if err != nil {
+			logrus.WithError(err).Error("Failed to upgrade WebSocket connection")
+			return
+		}
+		conn := conn.WrapWebSocketConn(c)
+		if _, err = pRouter.AuthenticatedConnect(conn, "websocket", pineconeRouter.PeerTypeRemote); err != nil {
+			logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
+		}
+	})
+	embed.Embed(httpRouter, *instancePort, "Pinecone Demo")
+
+	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
+	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+
+	pHTTP := pQUIC.HTTP()
+	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
+	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
+
+	// Build both ends of a HTTP multiplex.
+	httpServer := &http.Server{
+		Addr:         ":0",
+		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+		ReadTimeout:  10 * time.Second,
+		WriteTimeout: 10 * time.Second,
+		IdleTimeout:  60 * time.Second,
+		BaseContext: func(_ net.Listener) context.Context {
+			return context.Background()
+		},
+		Handler: pMux,
+	}
+
+	go connectToStaticPeer()
+	go func() {
+		pubkey := pRouter.PublicKey()
+		logrus.Info("Listening on ", hex.EncodeToString(pubkey[:]))
+		logrus.Fatal(httpServer.Serve(pQUIC))
+	}()
+	go func() {
+		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
+		logrus.Info("Listening on ", httpBindAddr)
+		logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
+	}()
+	go func() {
+		logrus.Info("Sending wake-up message to known nodes")
+		req := &api.PerformBroadcastEDURequest{}
+		res := &api.PerformBroadcastEDUResponse{}
+		if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil {
+			logrus.WithError(err).Error("Failed to send wake-up message to known nodes")
+		}
+	}()
+
+	base.WaitForShutdown()
+}

cmd/dendrite-demo-pinecone/rooms/rooms.go

@@ -0,0 +1,117 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rooms
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
+	pineconeRouter "github.com/matrix-org/pinecone/router"
+	pineconeSessions "github.com/matrix-org/pinecone/sessions"
+)
+
+type PineconeRoomProvider struct {
+	r         *pineconeRouter.Router
+	s         *pineconeSessions.Sessions
+	fedSender api.FederationSenderInternalAPI
+	fedClient *gomatrixserverlib.FederationClient
+}
+
+func NewPineconeRoomProvider(
+	r *pineconeRouter.Router,
+	s *pineconeSessions.Sessions,
+	fedSender api.FederationSenderInternalAPI,
+	fedClient *gomatrixserverlib.FederationClient,
+) *PineconeRoomProvider {
+	p := &PineconeRoomProvider{
+		r:         r,
+		s:         s,
+		fedSender: fedSender,
+		fedClient: fedClient,
+	}
+	return p
+}
+
+func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
+	known := p.r.KnownNodes()
+	//known = append(known, p.s.Sessions()...)
+	list := []gomatrixserverlib.ServerName{}
+	for _, k := range known {
+		list = append(list, gomatrixserverlib.ServerName(k.String()))
+	}
+	return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list)
+}
+
+// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
+// Returns a list of public rooms.
+func bulkFetchPublicRoomsFromServers(
+	ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
+	homeservers []gomatrixserverlib.ServerName,
+) (publicRooms []gomatrixserverlib.PublicRoom) {
+	limit := 200
+	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
+	// goroutines send rooms to this channel
+	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
+	// signalling channel to tell goroutines to stop sending rooms and quit
+	done := make(chan bool)
+	// signalling to say when we can close the room channel
+	var wg sync.WaitGroup
+	wg.Add(len(homeservers))
+	// concurrently query for public rooms
+	reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5)
+	for _, hs := range homeservers {
+		go func(homeserverDomain gomatrixserverlib.ServerName) {
+			defer wg.Done()
+			util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
+			fres, err := fedClient.GetPublicRooms(reqctx, homeserverDomain, int(limit), "", false, "")
+			if err != nil {
+				util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn(
+					"bulkFetchPublicRoomsFromServers: failed to query hs",
+				)
+				return
+			}
+			for _, room := range fres.Chunk {
+				// atomically send a room or stop
+				select {
+				case roomCh <- room:
+				case <-done:
+				case <-reqctx.Done():
+					util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms")
+					return
+				}
+			}
+		}(hs)
+	}
+
+	select {
+	case <-time.After(5 * time.Second):
+	default:
+		wg.Wait()
+	}
+	reqcancel()
+	close(done)
+	close(roomCh)
+
+	for room := range roomCh {
+		publicRooms = append(publicRooms, room)
+	}
+
+	return publicRooms
+}

cmd/dendrite-demo-yggdrasil/README.md

@@ -1,6 +1,6 @@
 # Yggdrasil Demo
 
-This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.13 or later.
+This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.15 or later.
 
 To run the homeserver, start at the root of the Dendrite repository and run:
 

cmd/dendrite-demo-yggdrasil/convert/25519.go

@@ -1,53 +0,0 @@
-// Copyright 2019 Google LLC
-//
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file or at
-// https://developers.google.com/open-source/licenses/bsd
-//
-// Original code from https://github.com/FiloSottile/age/blob/bbab440e198a4d67ba78591176c7853e62d29e04/internal/age/ssh.go
-
-package convert
-
-import (
-	"crypto/ed25519"
-	"crypto/sha512"
-	"math/big"
-
-	"golang.org/x/crypto/curve25519"
-)
-
-var curve25519P, _ = new(big.Int).SetString("57896044618658097711785492504343953926634992332820282019728792003956564819949", 10)
-
-func Ed25519PrivateKeyToCurve25519(pk ed25519.PrivateKey) []byte {
-	h := sha512.New()
-	_, _ = h.Write(pk.Seed())
-	out := h.Sum(nil)
-	return out[:curve25519.ScalarSize]
-}
-
-func Ed25519PublicKeyToCurve25519(pk ed25519.PublicKey) []byte {
-	// ed25519.PublicKey is a little endian representation of the y-coordinate,
-	// with the most significant bit set based on the sign of the x-coordinate.
-	bigEndianY := make([]byte, ed25519.PublicKeySize)
-	for i, b := range pk {
-		bigEndianY[ed25519.PublicKeySize-i-1] = b
-	}
-	bigEndianY[0] &= 0b0111_1111
-
-	// The Montgomery u-coordinate is derived through the bilinear map
-	//     u = (1 + y) / (1 - y)
-	// See https://blog.filippo.io/using-ed25519-keys-for-encryption.
-	y := new(big.Int).SetBytes(bigEndianY)
-	denom := big.NewInt(1)
-	denom.ModInverse(denom.Sub(denom, y), curve25519P) // 1 / (1 - y)
-	u := y.Mul(y.Add(y, big.NewInt(1)), denom)
-	u.Mod(u, curve25519P)
-
-	out := make([]byte, curve25519.PointSize)
-	uBytes := u.Bytes()
-	for i, b := range uBytes {
-		out[len(uBytes)-i-1] = b
-	}
-
-	return out
-}

cmd/dendrite-demo-yggdrasil/convert/25519_test.go

@@ -1,51 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package convert
-
-import (
-	"bytes"
-	"crypto/ed25519"
-	"encoding/hex"
-	"testing"
-
-	"golang.org/x/crypto/curve25519"
-)
-
-func TestKeyConversion(t *testing.T) {
-	edPub, edPriv, err := ed25519.GenerateKey(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("Signing public:", hex.EncodeToString(edPub))
-	t.Log("Signing private:", hex.EncodeToString(edPriv))
-
-	cuPriv := Ed25519PrivateKeyToCurve25519(edPriv)
-	t.Log("Encryption private:", hex.EncodeToString(cuPriv))
-
-	cuPub := Ed25519PublicKeyToCurve25519(edPub)
-	t.Log("Converted encryption public:", hex.EncodeToString(cuPub))
-
-	var realPub, realPriv [32]byte
-	copy(realPriv[:32], cuPriv[:32])
-	curve25519.ScalarBaseMult(&realPub, &realPriv)
-	t.Log("Scalar-multed encryption public:", hex.EncodeToString(realPub[:]))
-
-	if !bytes.Equal(realPriv[:], cuPriv[:]) {
-		t.Fatal("Private keys should be equal (this means the test is broken)")
-	}
-	if !bytes.Equal(realPub[:], cuPub[:]) {
-		t.Fatal("Public keys should be equal")
-	}
-}

cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go

@@ -0,0 +1,83 @@
+// +build elementweb
+
+package embed
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+
+	"github.com/gorilla/mux"
+	"github.com/tidwall/sjson"
+)
+
+// From within the Element Web directory:
+// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_elementweb.go -private -pkg embed .
+
+var cssFile = regexp.MustCompile("\\.css$")
+var jsFile = regexp.MustCompile("\\.js$")
+
+type mimeFixingHandler struct {
+	fs http.Handler
+}
+
+func (h mimeFixingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ruri := r.RequestURI
+	fmt.Println(ruri)
+	switch {
+	case cssFile.MatchString(ruri):
+		w.Header().Set("Content-Type", "text/css")
+	case jsFile.MatchString(ruri):
+		w.Header().Set("Content-Type", "application/javascript")
+	default:
+	}
+	h.fs.ServeHTTP(w, r)
+}
+
+func Embed(rootMux *mux.Router, listenPort int, serverName string) {
+	url := fmt.Sprintf("http://localhost:%d", listenPort)
+	embeddedFS := _escFS(false)
+	embeddedServ := mimeFixingHandler{http.FileServer(embeddedFS)}
+
+	rootMux.NotFoundHandler = embeddedServ
+	rootMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) {
+		configFile, err := embeddedFS.Open("/config.sample.json")
+		if err != nil {
+			w.WriteHeader(500)
+			io.WriteString(w, "Couldn't open the file: "+err.Error())
+			return
+		}
+		configFileInfo, err := configFile.Stat()
+		if err != nil {
+			w.WriteHeader(500)
+			io.WriteString(w, "Couldn't stat the file: "+err.Error())
+			return
+		}
+		buf := make([]byte, configFileInfo.Size())
+		n, err := configFile.Read(buf)
+		if err != nil {
+			w.WriteHeader(500)
+			io.WriteString(w, "Couldn't read the file: "+err.Error())
+			return
+		}
+		if int64(n) != configFileInfo.Size() {
+			w.WriteHeader(500)
+			io.WriteString(w, "The returned file size didn't match what we expected")
+			return
+		}
+		js, _ := sjson.SetBytes(buf, "default_server_config.m\\.homeserver.base_url", url)
+		js, _ = sjson.SetBytes(js, "default_server_config.m\\.homeserver.server_name", serverName)
+		js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Element %s", serverName))
+		js, _ = sjson.SetBytes(js, "disable_guests", true)
+		js, _ = sjson.SetBytes(js, "disable_3pid_login", true)
+		js, _ = sjson.DeleteBytes(js, "welcomeUserId")
+		_, _ = w.Write(js)
+	})
+
+	fmt.Println("*----------------------------------*")
+	fmt.Println("| This build includes Element Web! |")
+	fmt.Println("*----------------------------------*")
+	fmt.Println("Point your browser to:", url)
+	fmt.Println()
+}

cmd/dendrite-demo-yggdrasil/embed/embed_other.go

@@ -1,4 +1,4 @@
-// +build !riotweb
+// +build !elementweb
 
 package embed
 

cmd/dendrite-demo-yggdrasil/main.go

@@ -44,6 +44,8 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 var (
@@ -56,21 +58,23 @@ func main() {
 	flag.Parse()
 	internal.SetupPprof()
 
-	ygg, err := yggconn.Setup(*instanceName, ".")
+	ygg, err := yggconn.Setup(*instanceName, ".", *instancePeer)
 	if err != nil {
 		panic(err)
 	}
-	ygg.SetMulticastEnabled(true)
-	if instancePeer != nil && *instancePeer != "" {
-		if err = ygg.SetStaticPeer(*instancePeer); err != nil {
-			logrus.WithError(err).Error("Failed to set static peer")
+	/*
+		ygg.SetMulticastEnabled(true)
+		if instancePeer != nil && *instancePeer != "" {
+			if err = ygg.SetStaticPeer(*instancePeer); err != nil {
+				logrus.WithError(err).Error("Failed to set static peer")
+			}
 		}
-	}
+	*/
 
 	cfg := &config.Dendrite{}
 	cfg.Defaults()
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Global.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.Kafka.UseNaffka = true
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
@@ -98,7 +102,7 @@ func main() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
@@ -114,21 +118,9 @@ func main() {
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationsender.NewInternalAPI(
-		base, federation, rsAPI, keyRing,
+		base, federation, rsAPI, keyRing, true,
 	)
 
-	ygg.SetSessionFunc(func(address string) {
-		req := &api.PerformServersAliveRequest{
-			Servers: []gomatrixserverlib.ServerName{
-				gomatrixserverlib.ServerName(address),
-			},
-		}
-		res := &api.PerformServersAliveResponse{}
-		if err := fsAPI.PerformServersAlive(context.TODO(), req, res); err != nil {
-			logrus.WithError(err).Error("Failed to send wake-up message to newly connected node")
-		}
-	})
-
 	rsComponent.SetFederationSenderAPI(fsAPI)
 
 	monolith := setup.Monolith{
@@ -154,6 +146,7 @@ func main() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 	if err := mscs.Enable(base, &monolith); err != nil {
 		logrus.WithError(err).Fatalf("Failed to enable MSCs")

cmd/dendrite-demo-yggdrasil/yggconn/client.go

@@ -51,7 +51,6 @@ func (n *Node) CreateFederationClient(
 				ResponseHeaderTimeout: 10 * time.Second,
 				IdleConnTimeout:       30 * time.Second,
 				DialContext:           n.DialerContext,
-				TLSClientConfig:       n.tlsConfig,
 			},
 		},
 	)

cmd/dendrite-demo-yggdrasil/yggconn/node.go

@@ -17,7 +17,6 @@ package yggconn
 import (
 	"context"
 	"crypto/ed25519"
-	"crypto/tls"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -26,60 +25,48 @@ import (
 	"net"
 	"os"
 	"strings"
-	"sync"
-	"time"
 
 	"github.com/lucas-clemente/quic-go"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
 	"github.com/matrix-org/gomatrixserverlib"
-	"go.uber.org/atomic"
+	"github.com/neilalexander/utp"
 
+	ironwoodtypes "github.com/Arceliar/ironwood/types"
 	yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config"
+	yggdrasilcore "github.com/yggdrasil-network/yggdrasil-go/src/core"
+	yggdrasildefaults "github.com/yggdrasil-network/yggdrasil-go/src/defaults"
 	yggdrasilmulticast "github.com/yggdrasil-network/yggdrasil-go/src/multicast"
-	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
 
 	gologme "github.com/gologme/log"
 )
 
 type Node struct {
-	core         *yggdrasil.Core
-	config       *yggdrasilconfig.NodeConfig
-	state        *yggdrasilconfig.NodeState
-	multicast    *yggdrasilmulticast.Multicast
-	log          *gologme.Logger
-	listener     quic.Listener
-	tlsConfig    *tls.Config
-	quicConfig   *quic.Config
-	sessions     sync.Map // string -> *session
-	sessionCount atomic.Uint32
-	sessionFunc  func(address string)
-	coords       sync.Map // string -> yggdrasil.Coords
-	incoming     chan QUICStream
-	NewSession   func(remote gomatrixserverlib.ServerName)
+	core      *yggdrasilcore.Core
+	config    *yggdrasilconfig.NodeConfig
+	multicast *yggdrasilmulticast.Multicast
+	log       *gologme.Logger
+	listener  quic.Listener
+	utpSocket *utp.Socket
+	incoming  chan net.Conn
 }
 
-func (n *Node) Dialer(_, address string) (net.Conn, error) {
+func (n *Node) DialerContext(ctx context.Context, _, address string) (net.Conn, error) {
 	tokens := strings.Split(address, ":")
 	raw, err := hex.DecodeString(tokens[0])
 	if err != nil {
 		return nil, fmt.Errorf("hex.DecodeString: %w", err)
 	}
-	converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw))
-	convhex := hex.EncodeToString(converted)
-	return n.Dial("curve25519", convhex)
+	pk := make(ironwoodtypes.Addr, ed25519.PublicKeySize)
+	copy(pk, raw[:])
+	return n.utpSocket.DialAddrContext(ctx, pk)
 }
 
-func (n *Node) DialerContext(ctx context.Context, network, address string) (net.Conn, error) {
-	return n.Dialer(network, address)
-}
-
-func Setup(instanceName, storageDirectory string) (*Node, error) {
+func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) {
 	n := &Node{
-		core:      &yggdrasil.Core{},
-		config:    yggdrasilconfig.GenerateConfig(),
+		core:      &yggdrasilcore.Core{},
+		config:    yggdrasildefaults.GenerateConfig(),
 		multicast: &yggdrasilmulticast.Multicast{},
 		log:       gologme.New(os.Stdout, "YGG ", log.Flags()),
-		incoming:  make(chan QUICStream),
+		incoming:  make(chan net.Conn),
 	}
 
 	yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
@@ -93,24 +80,11 @@ func Setup(instanceName, storageDirectory string) (*Node, error) {
 		}
 	}
 
-	n.core.SetCoordChangeCallback(func(old, new yggdrasil.Coords) {
-		fmt.Println("COORDINATE CHANGE!")
-		fmt.Println("Old:", old)
-		fmt.Println("New:", new)
-		n.sessions.Range(func(k, v interface{}) bool {
-			if s, ok := v.(*session); ok {
-				fmt.Println("Killing session", k)
-				s.kill()
-			}
-			return true
-		})
-	})
-
 	n.config.Peers = []string{}
+	if peerURI != "" {
+		n.config.Peers = append(n.config.Peers, peerURI)
+	}
 	n.config.AdminListen = "none"
-	n.config.MulticastInterfaces = []string{}
-	n.config.EncryptionPrivateKey = hex.EncodeToString(n.EncryptionPrivateKey())
-	n.config.EncryptionPublicKey = hex.EncodeToString(n.EncryptionPublicKey())
 
 	j, err := json.MarshalIndent(n.config, "", "  ")
 	if err != nil {
@@ -123,34 +97,22 @@ func Setup(instanceName, storageDirectory string) (*Node, error) {
 	n.log.EnableLevel("error")
 	n.log.EnableLevel("warn")
 	n.log.EnableLevel("info")
-	n.state, err = n.core.Start(n.config, n.log)
+	if err = n.core.Start(n.config, n.log); err != nil {
+		panic(err)
+	}
+	n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core)
 	if err != nil {
 		panic(err)
 	}
-	if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil {
+	if err = n.multicast.Init(n.core, n.config, n.log, nil); err != nil {
 		panic(err)
 	}
 	if err = n.multicast.Start(); err != nil {
 		panic(err)
 	}
 
-	n.tlsConfig = n.generateTLSConfig()
-	n.quicConfig = &quic.Config{
-		MaxIncomingStreams:    0,
-		MaxIncomingUniStreams: 0,
-		KeepAlive:             true,
-		MaxIdleTimeout:        time.Minute * 30,
-		HandshakeTimeout:      time.Second * 15,
-	}
-	copy(n.quicConfig.StatelessResetKey, n.EncryptionPublicKey())
-
-	n.log.Println("Public curve25519:", n.core.EncryptionPublicKey())
-	n.log.Println("Public ed25519:", n.core.SigningPublicKey())
-
-	go func() {
-		time.Sleep(time.Second)
-		n.listenFromYgg()
-	}()
+	n.log.Println("Public key:", n.core.PublicKey())
+	go n.listenFromYgg()
 
 	return n, nil
 }
@@ -163,64 +125,33 @@ func (n *Node) Stop() {
 }
 
 func (n *Node) DerivedServerName() string {
-	return hex.EncodeToString(n.SigningPublicKey())
+	return hex.EncodeToString(n.PublicKey())
 }
 
-func (n *Node) DerivedSessionName() string {
-	return hex.EncodeToString(n.EncryptionPublicKey())
+func (n *Node) PrivateKey() ed25519.PrivateKey {
+	sk := make(ed25519.PrivateKey, ed25519.PrivateKeySize)
+	sb, err := hex.DecodeString(n.config.PrivateKey)
+	if err == nil {
+		copy(sk, sb[:])
+	} else {
+		panic(err)
+	}
+	return sk
 }
 
-func (n *Node) EncryptionPublicKey() []byte {
-	edkey := n.SigningPublicKey()
-	return convert.Ed25519PublicKeyToCurve25519(edkey)
-}
-
-func (n *Node) EncryptionPrivateKey() []byte {
-	edkey := n.SigningPrivateKey()
-	return convert.Ed25519PrivateKeyToCurve25519(edkey)
-}
-
-func (n *Node) SigningPublicKey() ed25519.PublicKey {
-	pubBytes, _ := hex.DecodeString(n.config.SigningPublicKey)
-	return ed25519.PublicKey(pubBytes)
-}
-
-func (n *Node) SigningPrivateKey() ed25519.PrivateKey {
-	privBytes, _ := hex.DecodeString(n.config.SigningPrivateKey)
-	return ed25519.PrivateKey(privBytes)
-}
-
-func (n *Node) SetSessionFunc(f func(address string)) {
-	n.sessionFunc = f
+func (n *Node) PublicKey() ed25519.PublicKey {
+	return n.core.PublicKey()
 }
 
 func (n *Node) PeerCount() int {
-	return len(n.core.GetPeers()) - 1
-}
-
-func (n *Node) SessionCount() int {
-	return int(n.sessionCount.Load())
+	return len(n.core.GetPeers())
 }
 
 func (n *Node) KnownNodes() []gomatrixserverlib.ServerName {
-	nodemap := map[string]struct{}{
-		//"b5ae50589e50991dd9dd7d59c5c5f7a4521e8da5b603b7f57076272abc58b374": {},
+	nodemap := map[string]struct{}{}
+	for _, peer := range n.core.GetPeers() {
+		nodemap[hex.EncodeToString(peer.Key)] = struct{}{}
 	}
-	for _, peer := range n.core.GetSwitchPeers() {
-		nodemap[hex.EncodeToString(peer.SigPublicKey[:])] = struct{}{}
-	}
-	n.sessions.Range(func(_, v interface{}) bool {
-		session, ok := v.(quic.Session)
-		if !ok {
-			return true
-		}
-		if len(session.ConnectionState().PeerCertificates) != 1 {
-			return true
-		}
-		subjectName := session.ConnectionState().PeerCertificates[0].Subject.CommonName
-		nodemap[subjectName] = struct{}{}
-		return true
-	})
 	var nodes []gomatrixserverlib.ServerName
 	for node := range nodemap {
 		nodes = append(nodes, gomatrixserverlib.ServerName(node))
@@ -229,53 +160,22 @@ func (n *Node) KnownNodes() []gomatrixserverlib.ServerName {
 }
 
 func (n *Node) SetMulticastEnabled(enabled bool) {
-	if enabled {
-		n.config.MulticastInterfaces = []string{".*"}
-	} else {
-		n.config.MulticastInterfaces = []string{}
-	}
-	n.multicast.UpdateConfig(n.config)
-	if !enabled {
-		n.DisconnectMulticastPeers()
-	}
+	// TODO: There's no dynamic reconfiguration in Yggdrasil v0.4
+	// so we need a solution for this.
 }
 
 func (n *Node) DisconnectMulticastPeers() {
-	for _, sp := range n.core.GetSwitchPeers() {
-		if !strings.HasPrefix(sp.Endpoint, "fe80") {
-			continue
-		}
-		if err := n.core.DisconnectPeer(sp.Port); err != nil {
-			n.log.Printf("Failed to disconnect port %d: %s", sp.Port, err)
-		}
-	}
+	// TODO: There's no dynamic reconfiguration in Yggdrasil v0.4
+	// so we need a solution for this.
 }
 
 func (n *Node) DisconnectNonMulticastPeers() {
-	for _, sp := range n.core.GetSwitchPeers() {
-		if strings.HasPrefix(sp.Endpoint, "fe80") {
-			continue
-		}
-		if err := n.core.DisconnectPeer(sp.Port); err != nil {
-			n.log.Printf("Failed to disconnect port %d: %s", sp.Port, err)
-		}
-	}
+	// TODO: There's no dynamic reconfiguration in Yggdrasil v0.4
+	// so we need a solution for this.
 }
 
 func (n *Node) SetStaticPeer(uri string) error {
-	n.config.Peers = []string{}
-	n.core.UpdateConfig(n.config)
-	n.DisconnectNonMulticastPeers()
-	if uri != "" {
-		n.log.Infoln("Adding static peer", uri)
-		if err := n.core.AddPeer(uri, ""); err != nil {
-			n.log.Warnln("Adding static peer failed:", err)
-			return err
-		}
-		if err := n.core.CallPeer(uri, ""); err != nil {
-			n.log.Warnln("Calling static peer failed:", err)
-			return err
-		}
-	}
+	// TODO: There's no dynamic reconfiguration in Yggdrasil v0.4
+	// so we need a solution for this.
 	return nil
 }

cmd/dendrite-demo-yggdrasil/yggconn/session.go

@@ -16,94 +16,17 @@ package yggconn
 
 import (
 	"context"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/hex"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	"math/big"
 	"net"
-	"time"
-
-	"github.com/lucas-clemente/quic-go"
-	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
 )
 
-type session struct {
-	node    *Node
-	session quic.Session
-	address string
-	context context.Context
-	cancel  context.CancelFunc
-}
-
-func (n *Node) newSession(sess quic.Session, address string) *session {
-	ctx, cancel := context.WithCancel(context.TODO())
-	return &session{
-		node:    n,
-		session: sess,
-		address: address,
-		context: ctx,
-		cancel:  cancel,
-	}
-}
-
-func (s *session) kill() {
-	s.cancel()
-}
-
 func (n *Node) listenFromYgg() {
-	var err error
-	n.listener, err = quic.Listen(
-		n.core,       // yggdrasil.PacketConn
-		n.tlsConfig,  // TLS config
-		n.quicConfig, // QUIC config
-	)
-	if err != nil {
-		panic(err)
-	}
-
 	for {
-		n.log.Infoln("Waiting to accept QUIC sessions")
-		session, err := n.listener.Accept(context.TODO())
+		conn, err := n.utpSocket.Accept()
 		if err != nil {
-			n.log.Println("n.listener.Accept:", err)
+			n.log.Println("n.utpSocket.Accept:", err)
 			return
 		}
-		if len(session.ConnectionState().PeerCertificates) != 1 {
-			_ = session.CloseWithError(0, "expected a peer certificate")
-			continue
-		}
-		address := session.ConnectionState().PeerCertificates[0].DNSNames[0]
-		n.log.Infoln("Accepted connection from", address)
-		go n.newSession(session, address).listenFromQUIC()
-		go n.sessionFunc(address)
-	}
-}
-
-func (s *session) listenFromQUIC() {
-	if existing, ok := s.node.sessions.Load(s.address); ok {
-		if existingSession, ok := existing.(*session); ok {
-			fmt.Println("Killing existing session to replace", s.address)
-			existingSession.kill()
-		}
-	}
-	s.node.sessionCount.Inc()
-	s.node.sessions.Store(s.address, s)
-	defer s.node.sessions.Delete(s.address)
-	defer s.node.sessionCount.Dec()
-	for {
-		st, err := s.session.AcceptStream(s.context)
-		if err != nil {
-			s.node.log.Println("session.AcceptStream:", err)
-			return
-		}
-		s.node.incoming <- QUICStream{st, s.session}
+		n.incoming <- conn
 	}
 }
 
@@ -129,155 +52,5 @@ func (n *Node) Dial(network, address string) (net.Conn, error) {
 
 // Implements http.Transport.DialContext
 func (n *Node) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
-	s, ok1 := n.sessions.Load(address)
-	session, ok2 := s.(*session)
-	if !ok1 || !ok2 {
-		// First of all, check if we think we know the coords of this
-		// node. If we do then we'll try to dial to it directly. This
-		// will either succeed or fail.
-		if v, ok := n.coords.Load(address); ok {
-			coords, ok := v.(yggdrasil.Coords)
-			if !ok {
-				n.coords.Delete(address)
-				return nil, errors.New("should have found yggdrasil.Coords but didn't")
-			}
-			n.log.Infof("Coords %s for %q cached, trying to dial", coords.String(), address)
-			var err error
-			// We think we know the coords. Try to dial the node.
-			if session, err = n.tryDial(address, coords); err != nil {
-				// We thought we knew the coords but it didn't result
-				// in a successful dial. Nuke them from the cache.
-				n.coords.Delete(address)
-				n.log.Infof("Cached coords %s for %q failed", coords.String(), address)
-			}
-		}
-
-		// We either don't know the coords for the node, or we failed
-		// to dial it before, in which case try to resolve the coords.
-		if _, ok := n.coords.Load(address); !ok {
-			var coords yggdrasil.Coords
-			var err error
-
-			// First look and see if the node is something that we already
-			// know about from our direct switch peers.
-			for _, peer := range n.core.GetSwitchPeers() {
-				if peer.PublicKey.String() == address {
-					coords = peer.Coords
-					n.log.Infof("%q is a direct peer, coords are %s", address, coords.String())
-					n.coords.Store(address, coords)
-					break
-				}
-			}
-
-			// If it isn' a node that we know directly then try to search
-			// the network.
-			if coords == nil {
-				n.log.Infof("Searching for coords for %q", address)
-				dest, derr := hex.DecodeString(address)
-				if derr != nil {
-					return nil, derr
-				}
-				if len(dest) != crypto.BoxPubKeyLen {
-					return nil, errors.New("invalid key length supplied")
-				}
-				var pubKey crypto.BoxPubKey
-				copy(pubKey[:], dest)
-				nodeID := crypto.GetNodeID(&pubKey)
-				nodeMask := &crypto.NodeID{}
-				for i := range nodeMask {
-					nodeMask[i] = 0xFF
-				}
-
-				fmt.Println("Resolving coords")
-				coords, err = n.core.Resolve(nodeID, nodeMask)
-				if err != nil {
-					return nil, fmt.Errorf("n.core.Resolve: %w", err)
-				}
-				fmt.Println("Found coords:", coords)
-				n.coords.Store(address, coords)
-			}
-
-			// We now know the coords in theory. Let's try dialling the
-			// node again.
-			if session, err = n.tryDial(address, coords); err != nil {
-				return nil, fmt.Errorf("n.tryDial: %w", err)
-			}
-		}
-	}
-
-	if session == nil {
-		return nil, fmt.Errorf("should have found session but didn't")
-	}
-
-	st, err := session.session.OpenStream()
-	if err != nil {
-		n.log.Println("session.OpenStream:", err)
-		_ = session.session.CloseWithError(0, "expected to be able to open session")
-		return nil, err
-	}
-	return QUICStream{st, session.session}, nil
-}
-
-func (n *Node) tryDial(address string, coords yggdrasil.Coords) (*session, error) {
-	quicSession, err := quic.Dial(
-		n.core,       // yggdrasil.PacketConn
-		coords,       // dial address
-		address,      // dial SNI
-		n.tlsConfig,  // TLS config
-		n.quicConfig, // QUIC config
-	)
-	if err != nil {
-		return nil, err
-	}
-	if len(quicSession.ConnectionState().PeerCertificates) != 1 {
-		_ = quicSession.CloseWithError(0, "expected a peer certificate")
-		return nil, errors.New("didn't receive a peer certificate")
-	}
-	if len(quicSession.ConnectionState().PeerCertificates[0].DNSNames) != 1 {
-		_ = quicSession.CloseWithError(0, "expected a DNS name")
-		return nil, errors.New("didn't receive a DNS name")
-	}
-	if gotAddress := quicSession.ConnectionState().PeerCertificates[0].DNSNames[0]; address != gotAddress {
-		_ = quicSession.CloseWithError(0, "you aren't the host I was hoping for")
-		return nil, fmt.Errorf("expected %q but dialled %q", address, gotAddress)
-	}
-	session := n.newSession(quicSession, address)
-	go session.listenFromQUIC()
-	go n.sessionFunc(address)
-	return session, nil
-}
-
-func (n *Node) generateTLSConfig() *tls.Config {
-	key, err := rsa.GenerateKey(rand.Reader, 1024)
-	if err != nil {
-		panic(err)
-	}
-	template := x509.Certificate{
-		Subject: pkix.Name{
-			CommonName: n.DerivedServerName(),
-		},
-		SerialNumber: big.NewInt(1),
-		NotAfter:     time.Now().Add(time.Hour * 24 * 365),
-		DNSNames:     []string{n.DerivedSessionName()},
-	}
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
-	if err != nil {
-		panic(err)
-	}
-	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
-
-	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
-	if err != nil {
-		panic(err)
-	}
-	return &tls.Config{
-		Certificates:       []tls.Certificate{tlsCert},
-		NextProtos:         []string{"quic-matrix-ygg"},
-		InsecureSkipVerify: true,
-		ClientAuth:         tls.RequireAnyClientCert,
-		GetClientCertificate: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-			return &tlsCert, nil
-		},
-	}
+	return n.utpSocket.DialContext(ctx, network, address)
 }

cmd/dendrite-demo-yggdrasil/yggconn/stream.go

@@ -1,20 +0,0 @@
-package yggconn
-
-import (
-	"net"
-
-	"github.com/lucas-clemente/quic-go"
-)
-
-type QUICStream struct {
-	quic.Stream
-	session quic.Session
-}
-
-func (s QUICStream) LocalAddr() net.Addr {
-	return s.session.LocalAddr()
-}
-
-func (s QUICStream) RemoteAddr() net.Addr {
-	return s.session.RemoteAddr()
-}

cmd/dendrite-monolith-server/main.go

@@ -30,7 +30,10 @@ import (
 	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/signingkeyserver"
 	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 var (
@@ -99,7 +102,7 @@ func main() {
 	}
 
 	fsAPI := federationsender.NewInternalAPI(
-		base, federation, rsAPI, keyRing,
+		base, federation, rsAPI, keyRing, false,
 	)
 	if base.UseHTTPAPIs {
 		federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI)
@@ -109,9 +112,19 @@ func main() {
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
 	rsImpl.SetFederationSenderAPI(fsAPI)
 
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
+	if traceInternal {
+		userAPI = &uapi.UserInternalAPITrace{
+			Impl: userAPI,
+		}
+	}
+	// needs to be after the SetUserAPI call above
+	if base.UseHTTPAPIs {
+		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
+		keyAPI = base.KeyServerHTTPClient()
+	}
 
 	eduInputAPI := eduserver.NewInternalAPI(
 		base, cache.New(), userAPI,
@@ -149,6 +162,7 @@ func main() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 
 	if len(base.Cfg.MSCs.MSCs) > 0 {

cmd/dendrite-polylith-multi/main.go

@@ -23,6 +23,8 @@ import (
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite)

cmd/dendrite-polylith-multi/personalities/clientapi.go

@@ -33,7 +33,7 @@ func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	keyAPI := base.KeyServerHTTPClient()
 
 	clientapi.AddPublicRoutes(
-		base.PublicClientAPIMux, &base.Cfg.ClientAPI, accountDB, federation,
+		base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI, accountDB, federation,
 		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, keyAPI, nil,
 		&cfg.MSCs,
 	)

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -33,7 +33,7 @@ func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 		base.PublicFederationAPIMux, base.PublicKeyAPIMux,
 		&base.Cfg.FederationAPI, userAPI, federation, keyRing,
 		rsAPI, fsAPI, base.EDUServerClient(), keyAPI,
-		&base.Cfg.MSCs,
+		&base.Cfg.MSCs, nil,
 	)
 
 	base.SetupAndServeHTTP(

cmd/dendrite-polylith-multi/personalities/federationsender.go

@@ -28,7 +28,7 @@ func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {
 
 	rsAPI := base.RoomserverHTTPClient()
 	fsAPI := federationsender.NewInternalAPI(
-		base, federation, rsAPI, keyRing,
+		base, federation, rsAPI, keyRing, false,
 	)
 	federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI)
 

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -22,7 +22,7 @@ import (
 
 func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	fsAPI := base.FederationSenderHTTPClient()
-	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI)
+	intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
 	intAPI.SetUserAPI(base.UserAPIClient())
 
 	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)

cmd/dendrite-upgrade-tests/main.go

@@ -0,0 +1,503 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Masterminds/semver/v3"
+	"github.com/codeclysm/extract"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/docker/client"
+	"github.com/docker/go-connections/nat"
+)
+
+var (
+	flagTempDir          = flag.String("tmp", "tmp", "Path to temporary directory to dump tarballs to")
+	flagFrom             = flag.String("from", "HEAD-1", "The version to start from e.g '0.3.1'. If 'HEAD-N' then starts N versions behind HEAD.")
+	flagTo               = flag.String("to", "HEAD", "The version to end on e.g '0.3.3'.")
+	flagBuildConcurrency = flag.Int("build-concurrency", runtime.NumCPU(), "The amount of build concurrency when building images")
+	flagHead             = flag.String("head", "", "Location to a dendrite repository to treat as HEAD instead of Github")
+	flagDockerHost       = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.")
+	alphaNumerics        = regexp.MustCompile("[^a-zA-Z0-9]+")
+)
+
+const HEAD = "HEAD"
+
+// Embed the Dockerfile to use when building dendrite versions.
+// We cannot use the dockerfile associated with the repo with each version sadly due to changes in
+// Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
+// due to the error:
+//   When using COPY with more than one source file, the destination must be a directory and end with a /
+// We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
+const Dockerfile = `FROM golang:1.13-stretch as build
+RUN apt-get update && apt-get install -y postgresql
+WORKDIR /build
+
+# Copy the build context to the repo as this is the right dendrite code. This is different to the
+# Complement Dockerfile which wgets a branch.
+COPY . .
+
+RUN go build ./cmd/dendrite-monolith-server
+RUN go build ./cmd/generate-keys
+RUN go build ./cmd/generate-config
+RUN ./generate-config --ci > dendrite.yaml
+RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
+
+# Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password
+RUN sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml 
+# No password when connecting over localhost
+RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/9.6/main/pg_hba.conf
+# Bump up max conns for moar concurrency
+RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf
+RUN sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml
+
+# This entry script starts postgres, waits for it to be up then starts dendrite
+RUN echo '\
+#!/bin/bash -eu \n\
+pg_lsclusters \n\
+pg_ctlcluster 9.6 main start \n\
+ \n\
+until pg_isready \n\
+do \n\
+  echo "Waiting for postgres"; \n\
+  sleep 1; \n\
+done \n\
+ \n\
+sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
+./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+' > run_dendrite.sh && chmod +x run_dendrite.sh
+
+ENV SERVER_NAME=localhost
+EXPOSE 8008 8448
+CMD /build/run_dendrite.sh `
+
+const dendriteUpgradeTestLabel = "dendrite_upgrade_test"
+
+// downloadArchive downloads an arbitrary github archive of the form:
+//   https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz
+// and re-tarballs it without the top-level directory which contains branch information. It inserts
+// the contents of `dockerfile` as a root file `Dockerfile` in the re-tarballed directory such that
+// you can directly feed the retarballed archive to `ImageBuild` to have it run said dockerfile.
+// Returns the tarball buffer on success.
+func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []byte) (*bytes.Buffer, error) {
+	resp, err := cli.Get(archiveURL)
+	if err != nil {
+		return nil, err
+	}
+	// nolint:errcheck
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("got HTTP %d", resp.StatusCode)
+	}
+	_ = os.RemoveAll(tmpDir)
+	if err = os.Mkdir(tmpDir, os.ModePerm); err != nil {
+		return nil, fmt.Errorf("failed to make temporary directory: %s", err)
+	}
+	// nolint:errcheck
+	defer os.RemoveAll(tmpDir)
+	// dump the tarball temporarily, stripping the top-level directory
+	err = extract.Archive(context.Background(), resp.Body, tmpDir, func(inPath string) string {
+		// remove top level
+		segments := strings.Split(inPath, "/")
+		return strings.Join(segments[1:], "/")
+	})
+	if err != nil {
+		return nil, err
+	}
+	// add top level Dockerfile
+	err = ioutil.WriteFile(path.Join(tmpDir, "Dockerfile"), dockerfile, os.ModePerm)
+	if err != nil {
+		return nil, fmt.Errorf("failed to inject /Dockerfile: %w", err)
+	}
+	// now re-tarball it :/
+	var tarball bytes.Buffer
+	err = compress(tmpDir, &tarball)
+	if err != nil {
+		return nil, err
+	}
+	return &tarball, nil
+}
+
+// buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
+func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) {
+	var tarball *bytes.Buffer
+	var err error
+	// If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI
+	// where we want to use the working directory.
+	if branchOrTagName == HEAD && *flagHead != "" {
+		log.Printf("%s: Using %s as HEAD", branchOrTagName, *flagHead)
+		// add top level Dockerfile
+		err = ioutil.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
+		if err != nil {
+			return "", fmt.Errorf("Custom HEAD: failed to inject /Dockerfile: %w", err)
+		}
+		// now tarball it
+		var buffer bytes.Buffer
+		err = compress(*flagHead, &buffer)
+		if err != nil {
+			return "", fmt.Errorf("failed to tarball custom HEAD %s : %s", *flagHead, err)
+		}
+		tarball = &buffer
+	} else {
+		log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir)
+		// pull an archive, this contains a top-level directory which screws with the build context
+		// which we need to fix up post download
+		u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName)
+		tarball, err = downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile))
+		if err != nil {
+			return "", fmt.Errorf("failed to download archive %s: %w", u, err)
+		}
+		log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len())
+	}
+
+	log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName)
+	res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
+		Tags: []string{"dendrite-upgrade"},
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to start building image: %s", err)
+	}
+	// nolint:errcheck
+	defer res.Body.Close()
+	decoder := json.NewDecoder(res.Body)
+	// {"aux":{"ID":"sha256:247082c717963bc2639fc2daed08838d67811ea12356cd4fda43e1ffef94f2eb"}}
+	var imageID string
+	for decoder.More() {
+		var dl struct {
+			Stream string                 `json:"stream"`
+			Aux    map[string]interface{} `json:"aux"`
+		}
+		if err := decoder.Decode(&dl); err != nil {
+			return "", fmt.Errorf("failed to decode build image output line: %w", err)
+		}
+		log.Printf("%s: %s", branchOrTagName, dl.Stream)
+		if dl.Aux != nil {
+			imgID, ok := dl.Aux["ID"]
+			if ok {
+				imageID = imgID.(string)
+			}
+		}
+	}
+	return imageID, nil
+}
+
+func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Version, err error) {
+	u := "https://api.github.com/repos/matrix-org/dendrite/tags"
+	res, err := httpClient.Get(u)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("%s returned HTTP %d", u, res.StatusCode)
+	}
+	resp := []struct {
+		Name string `json:"name"`
+	}{}
+	if err = json.NewDecoder(res.Body).Decode(&resp); err != nil {
+		return nil, err
+	}
+	for _, r := range resp {
+		v, err := semver.NewVersion(r.Name)
+		if err != nil {
+			continue // not a semver, that's ok and isn't an error, we allow tags that aren't semvers
+		}
+		semVers = append(semVers, v)
+	}
+	sort.Sort(semver.Collection(semVers))
+	return semVers, nil
+}
+
+func calculateVersions(cli *http.Client, from, to string) []string {
+	semvers, err := getAndSortVersionsFromGithub(cli)
+	if err != nil {
+		log.Fatalf("failed to collect semvers from github: %s", err)
+	}
+	// snip the lower bound depending on --from
+	if from != "" {
+		if strings.HasPrefix(from, "HEAD-") {
+			var headN int
+			headN, err = strconv.Atoi(strings.TrimPrefix(from, "HEAD-"))
+			if err != nil {
+				log.Fatalf("invalid --from, try 'HEAD-1'")
+			}
+			if headN >= len(semvers) {
+				log.Fatalf("only have %d versions, but asked to go to HEAD-%d", len(semvers), headN)
+			}
+			if headN > 0 {
+				semvers = semvers[len(semvers)-headN:]
+			}
+		} else {
+			fromVer, err := semver.NewVersion(from)
+			if err != nil {
+				log.Fatalf("invalid --from: %s", err)
+			}
+			i := 0
+			for i = 0; i < len(semvers); i++ {
+				if semvers[i].LessThan(fromVer) {
+					continue
+				}
+				break
+			}
+			semvers = semvers[i:]
+		}
+	}
+	if to != "" && to != HEAD {
+		toVer, err := semver.NewVersion(to)
+		if err != nil {
+			log.Fatalf("invalid --to: %s", err)
+		}
+		var i int
+		for i = len(semvers) - 1; i >= 0; i-- {
+			if semvers[i].GreaterThan(toVer) {
+				continue
+			}
+			break
+		}
+		semvers = semvers[:i+1]
+	}
+	var versions []string
+	for _, sv := range semvers {
+		versions = append(versions, sv.Original())
+	}
+	if to == HEAD {
+		versions = append(versions, HEAD)
+	}
+	return versions
+}
+
+func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, branchOrTagNames []string) map[string]string {
+	// concurrently build all versions, this can be done in any order. The mutex protects the map
+	branchToImageID := make(map[string]string)
+	var mu sync.Mutex
+
+	var wg sync.WaitGroup
+	wg.Add(concurrency)
+	ch := make(chan string, len(branchOrTagNames))
+	for _, branchName := range branchOrTagNames {
+		ch <- branchName
+	}
+	close(ch)
+
+	for i := 0; i < concurrency; i++ {
+		go func() {
+			defer wg.Done()
+			for branchName := range ch {
+				tmpDir := baseTempDir + alphaNumerics.ReplaceAllString(branchName, "")
+				imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName)
+				if err != nil {
+					log.Fatalf("%s: failed to build dendrite image: %s", branchName, err)
+				}
+				mu.Lock()
+				branchToImageID[branchName] = imgID
+				mu.Unlock()
+			}
+		}()
+	}
+	wg.Wait()
+	return branchToImageID
+}
+
+func runImage(dockerClient *client.Client, volumeName, version, imageID string) (csAPIURL, containerID string, err error) {
+	log.Printf("%s: running image %s\n", version, imageID)
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+	body, err := dockerClient.ContainerCreate(ctx, &container.Config{
+		Image: imageID,
+		Env:   []string{"SERVER_NAME=hs1"},
+		Labels: map[string]string{
+			dendriteUpgradeTestLabel: "yes",
+		},
+	}, &container.HostConfig{
+		PublishAllPorts: true,
+		Mounts: []mount.Mount{
+			{
+				Type:   mount.TypeVolume,
+				Source: volumeName,
+				Target: "/var/lib/postgresql/9.6/main",
+			},
+		},
+	}, nil, nil, "dendrite_upgrade_test_"+version)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to ContainerCreate: %s", err)
+	}
+	containerID = body.ID
+
+	err = dockerClient.ContainerStart(ctx, containerID, types.ContainerStartOptions{})
+	if err != nil {
+		return "", "", fmt.Errorf("failed to ContainerStart: %s", err)
+	}
+	inspect, err := dockerClient.ContainerInspect(ctx, containerID)
+	if err != nil {
+		return "", "", err
+	}
+	csapiPortInfo, ok := inspect.NetworkSettings.Ports[nat.Port("8008/tcp")]
+	if !ok {
+		return "", "", fmt.Errorf("port 8008 not exposed - exposed ports: %v", inspect.NetworkSettings.Ports)
+	}
+	baseURL := fmt.Sprintf("http://%s:%s", *flagDockerHost, csapiPortInfo[0].HostPort)
+	versionsURL := fmt.Sprintf("%s/_matrix/client/versions", baseURL)
+	// hit /versions to check it is up
+	var lastErr error
+	for i := 0; i < 500; i++ {
+		res, err := http.Get(versionsURL)
+		if err != nil {
+			lastErr = fmt.Errorf("GET %s => error: %s", versionsURL, err)
+			time.Sleep(50 * time.Millisecond)
+			continue
+		}
+		if res.StatusCode != 200 {
+			lastErr = fmt.Errorf("GET %s => HTTP %s", versionsURL, res.Status)
+			time.Sleep(50 * time.Millisecond)
+			continue
+		}
+		lastErr = nil
+		break
+	}
+	if lastErr != nil {
+		logs, err := dockerClient.ContainerLogs(context.Background(), containerID, types.ContainerLogsOptions{
+			ShowStdout: true,
+			ShowStderr: true,
+		})
+		// ignore errors when cannot get logs, it's just for debugging anyways
+		if err == nil {
+			logbody, err := ioutil.ReadAll(logs)
+			if err == nil {
+				log.Printf("Container logs:\n\n%s\n\n", string(logbody))
+			}
+		}
+	}
+	return baseURL, containerID, lastErr
+}
+
+func destroyContainer(dockerClient *client.Client, containerID string) {
+	err := dockerClient.ContainerRemove(context.TODO(), containerID, types.ContainerRemoveOptions{
+		Force: true,
+	})
+	if err != nil {
+		log.Printf("failed to remove container %s : %s", containerID, err)
+	}
+}
+
+func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchToImageID map[string]string) error {
+	csAPIURL, containerID, err := runImage(dockerClient, volumeName, v, branchToImageID[v])
+	if err != nil {
+		return fmt.Errorf("failed to run container for branch %v: %v", v, err)
+	}
+	defer destroyContainer(dockerClient, containerID)
+	log.Printf("URL %s -> %s \n", csAPIURL, containerID)
+	if err = runTests(csAPIURL, v); err != nil {
+		return fmt.Errorf("failed to run tests on version %s: %s", v, err)
+	}
+	return nil
+}
+
+func verifyTests(dockerClient *client.Client, volumeName string, versions []string, branchToImageID map[string]string) error {
+	lastVer := versions[len(versions)-1]
+	csAPIURL, containerID, err := runImage(dockerClient, volumeName, lastVer, branchToImageID[lastVer])
+	if err != nil {
+		return fmt.Errorf("failed to run container for branch %v: %v", lastVer, err)
+	}
+	defer destroyContainer(dockerClient, containerID)
+	return verifyTestsRan(csAPIURL, versions)
+}
+
+// cleanup old containers/volumes from a previous run
+func cleanup(dockerClient *client.Client) {
+	// ignore all errors, we are just cleaning up and don't want to fail just because we fail to cleanup
+	containers, _ := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{
+		Filters: label(dendriteUpgradeTestLabel),
+	})
+	for _, c := range containers {
+		s := time.Second
+		_ = dockerClient.ContainerStop(context.Background(), c.ID, &s)
+		_ = dockerClient.ContainerRemove(context.Background(), c.ID, types.ContainerRemoveOptions{
+			Force: true,
+		})
+	}
+	_ = dockerClient.VolumeRemove(context.Background(), "dendrite_upgrade_test", true)
+}
+
+func label(in string) filters.Args {
+	f := filters.NewArgs()
+	f.Add("label", in)
+	return f
+}
+
+func main() {
+	flag.Parse()
+	httpClient := &http.Client{
+		Timeout: 60 * time.Second,
+	}
+	dockerClient, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		log.Fatalf("failed to make docker client: %s", err)
+	}
+	if *flagFrom == "" {
+		flag.Usage()
+		os.Exit(1)
+	}
+	cleanup(dockerClient)
+	versions := calculateVersions(httpClient, *flagFrom, *flagTo)
+	log.Printf("Testing dendrite versions: %v\n", versions)
+
+	branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions)
+
+	// make a shared postgres volume
+	volume, err := dockerClient.VolumeCreate(context.Background(), volume.VolumeCreateBody{
+		Name: "dendrite_upgrade_test",
+		Labels: map[string]string{
+			dendriteUpgradeTestLabel: "yes",
+		},
+	})
+	if err != nil {
+		log.Fatalf("failed to make docker volume: %s", err)
+	}
+
+	failed := false
+	defer func() {
+		perr := recover()
+		log.Println("removing postgres volume")
+		verr := dockerClient.VolumeRemove(context.Background(), volume.Name, true)
+		if perr == nil {
+			perr = verr
+		}
+		if perr != nil {
+			panic(perr)
+		}
+		if failed {
+			os.Exit(1)
+		}
+	}()
+
+	// run through images sequentially
+	for _, v := range versions {
+		if err = loadAndRunTests(dockerClient, volume.Name, v, branchToImageID); err != nil {
+			log.Printf("failed to run tests for %v: %s\n", v, err)
+			failed = true
+			break
+		}
+	}
+	if err := verifyTests(dockerClient, volume.Name, versions, branchToImageID); err != nil {
+		log.Printf("failed to verify test results: %s", err)
+		failed = true
+	}
+}

cmd/dendrite-upgrade-tests/tar.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// From https://gist.github.com/mimoo/25fc9716e0f1353791f5908f94d6e726
+// Modified to strip off top-level when compressing
+func compress(src string, buf io.Writer) error {
+	// tar > gzip > buf
+	zr := gzip.NewWriter(buf)
+	tw := tar.NewWriter(zr)
+
+	// walk through every file in the folder
+	err := filepath.Walk(src, func(file string, fi os.FileInfo, e error) error {
+		// generate tar header
+		header, err := tar.FileInfoHeader(fi, file)
+		if err != nil {
+			return err
+		}
+
+		// must provide real name
+		// (see https://golang.org/src/archive/tar/common.go?#L626)
+		header.Name = strings.TrimPrefix(filepath.ToSlash(file), src+"/")
+		// write header
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+		// if not a dir, write file content
+		if !fi.IsDir() {
+			data, err := os.Open(file)
+			if err != nil {
+				return err
+			}
+			if _, err = io.Copy(tw, data); err != nil {
+				return err
+			}
+			if err = data.Close(); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// produce tar
+	if err := tw.Close(); err != nil {
+		return err
+	}
+	// produce gzip
+	if err := zr.Close(); err != nil {
+		return err
+	}
+	//
+	return nil
+}

cmd/dendrite-upgrade-tests/tests.go

@@ -0,0 +1,192 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const userPassword = "this_is_a_long_password"
+
+type user struct {
+	userID    string
+	localpart string
+	client    *gomatrix.Client
+}
+
+// runTests performs the following operations:
+//  - register alice and bob with branch name muxed into the localpart
+//  - create a DM room for the 2 users and exchange messages
+//  - create/join a public #global room and exchange messages
+func runTests(baseURL, branchName string) error {
+	// register 2 users
+	users := []user{
+		{
+			localpart: "alice" + branchName,
+		},
+		{
+			localpart: "bob" + branchName,
+		},
+	}
+	for i, u := range users {
+		client, err := gomatrix.NewClient(baseURL, "", "")
+		if err != nil {
+			return err
+		}
+		resp, err := client.RegisterDummy(&gomatrix.ReqRegister{
+			Username: strings.ToLower(u.localpart),
+			Password: userPassword,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to register %s: %s", u.localpart, err)
+		}
+		client, err = gomatrix.NewClient(baseURL, resp.UserID, resp.AccessToken)
+		if err != nil {
+			return err
+		}
+		users[i].client = client
+		users[i].userID = resp.UserID
+	}
+
+	// create DM room, join it and exchange messages
+	createRoomResp, err := users[0].client.CreateRoom(&gomatrix.ReqCreateRoom{
+		Preset:   "trusted_private_chat",
+		Invite:   []string{users[1].userID},
+		IsDirect: true,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create DM room: %s", err)
+	}
+	dmRoomID := createRoomResp.RoomID
+	if _, err = users[1].client.JoinRoom(dmRoomID, "", nil); err != nil {
+		return fmt.Errorf("failed to join DM room: %s", err)
+	}
+	msgs := []struct {
+		client *gomatrix.Client
+		text   string
+	}{
+		{
+			client: users[0].client, text: "1: " + branchName,
+		},
+		{
+			client: users[1].client, text: "2: " + branchName,
+		},
+		{
+			client: users[0].client, text: "3: " + branchName,
+		},
+		{
+			client: users[1].client, text: "4: " + branchName,
+		},
+	}
+	for _, msg := range msgs {
+		_, err = msg.client.SendText(dmRoomID, msg.text)
+		if err != nil {
+			return fmt.Errorf("failed to send text in dm room: %s", err)
+		}
+	}
+
+	// attempt to create/join the shared public room
+	publicRoomID := ""
+	createRoomResp, err = users[0].client.CreateRoom(&gomatrix.ReqCreateRoom{
+		RoomAliasName: "global",
+		Preset:        "public_chat",
+	})
+	if err != nil { // this is okay and expected if the room already exists and the aliases clash
+		// try to join it
+		_, domain, err2 := gomatrixserverlib.SplitID('@', users[0].userID)
+		if err2 != nil {
+			return fmt.Errorf("failed to split user ID: %s, %s", users[0].userID, err2)
+		}
+		joinRoomResp, err2 := users[0].client.JoinRoom(fmt.Sprintf("#global:%s", domain), "", nil)
+		if err2 != nil {
+			return fmt.Errorf("alice failed to join public room: %s", err2)
+		}
+		publicRoomID = joinRoomResp.RoomID
+	} else {
+		publicRoomID = createRoomResp.RoomID
+	}
+	if _, err = users[1].client.JoinRoom(publicRoomID, "", nil); err != nil {
+		return fmt.Errorf("bob failed to join public room: %s", err)
+	}
+	// send messages
+	for _, msg := range msgs {
+		_, err = msg.client.SendText(publicRoomID, "public "+msg.text)
+		if err != nil {
+			return fmt.Errorf("failed to send text in public room: %s", err)
+		}
+	}
+	log.Printf("OK! rooms(public=%s, dm=%s) users(%s, %s)\n", publicRoomID, dmRoomID, users[0].userID, users[1].userID)
+	return nil
+}
+
+// verifyTestsRan checks that the HS has the right rooms/messages
+func verifyTestsRan(baseURL string, branchNames []string) error {
+	log.Println("Verifying tests....")
+	// check we can login as all users
+	var resp *gomatrix.RespLogin
+	for _, branchName := range branchNames {
+		client, err := gomatrix.NewClient(baseURL, "", "")
+		if err != nil {
+			return err
+		}
+		userLocalparts := []string{
+			"alice" + branchName,
+			"bob" + branchName,
+		}
+		for _, userLocalpart := range userLocalparts {
+			resp, err = client.Login(&gomatrix.ReqLogin{
+				Type:     "m.login.password",
+				User:     strings.ToLower(userLocalpart),
+				Password: userPassword,
+			})
+			if err != nil {
+				return fmt.Errorf("failed to login as %s: %s", userLocalpart, err)
+			}
+			if resp.AccessToken == "" {
+				return fmt.Errorf("failed to login, bad response: %+v", resp)
+			}
+		}
+	}
+	log.Println("    accounts exist: OK")
+	client, err := gomatrix.NewClient(baseURL, resp.UserID, resp.AccessToken)
+	if err != nil {
+		return err
+	}
+	_, domain, err := gomatrixserverlib.SplitID('@', client.UserID)
+	if err != nil {
+		return err
+	}
+	u := client.BuildURL("directory", "room", fmt.Sprintf("#global:%s", domain))
+	r := struct {
+		RoomID string `json:"room_id"`
+	}{}
+	err = client.MakeRequest("GET", u, nil, &r)
+	if err != nil {
+		return fmt.Errorf("failed to /directory: %s", err)
+	}
+	if r.RoomID == "" {
+		return fmt.Errorf("/directory lookup returned no room ID")
+	}
+	log.Println("    public room exists: OK")
+
+	history, err := client.Messages(r.RoomID, client.Store.LoadNextBatch(client.UserID), "", 'b', 100)
+	if err != nil {
+		return fmt.Errorf("failed to get /messages: %s", err)
+	}
+	// we expect 4 messages per version
+	msgCount := 0
+	for _, ev := range history.Chunk {
+		if ev.Type == "m.room.message" {
+			msgCount += 1
+		}
+	}
+	wantMsgCount := len(branchNames) * 4
+	if msgCount != wantMsgCount {
+		return fmt.Errorf("got %d messages in global room, want %d", msgCount, wantMsgCount)
+	}
+	log.Println("    messages exist: OK")
+	return nil
+}

cmd/dendritejs-pinecone/jsServer.go

@@ -0,0 +1,100 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"syscall/js"
+)
+
+// JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it.
+type JSServer struct {
+	// The router which will service requests
+	Mux http.Handler
+}
+
+// OnRequestFromJS is the function that JS will invoke when there is a new request.
+// The JS function signature is:
+//   function(reqString: string): Promise<{result: string, error: string}>
+// Usage is like:
+//   const res = await global._go_js_server.fetch(reqString);
+//   if (res.error) {
+//     // handle error: this is a 'network' error, not a non-2xx error.
+//   }
+//   const rawHttpResponse = res.result;
+func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
+	// we HAVE to spawn a new goroutine and return immediately or else Go will deadlock
+	// if this request blocks at all e.g for /sync calls
+	httpStr := args[0].String()
+	promise := js.Global().Get("Promise").New(js.FuncOf(func(pthis js.Value, pargs []js.Value) interface{} {
+		// The initial callback code for new Promise() is also called on the critical path, which is why
+		// we need to put this in an immediately invoked goroutine.
+		go func() {
+			resolve := pargs[0]
+			resStr, err := h.handle(httpStr)
+			errStr := ""
+			if err != nil {
+				errStr = err.Error()
+			}
+			resolve.Invoke(map[string]interface{}{
+				"result": resStr,
+				"error":  errStr,
+			})
+		}()
+		return nil
+	}))
+	return promise
+}
+
+// handle invokes the http.ServeMux for this request and returns the raw HTTP response.
+func (h *JSServer) handle(httpStr string) (resStr string, err error) {
+	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(httpStr)))
+	if err != nil {
+		return
+	}
+	w := httptest.NewRecorder()
+
+	h.Mux.ServeHTTP(w, req)
+
+	res := w.Result()
+	var resBuffer strings.Builder
+	err = res.Write(&resBuffer)
+	return resBuffer.String(), err
+}
+
+// ListenAndServe registers a variable in JS-land with the given namespace. This variable is
+// a function which JS-land can call to 'send' HTTP requests. The function is attached to
+// a global object called "_go_js_server". See OnRequestFromJS for more info.
+func (h *JSServer) ListenAndServe(namespace string) {
+	globalName := "_go_js_server"
+	// register a hook in JS-land for it to invoke stuff
+	server := js.Global().Get(globalName)
+	if !server.Truthy() {
+		server = js.Global().Get("Object").New()
+		js.Global().Set(globalName, server)
+	}
+
+	server.Set(namespace, js.FuncOf(h.OnRequestFromJS))
+
+	fmt.Printf("Listening for requests from JS on function %s.%s\n", globalName, namespace)
+	// Block forever to mimic http.ListenAndServe
+	select {}
+}

cmd/dendritejs-pinecone/main.go

@@ -0,0 +1,261 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package main
+
+import (
+	"crypto/ed25519"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"os"
+	"syscall/js"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/userapi"
+
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/sirupsen/logrus"
+
+	_ "github.com/matrix-org/go-sqlite3-js"
+
+	pineconeRouter "github.com/matrix-org/pinecone/router"
+	pineconeSessions "github.com/matrix-org/pinecone/sessions"
+)
+
+var GitCommit string
+
+func init() {
+	fmt.Printf("[%s] dendrite.js starting...\n", GitCommit)
+}
+
+const publicPeer = "wss://pinecone.matrix.org/public"
+const keyNameEd25519 = "_go_ed25519_key"
+
+func readKeyFromLocalStorage() (key ed25519.PrivateKey, err error) {
+	localforage := js.Global().Get("localforage")
+	if !localforage.Truthy() {
+		err = fmt.Errorf("readKeyFromLocalStorage: no localforage")
+		return
+	}
+	// https://localforage.github.io/localForage/
+	item, ok := await(localforage.Call("getItem", keyNameEd25519))
+	if !ok || !item.Truthy() {
+		err = fmt.Errorf("readKeyFromLocalStorage: no key in localforage")
+		return
+	}
+	fmt.Println("Found key in localforage")
+	// extract []byte and make an ed25519 key
+	seed := make([]byte, 32, 32)
+	js.CopyBytesToGo(seed, item)
+
+	return ed25519.NewKeyFromSeed(seed), nil
+}
+
+func writeKeyToLocalStorage(key ed25519.PrivateKey) error {
+	localforage := js.Global().Get("localforage")
+	if !localforage.Truthy() {
+		return fmt.Errorf("writeKeyToLocalStorage: no localforage")
+	}
+
+	// make a Uint8Array from the key's seed
+	seed := key.Seed()
+	jsSeed := js.Global().Get("Uint8Array").New(len(seed))
+	js.CopyBytesToJS(jsSeed, seed)
+	// write it
+	localforage.Call("setItem", keyNameEd25519, jsSeed)
+	return nil
+}
+
+// taken from https://go-review.googlesource.com/c/go/+/150917
+
+// await waits until the promise v has been resolved or rejected and returns the promise's result value.
+// The boolean value ok is true if the promise has been resolved, false if it has been rejected.
+// If v is not a promise, v itself is returned as the value and ok is true.
+func await(v js.Value) (result js.Value, ok bool) {
+	if v.Type() != js.TypeObject || v.Get("then").Type() != js.TypeFunction {
+		return v, true
+	}
+	done := make(chan struct{})
+	onResolve := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
+		result = args[0]
+		ok = true
+		close(done)
+		return nil
+	})
+	defer onResolve.Release()
+	onReject := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
+		result = args[0]
+		ok = false
+		close(done)
+		return nil
+	})
+	defer onReject.Release()
+	v.Call("then", onResolve, onReject)
+	<-done
+	return
+}
+
+func generateKey() ed25519.PrivateKey {
+	// attempt to look for a seed in JS-land and if it exists use it.
+	priv, err := readKeyFromLocalStorage()
+	if err == nil {
+		fmt.Println("Read key from localStorage")
+		return priv
+	}
+	// generate a new key
+	fmt.Println(err, " : Generating new ed25519 key")
+	_, priv, err = ed25519.GenerateKey(nil)
+	if err != nil {
+		logrus.Fatalf("Failed to generate ed25519 key: %s", err)
+	}
+	if err := writeKeyToLocalStorage(priv); err != nil {
+		fmt.Println("failed to write key to localStorage: ", err)
+		// non-fatal, we'll just have amnesia for a while
+	}
+	return priv
+}
+
+func main() {
+	startup()
+
+	// We want to block forever to let the fetch and libp2p handler serve the APIs
+	select {}
+}
+
+func startup() {
+	sk := generateKey()
+	pk := sk.Public().(ed25519.PublicKey)
+
+	logger := log.New(os.Stdout, "", 0)
+	pRouter := pineconeRouter.NewRouter(logger, "dendrite", sk, pk, nil)
+	pSessions := pineconeSessions.NewSessions(logger, pRouter)
+
+	cfg := &config.Dendrite{}
+	cfg.Defaults()
+	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
+	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
+	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"
+	cfg.FederationSender.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
+	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
+	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
+	cfg.SigningKeyServer.Database.ConnectionString = "file:/idb/dendritejs_signingkeyserver.db"
+	cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db"
+	cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db"
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Database.ConnectionString = "file:/idb/dendritejs_naffka.db"
+	cfg.Global.TrustedIDServers = []string{}
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Global.PrivateKey = sk
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+
+	if err := cfg.Derive(); err != nil {
+		logrus.Fatalf("Failed to derive values from config: %s", err)
+	}
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	federation := conn.CreateFederationClient(base, pSessions)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
+	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
+	keyAPI.SetUserAPI(userAPI)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+
+	rsAPI := roomserver.NewInternalAPI(base, keyRing)
+	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI)
+	asQuery := appservice.NewInternalAPI(
+		base, userAPI, rsAPI,
+	)
+	rsAPI.SetAppserviceAPI(asQuery)
+	fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, keyRing, true)
+	rsAPI.SetFederationSenderAPI(fedSenderAPI)
+
+	monolith := setup.Monolith{
+		Config:    base.Cfg,
+		AccountDB: accountDB,
+		Client:    conn.CreateClient(base, pSessions),
+		FedClient: federation,
+		KeyRing:   keyRing,
+
+		AppserviceAPI:       asQuery,
+		EDUInternalAPI:      eduInputAPI,
+		FederationSenderAPI: fedSenderAPI,
+		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
+		KeyAPI:              keyAPI,
+		//ServerKeyAPI:        serverKeyAPI,
+		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation),
+	}
+	monolith.AddAllPublicRoutes(
+		base.ProcessContext,
+		base.PublicClientAPIMux,
+		base.PublicFederationAPIMux,
+		base.PublicKeyAPIMux,
+		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
+	)
+
+	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+
+	p2pRouter := pSessions.HTTP().Mux()
+	p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux)
+	p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux)
+
+	// Expose the matrix APIs via fetch - for local traffic
+	go func() {
+		logrus.Info("Listening for service-worker fetch traffic")
+		s := JSServer{
+			Mux: httpRouter,
+		}
+		s.ListenAndServe("fetch")
+	}()
+
+	// Connect to the static peer
+	go func() {
+		for {
+			if pRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 {
+				if err := conn.ConnectToPeer(pRouter, publicPeer); err != nil {
+					logrus.WithError(err).Error("Failed to connect to static peer")
+				}
+			}
+			select {
+			case <-base.ProcessContext.Context().Done():
+				return
+			case <-time.After(time.Second * 5):
+			}
+		}
+	}()
+}

cmd/dendritejs-pinecone/main_noop.go

@@ -0,0 +1,23 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !wasm
+
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Println("dendritejs: no-op when not compiling for WebAssembly")
+}

cmd/dendritejs-pinecone/main_test.go

@@ -0,0 +1,25 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package main
+
+import (
+	"testing"
+)
+
+func TestStartup(t *testing.T) {
+	startup()
+}

cmd/dendritejs/main.go

@@ -192,7 +192,7 @@ func main() {
 
 	accountDB := base.CreateAccountsDB()
 	federation := createFederationClient(cfg, node)
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
@@ -210,7 +210,7 @@ func main() {
 		base, userAPI, rsAPI,
 	)
 	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing)
+	fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing, true)
 	rsAPI.SetFederationSenderAPI(fedSenderAPI)
 	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation)
 
@@ -236,6 +236,7 @@ func main() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()

cmd/kafka-producer/main.go

@@ -1,93 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"os"
-	"strings"
-
-	sarama "github.com/Shopify/sarama"
-)
-
-const usage = `Usage: %s
-
-Reads a list of newline separated messages from stdin and writes them to a single partition in kafka.
-
-Arguments:
-
-`
-
-var (
-	brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster. You can also set the KAFKA_PEERS environment variable")
-	topic      = flag.String("topic", "", "REQUIRED: the topic to produce to")
-	partition  = flag.Int("partition", 0, "The partition to produce to. All the messages will be written to this partition.")
-)
-
-func main() {
-	flag.Usage = func() {
-		fmt.Fprintf(os.Stderr, usage, os.Args[0])
-		flag.PrintDefaults()
-	}
-
-	flag.Parse()
-
-	if *brokerList == "" {
-		fmt.Fprintln(os.Stderr, "no -brokers specified. Alternatively, set the KAFKA_PEERS environment variable")
-		os.Exit(1)
-	}
-
-	if *topic == "" {
-		fmt.Fprintln(os.Stderr, "no -topic specified")
-		os.Exit(1)
-	}
-
-	config := sarama.NewConfig()
-	config.Producer.RequiredAcks = sarama.WaitForAll
-	config.Producer.Return.Successes = true
-	config.Producer.Partitioner = sarama.NewManualPartitioner
-
-	producer, err := sarama.NewSyncProducer(strings.Split(*brokerList, ","), config)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "Failed to open Kafka producer:", err)
-		os.Exit(1)
-	}
-	defer func() {
-		if err := producer.Close(); err != nil {
-			fmt.Fprintln(os.Stderr, "Failed to close Kafka producer cleanly:", err)
-		}
-	}()
-
-	scanner := bufio.NewScanner(os.Stdin)
-	for scanner.Scan() {
-		line := scanner.Bytes()
-		message := &sarama.ProducerMessage{
-			Topic:     *topic,
-			Partition: int32(*partition),
-			Value:     sarama.ByteEncoder(line),
-		}
-		if _, _, err := producer.SendMessage(message); err != nil {
-			fmt.Fprintln(os.Stderr, "Failed to send message:", err)
-			os.Exit(1)
-		}
-
-	}
-	if err := scanner.Err(); err != nil {
-		fmt.Fprintln(os.Stderr, "reading standard input:", err)
-	}
-
-}

cmd/mediaapi-integration-tests/TESTS.md

@@ -1,69 +0,0 @@
-# Media API Tests
-
-## Implemented
-
-* functional
-    * upload
-        * normal case
-    * download
-        * local file
-            * existing
-            * non-existing
-        * remote file
-            * existing
-    * thumbnail
-        * original file formats
-            * JPEG
-        * local file
-            * existing
-        * remote file
-            * existing
-        * cache
-            * cold
-            * hot
-        * pre-generation according to configuration
-            * scale
-            * crop
-        * dynamic generation
-            * cold cache
-            * larger than original
-            * scale
-
-## TODO
-
-* functional
-    * upload
-        * file too large
-        * 0-byte file?
-        * invalid filename
-        * invalid content-type
-    * download
-        * invalid origin
-        * invalid media id
-    * thumbnail
-        * original file formats
-            * GIF
-            * PNG
-            * BMP
-            * SVG
-            * PDF
-            * TIFF
-            * WEBP
-        * local file
-            * non-existing
-        * remote file
-            * non-existing
-        * pre-generation according to configuration
-            * manual verification + hash check for regressions?
-        * dynamic generation
-            * hot cache
-            * limit on dimensions?
-            * 0x0
-            * crop
-* load
-    * 100 parallel requests
-        * same file
-        * different local files
-        * different remote files
-        * pre-generated thumbnails
-        * non-pre-generated thumbnails

cmd/mediaapi-integration-tests/main.go

@@ -1,269 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"time"
-
-	"github.com/matrix-org/dendrite/internal/test"
-	"github.com/matrix-org/gomatrixserverlib"
-	"gopkg.in/yaml.v2"
-)
-
-var (
-	// How long to wait for the server to write the expected output messages.
-	// This needs to be high enough to account for the time it takes to create
-	// the postgres database tables which can take a while on travis.
-	timeoutString = test.Defaulting(os.Getenv("TIMEOUT"), "10s")
-	// The name of maintenance database to connect to in order to create the test database.
-	postgresDatabase = test.Defaulting(os.Getenv("POSTGRES_DATABASE"), "postgres")
-	// The name of the test database to create.
-	testDatabaseName = test.Defaulting(os.Getenv("DATABASE_NAME"), "mediaapi_test")
-	// Postgres docker container name (for running psql). If not set, psql must be in PATH.
-	postgresContainerName = os.Getenv("POSTGRES_CONTAINER")
-	// Test image to be uploaded/downloaded
-	testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "cmd/mediaapi-integration-tests/totem.jpg")
-	kafkaURI = test.Defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
-)
-
-var thumbnailSizes = (`
-- width: 32
-  height: 32
-  method: crop
-- width: 96
-  height: 96
-  method: crop
-- width: 320
-  height: 240
-  method: scale
-- width: 640
-  height: 480
-  method: scale
-- width: 800
-  height: 600
-  method: scale
-`)
-
-const serverType = "media-api"
-
-const testMediaID = "1VuVy8u_hmDllD8BrcY0deM34Bl7SPJeY9J6BkMmpx0"
-const testContentType = "image/jpeg"
-const testOrigin = "localhost:18001"
-
-var testDatabaseTemplate = "dbname=%s sslmode=disable binary_parameters=yes"
-
-var timeout time.Duration
-
-var port = 10000
-
-func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error, *exec.Cmd, string, string) {
-	dir, err := ioutil.TempDir("", serverType+"-server-test"+suffix)
-	if err != nil {
-		panic(err)
-	}
-
-	proxyAddr := "localhost:1800" + suffix
-
-	database := fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix)
-	cfg, nextPort, err := test.MakeConfig(dir, kafkaURI, database, "localhost", port)
-	if err != nil {
-		panic(err)
-	}
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(proxyAddr)
-	cfg.MediaAPI.DynamicThumbnails = dynamicThumbnails
-	if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.MediaAPI.ThumbnailSizes); err != nil {
-		panic(err)
-	}
-
-	port = nextPort
-	if err = test.WriteConfig(cfg, dir); err != nil {
-		panic(err)
-	}
-
-	serverArgs := []string{
-		"--config", filepath.Join(dir, test.ConfigFile),
-	}
-
-	databases := []string{
-		testDatabaseName + suffix,
-	}
-
-	proxyCmd, _ := test.StartProxy(proxyAddr, cfg)
-
-	test.InitDatabase(
-		postgresDatabase,
-		postgresContainerName,
-		databases,
-	)
-
-	cmd, cmdChan := test.CreateBackgroundCommand(
-		filepath.Join(filepath.Dir(os.Args[0]), "dendrite-"+serverType+"-server"),
-		serverArgs,
-	)
-
-	fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.MediaAPI.InternalAPI.Listen, dir)
-	return cmd, cmdChan, proxyCmd, proxyAddr, dir
-}
-
-func cleanUpServer(cmd *exec.Cmd, dir string) {
-	// ensure server is dead, only cleaning up so don't care about errors this returns
-	cmd.Process.Kill() // nolint: errcheck
-	if err := os.RemoveAll(dir); err != nil {
-		fmt.Printf("WARNING: Failed to remove temporary directory %v: %q\n", dir, err)
-	}
-}
-
-// Runs a battery of media API server tests
-// The tests will pause at various points in this list to conduct tests on the HTTP responses before continuing.
-func main() {
-	fmt.Println("==TESTING==", os.Args[0])
-
-	var err error
-	timeout, err = time.ParseDuration(timeoutString)
-	if err != nil {
-		fmt.Printf("ERROR: Invalid timeout string %v: %q\n", timeoutString, err)
-		return
-	}
-
-	// create server1 with only pre-generated thumbnails allowed
-	server1Cmd, server1CmdChan, server1ProxyCmd, server1ProxyAddr, server1Dir := startMediaAPI("1", false)
-	defer cleanUpServer(server1Cmd, server1Dir)
-	defer server1ProxyCmd.Process.Kill() // nolint: errcheck
-	testDownload(server1ProxyAddr, server1ProxyAddr, "doesnotexist", 404, server1CmdChan)
-
-	// upload a JPEG file
-	testUpload(
-		server1ProxyAddr, testJPEG,
-	)
-
-	// download that JPEG file
-	testDownload(server1ProxyAddr, testOrigin, testMediaID, 200, server1CmdChan)
-
-	// thumbnail that JPEG file
-	testThumbnail(64, 64, "crop", server1ProxyAddr, server1CmdChan)
-
-	// create server2 with dynamic thumbnail generation
-	server2Cmd, server2CmdChan, server2ProxyCmd, server2ProxyAddr, server2Dir := startMediaAPI("2", true)
-	defer cleanUpServer(server2Cmd, server2Dir)
-	defer server2ProxyCmd.Process.Kill() // nolint: errcheck
-	testDownload(server2ProxyAddr, server2ProxyAddr, "doesnotexist", 404, server2CmdChan)
-
-	// pre-generated thumbnail that JPEG file via server2
-	testThumbnail(800, 600, "scale", server2ProxyAddr, server2CmdChan)
-
-	// download that JPEG file via server2
-	testDownload(server2ProxyAddr, testOrigin, testMediaID, 200, server2CmdChan)
-
-	// dynamic thumbnail that JPEG file via server2
-	testThumbnail(1920, 1080, "scale", server2ProxyAddr, server2CmdChan)
-
-	// thumbnail that JPEG file via server2
-	testThumbnail(10000, 10000, "scale", server2ProxyAddr, server2CmdChan)
-
-}
-
-func getMediaURI(host, endpoint, query string, components []string) string {
-	pathComponents := []string{host, "_matrix/media/v1", endpoint}
-	pathComponents = append(pathComponents, components...)
-	return "https://" + path.Join(pathComponents...) + query
-}
-
-func testUpload(host, filePath string) {
-	fmt.Printf("==TESTING== upload %v to %v\n", filePath, host)
-	file, err := os.Open(filePath)
-	defer file.Close() // nolint: errcheck, staticcheck, megacheck
-	if err != nil {
-		panic(err)
-	}
-	filename := filepath.Base(filePath)
-	stat, err := file.Stat()
-	if os.IsNotExist(err) {
-		panic(err)
-	}
-	fileSize := stat.Size()
-
-	req, err := http.NewRequest(
-		"POST",
-		getMediaURI(host, "upload", "?filename="+filename, nil),
-		file,
-	)
-	if err != nil {
-		panic(err)
-	}
-	req.ContentLength = fileSize
-	req.Header.Set("Content-Type", testContentType)
-
-	wantedBody := `{"content_uri": "mxc://localhost:18001/` + testMediaID + `"}`
-	testReq := &test.Request{
-		Req:              req,
-		WantedStatusCode: 200,
-		WantedBody:       test.CanonicalJSONInput([]string{wantedBody})[0],
-	}
-	if err := testReq.Do(); err != nil {
-		panic(err)
-	}
-	fmt.Printf("==TESTING== upload %v to %v PASSED\n", filePath, host)
-}
-
-func testDownload(host, origin, mediaID string, wantedStatusCode int, serverCmdChan chan error) {
-	req, err := http.NewRequest(
-		"GET",
-		getMediaURI(host, "download", "", []string{
-			origin,
-			mediaID,
-		}),
-		nil,
-	)
-	if err != nil {
-		panic(err)
-	}
-	testReq := &test.Request{
-		Req:              req,
-		WantedStatusCode: wantedStatusCode,
-		WantedBody:       "",
-	}
-	testReq.Run(fmt.Sprintf("download mxc://%v/%v from %v", origin, mediaID, host), timeout, serverCmdChan)
-}
-
-func testThumbnail(width, height int, resizeMethod, host string, serverCmdChan chan error) {
-	query := fmt.Sprintf("?width=%v&height=%v", width, height)
-	if resizeMethod != "" {
-		query += "&method=" + resizeMethod
-	}
-	req, err := http.NewRequest(
-		"GET",
-		getMediaURI(host, "thumbnail", query, []string{
-			testOrigin,
-			testMediaID,
-		}),
-		nil,
-	)
-	if err != nil {
-		panic(err)
-	}
-	testReq := &test.Request{
-		Req:              req,
-		WantedStatusCode: 200,
-		WantedBody:       "",
-	}
-	testReq.Run(fmt.Sprintf("thumbnail mxc://%v/%v%v from %v", testOrigin, testMediaID, query, host), timeout, serverCmdChan)
-}

cmd/roomserver-integration-tests/main.go

@@ -1,442 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"encoding/json"
-
-	"net/http"
-
-	"github.com/matrix-org/dendrite/internal/caching"
-	"github.com/matrix-org/dendrite/internal/test"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/inthttp"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-var (
-	// Path to where kafka is installed.
-	kafkaDir = defaulting(os.Getenv("KAFKA_DIR"), "kafka")
-	// The URI the kafka zookeeper is listening on.
-	zookeeperURI = defaulting(os.Getenv("ZOOKEEPER_URI"), "localhost:2181")
-	// The URI the kafka server is listening on.
-	kafkaURI = defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
-	// How long to wait for the roomserver to write the expected output messages.
-	// This needs to be high enough to account for the time it takes to create
-	// the postgres database tables which can take a while on travis.
-	timeoutString = defaulting(os.Getenv("TIMEOUT"), "60s")
-	// Timeout for http client
-	timeoutHTTPClient = defaulting(os.Getenv("TIMEOUT_HTTP"), "30s")
-	// The name of maintenance database to connect to in order to create the test database.
-	postgresDatabase = defaulting(os.Getenv("POSTGRES_DATABASE"), "postgres")
-	// The name of the test database to create.
-	testDatabaseName = defaulting(os.Getenv("DATABASE_NAME"), "roomserver_test")
-	// The postgres connection config for connecting to the test database.
-	testDatabase = defaulting(os.Getenv("DATABASE"), fmt.Sprintf("dbname=%s binary_parameters=yes", testDatabaseName))
-)
-
-var exe = test.KafkaExecutor{
-	ZookeeperURI:   zookeeperURI,
-	KafkaDirectory: kafkaDir,
-	KafkaURI:       kafkaURI,
-	// Send stdout and stderr to our stderr so that we see error messages from
-	// the kafka process.
-	OutputWriter: os.Stderr,
-}
-
-func defaulting(value, defaultValue string) string {
-	if value == "" {
-		value = defaultValue
-	}
-	return value
-}
-
-var (
-	timeout     time.Duration
-	timeoutHTTP time.Duration
-)
-
-func init() {
-	var err error
-	timeout, err = time.ParseDuration(timeoutString)
-	if err != nil {
-		panic(err)
-	}
-	timeoutHTTP, err = time.ParseDuration(timeoutHTTPClient)
-	if err != nil {
-		panic(err)
-	}
-}
-
-func createDatabase(database string) error {
-	cmd := exec.Command("psql", postgresDatabase)
-	cmd.Stdin = strings.NewReader(
-		fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;", database, database),
-	)
-	// Send stdout and stderr to our stderr so that we see error messages from
-	// the psql process
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
-}
-
-// runAndReadFromTopic runs a command and waits for a number of messages to be
-// written to a kafka topic. It returns if the command exits, the number of
-// messages is reached or after a timeout. It kills the command before it returns.
-// It returns a list of the messages read from the command on success or an error
-// on failure.
-func runAndReadFromTopic(runCmd *exec.Cmd, readyURL string, doInput func(), topic string, count int, checkQueryAPI func()) ([]string, error) {
-	type result struct {
-		// data holds all of stdout on success.
-		data []byte
-		// err is set on failure.
-		err error
-	}
-	done := make(chan result)
-	readCmd := exec.Command(
-		filepath.Join(kafkaDir, "bin", "kafka-console-consumer.sh"),
-		"--bootstrap-server", kafkaURI,
-		"--topic", topic,
-		"--from-beginning",
-		"--max-messages", fmt.Sprintf("%d", count),
-	)
-	// Send stderr to our stderr so the user can see any error messages.
-	readCmd.Stderr = os.Stderr
-
-	// Kill both processes before we exit.
-	defer func() { runCmd.Process.Kill() }()  // nolint: errcheck
-	defer func() { readCmd.Process.Kill() }() // nolint: errcheck
-
-	// Run the command, read the messages and wait for a timeout in parallel.
-	go func() {
-		// Read all of stdout.
-		defer func() {
-			if err := recover(); err != nil {
-				if errv, ok := err.(error); ok {
-					done <- result{nil, errv}
-				} else {
-					panic(err)
-				}
-			}
-		}()
-		data, err := readCmd.Output()
-		checkQueryAPI()
-		done <- result{data, err}
-	}()
-	go func() {
-		err := runCmd.Run()
-		done <- result{nil, err}
-	}()
-	go func() {
-		time.Sleep(timeout)
-		done <- result{nil, fmt.Errorf("Timeout reading %d messages from topic %q", count, topic)}
-	}()
-
-	// Poll the HTTP listener of the process waiting for it to be ready to receive requests.
-	ready := make(chan struct{})
-	go func() {
-		delay := 10 * time.Millisecond
-		for {
-			time.Sleep(delay)
-			if delay < 100*time.Millisecond {
-				delay *= 2
-			}
-			resp, err := http.Get(readyURL)
-			if err != nil {
-				continue
-			}
-			if resp.StatusCode == 200 {
-				break
-			}
-		}
-		ready <- struct{}{}
-	}()
-
-	// Wait for the roomserver to be ready to receive input or for it to crash.
-	select {
-	case <-ready:
-	case r := <-done:
-		return nil, r.err
-	}
-
-	// Write the input now that the server is running.
-	doInput()
-
-	// Wait for one of the tasks to finsh.
-	r := <-done
-
-	if r.err != nil {
-		return nil, r.err
-	}
-
-	// The kafka console consumer writes a newline character after each message.
-	// So we split on newline characters
-	lines := strings.Split(string(r.data), "\n")
-	if len(lines) > 0 {
-		// Remove the blank line at the end of the data.
-		lines = lines[:len(lines)-1]
-	}
-	return lines, nil
-}
-
-func writeToRoomServer(input []string, roomserverURL string) error {
-	var request api.InputRoomEventsRequest
-	var response api.InputRoomEventsResponse
-	var err error
-	request.InputRoomEvents = make([]api.InputRoomEvent, len(input))
-	for i := range input {
-		if err = json.Unmarshal([]byte(input[i]), &request.InputRoomEvents[i]); err != nil {
-			return err
-		}
-	}
-	x, err := inthttp.NewRoomserverClient(roomserverURL, &http.Client{Timeout: timeoutHTTP}, nil)
-	if err != nil {
-		return err
-	}
-	x.InputRoomEvents(context.Background(), &request, &response)
-	return response.Err()
-}
-
-// testRoomserver is used to run integration tests against a single roomserver.
-// It creates new kafka topics for the input and output of the roomserver.
-// It writes the input messages to the input kafka topic, formatting each message
-// as canonical JSON so that it fits on a single line.
-// It then runs the roomserver and waits for a number of messages to be written
-// to the output topic.
-// Once those messages have been written it runs the checkQueries function passing
-// a api.RoomserverQueryAPI client. The caller can use this function to check the
-// behaviour of the query API.
-func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverInternalAPI)) {
-	dir, err := ioutil.TempDir("", "room-server-test")
-	if err != nil {
-		panic(err)
-	}
-
-	cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000)
-	if err != nil {
-		panic(err)
-	}
-	if err = test.WriteConfig(cfg, dir); err != nil {
-		panic(err)
-	}
-
-	outputTopic := cfg.Global.Kafka.TopicFor(config.TopicOutputRoomEvent)
-
-	err = exe.DeleteTopic(outputTopic)
-	if err != nil {
-		panic(err)
-	}
-
-	if err = exe.CreateTopic(outputTopic); err != nil {
-		panic(err)
-	}
-
-	if err = createDatabase(testDatabaseName); err != nil {
-		panic(err)
-	}
-
-	cache, err := caching.NewInMemoryLRUCache(false)
-	if err != nil {
-		panic(err)
-	}
-
-	doInput := func() {
-		fmt.Printf("Roomserver is ready to receive input, sending %d events\n", len(input))
-		if err = writeToRoomServer(input, cfg.RoomServerURL()); err != nil {
-			panic(err)
-		}
-	}
-
-	cmd := exec.Command(filepath.Join(filepath.Dir(os.Args[0]), "dendrite-room-server"))
-
-	// Append the roomserver config to the existing environment.
-	// We append to the environment rather than replacing so that any additional
-	// postgres and golang environment variables such as PGHOST are passed to
-	// the roomserver process.
-	cmd.Stderr = os.Stderr
-	cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)}
-
-	gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() {
-		queryAPI, _ := inthttp.NewRoomserverClient("http://"+string(cfg.RoomServer.InternalAPI.Connect), &http.Client{Timeout: timeoutHTTP}, cache)
-		checkQueries(queryAPI)
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	if len(wantOutput) != len(gotOutput) {
-		panic(fmt.Errorf("Wanted %d lines of output got %d lines", len(wantOutput), len(gotOutput)))
-	}
-
-	for i := range wantOutput {
-		if !equalJSON(wantOutput[i], gotOutput[i]) {
-			panic(fmt.Errorf("Wanted %q at index %d got %q", wantOutput[i], i, gotOutput[i]))
-		}
-	}
-}
-
-func equalJSON(a, b string) bool {
-	canonicalA, err := gomatrixserverlib.CanonicalJSON([]byte(a))
-	if err != nil {
-		panic(err)
-	}
-	canonicalB, err := gomatrixserverlib.CanonicalJSON([]byte(b))
-	if err != nil {
-		panic(err)
-	}
-	return string(canonicalA) == string(canonicalB)
-}
-
-func main() {
-	fmt.Println("==TESTING==", os.Args[0])
-
-	input := []string{
-		`{
-			"auth_event_ids": [],
-			"kind": 1,
-			"event": {
-				"origin": "matrix.org",
-				"signatures": {
-					"matrix.org": {
-						"ed25519:auto": "3kXGwNtdj+zqEXlI8PWLiB76xtrQ7SxcvPuXAEVCTo+QPoBoUvLi1RkHs6O5mDz7UzIowK5bi1seAN4vOh0OBA"
-					}
-				},
-				"origin_server_ts": 1463671337837,
-				"sender": "@richvdh:matrix.org",
-				"event_id": "$1463671337126266wrSBX:matrix.org",
-				"prev_events": [],
-				"state_key": "",
-				"content": {"creator": "@richvdh:matrix.org"},
-				"depth": 1,
-				"prev_state": [],
-				"room_id": "!HCXfdvrfksxuYnIFiJ:matrix.org",
-				"auth_events": [],
-				"hashes": {"sha256": "Q05VLC8nztN2tguy+KnHxxhitI95wK9NelnsDaXRqeo"},
-				"type": "m.room.create"}
-		}`, `{
-			"auth_event_ids": ["$1463671337126266wrSBX:matrix.org"],
-			"kind": 2,
-			"state_event_ids": ["$1463671337126266wrSBX:matrix.org"],
-			"event": {
-				"origin": "matrix.org",
-				"signatures": {
-					"matrix.org": {
-						"ed25519:auto": "a2b3xXYVPPFeG1sHCU3hmZnAaKqZFgzGZozijRGblG5Y//ewRPAn1A2mCrI2UM5I+0zqr70cNpHgF8bmNFu4BA"
-					}
-				},
-				"origin_server_ts": 1463671339844,
-				"sender": "@richvdh:matrix.org",
-				"event_id": "$1463671339126270PnVwC:matrix.org",
-				"prev_events": [[
-					"$1463671337126266wrSBX:matrix.org", {"sha256": "h/VS07u8KlMwT3Ee8JhpkC7sa1WUs0Srgs+l3iBv6c0"}
-				]],
-				"membership": "join",
-				"state_key": "@richvdh:matrix.org",
-				"content": {
-					"membership": "join",
-					"avatar_url": "mxc://matrix.org/ZafPzsxMJtLaSaJXloBEKiws",
-					"displayname": "richvdh"
-				},
-				"depth": 2,
-				"prev_state": [],
-				"room_id": "!HCXfdvrfksxuYnIFiJ:matrix.org",
-				"auth_events": [[
-					"$1463671337126266wrSBX:matrix.org", {"sha256": "h/VS07u8KlMwT3Ee8JhpkC7sa1WUs0Srgs+l3iBv6c0"}
-				]],
-				"hashes": {"sha256": "t9t3sZV1Eu0P9Jyrs7pge6UTa1zuTbRdVxeUHnrQVH0"},
-				"type": "m.room.member"},
-			"has_state": true
-		}`,
-	}
-
-	want := []string{
-		`{"type":"new_room_event","new_room_event":{
-			"event":{
-				"auth_events":[[
-					"$1463671337126266wrSBX:matrix.org",{"sha256":"h/VS07u8KlMwT3Ee8JhpkC7sa1WUs0Srgs+l3iBv6c0"}
-				]],
-				"content":{
-					"avatar_url":"mxc://matrix.org/ZafPzsxMJtLaSaJXloBEKiws",
-					"displayname":"richvdh",
-					"membership":"join"
-				},
-				"depth": 2,
-				"event_id": "$1463671339126270PnVwC:matrix.org",
-				"hashes": {"sha256":"t9t3sZV1Eu0P9Jyrs7pge6UTa1zuTbRdVxeUHnrQVH0"},
-				"membership": "join",
-				"origin": "matrix.org",
-				"origin_server_ts": 1463671339844,
-				"prev_events": [[
-					"$1463671337126266wrSBX:matrix.org",{"sha256":"h/VS07u8KlMwT3Ee8JhpkC7sa1WUs0Srgs+l3iBv6c0"}
-				]],
-				"prev_state":[],
-				"room_id":"!HCXfdvrfksxuYnIFiJ:matrix.org",
-				"sender":"@richvdh:matrix.org",
-				"signatures":{
-					"matrix.org":{
-						"ed25519:auto":"a2b3xXYVPPFeG1sHCU3hmZnAaKqZFgzGZozijRGblG5Y//ewRPAn1A2mCrI2UM5I+0zqr70cNpHgF8bmNFu4BA"
-					}
-				},
-				"state_key":"@richvdh:matrix.org",
-				"type":"m.room.member"
-			},
-			"state_before_removes_event_ids":["$1463671339126270PnVwC:matrix.org"],
-			"state_before_adds_event_ids":null,
-			"latest_event_ids":["$1463671339126270PnVwC:matrix.org"],
-			"adds_state_event_ids":["$1463671337126266wrSBX:matrix.org", "$1463671339126270PnVwC:matrix.org"],
-			"removes_state_event_ids":null,
-			"last_sent_event_id":"",
-			"send_as_server":"",
-			"transaction_id": null
-		}}`,
-	}
-
-	testRoomserver(input, want, func(q api.RoomserverInternalAPI) {
-		var response api.QueryLatestEventsAndStateResponse
-		if err := q.QueryLatestEventsAndState(
-			context.Background(),
-			&api.QueryLatestEventsAndStateRequest{
-				RoomID: "!HCXfdvrfksxuYnIFiJ:matrix.org",
-				StateToFetch: []gomatrixserverlib.StateKeyTuple{
-					{EventType: "m.room.member", StateKey: "@richvdh:matrix.org"},
-				},
-			},
-			&response,
-		); err != nil {
-			panic(err)
-		}
-		if !response.RoomExists {
-			panic(fmt.Errorf(`Wanted room "!HCXfdvrfksxuYnIFiJ:matrix.org" to exist`))
-		}
-		if len(response.LatestEvents) != 1 || response.LatestEvents[0].EventID != "$1463671339126270PnVwC:matrix.org" {
-			panic(fmt.Errorf(`Wanted "$1463671339126270PnVwC:matrix.org" to be the latest event got %#v`, response.LatestEvents))
-		}
-		if len(response.StateEvents) != 1 || response.StateEvents[0].EventID() != "$1463671339126270PnVwC:matrix.org" {
-			panic(fmt.Errorf(`Wanted "$1463671339126270PnVwC:matrix.org" to be the state event got %#v`, response.StateEvents))
-		}
-	})
-
-	fmt.Println("==PASSED==", os.Args[0])
-}

cmd/syncserver-integration-tests/main.go

@@ -1,563 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"time"
-
-	"github.com/matrix-org/dendrite/internal/test"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-var (
-	// Path to where kafka is installed.
-	kafkaDir = test.Defaulting(os.Getenv("KAFKA_DIR"), "kafka")
-	// The URI the kafka zookeeper is listening on.
-	zookeeperURI = test.Defaulting(os.Getenv("ZOOKEEPER_URI"), "localhost:2181")
-	// The URI the kafka server is listening on.
-	kafkaURI = test.Defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
-	// The address the syncserver should listen on.
-	syncserverAddr = test.Defaulting(os.Getenv("SYNCSERVER_URI"), "localhost:9876")
-	// How long to wait for the syncserver to write the expected output messages.
-	// This needs to be high enough to account for the time it takes to create
-	// the postgres database tables which can take a while on travis.
-	timeoutString = test.Defaulting(os.Getenv("TIMEOUT"), "10s")
-	// The name of maintenance database to connect to in order to create the test database.
-	postgresDatabase = test.Defaulting(os.Getenv("POSTGRES_DATABASE"), "postgres")
-	// Postgres docker container name (for running psql). If not set, psql must be in PATH.
-	postgresContainerName = os.Getenv("POSTGRES_CONTAINER")
-	// The name of the test database to create.
-	testDatabaseName = test.Defaulting(os.Getenv("DATABASE_NAME"), "syncserver_test")
-	// The postgres connection config for connecting to the test database.
-	testDatabase = test.Defaulting(os.Getenv("DATABASE"), fmt.Sprintf("dbname=%s sslmode=disable binary_parameters=yes", testDatabaseName))
-)
-
-const inputTopic = "syncserverInput"
-const clientTopic = "clientapiserverOutput"
-
-var exe = test.KafkaExecutor{
-	ZookeeperURI:   zookeeperURI,
-	KafkaDirectory: kafkaDir,
-	KafkaURI:       kafkaURI,
-	// Send stdout and stderr to our stderr so that we see error messages from
-	// the kafka process.
-	OutputWriter: os.Stderr,
-}
-
-var timeout time.Duration
-var clientEventTestData []string
-
-func init() {
-	var err error
-	timeout, err = time.ParseDuration(timeoutString)
-	if err != nil {
-		panic(err)
-	}
-
-	for _, s := range outputRoomEventTestData {
-		clientEventTestData = append(clientEventTestData, clientEventJSONForOutputRoomEvent(s))
-	}
-}
-
-func createTestUser(database, username, token string) error {
-	cmd := exec.Command(
-		filepath.Join(filepath.Dir(os.Args[0]), "create-account"),
-		"--database", database,
-		"--username", username,
-		"--token", token,
-	)
-
-	// Send stdout and stderr to our stderr so that we see error messages from
-	// the create-account process
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
-}
-
-// clientEventJSONForOutputRoomEvent parses the given output room event and extracts the 'Event' JSON. It is
-// trimmed to the client format and then canonicalised and returned as a string.
-// Panics if there are any problems.
-func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string {
-	var out api.OutputEvent
-	if err := json.Unmarshal([]byte(outputRoomEvent), &out); err != nil {
-		panic("failed to unmarshal output room event: " + err.Error())
-	}
-	clientEvs := gomatrixserverlib.ToClientEvents([]*gomatrixserverlib.Event{
-		out.NewRoomEvent.Event.Event,
-	}, gomatrixserverlib.FormatSync)
-	b, err := json.Marshal(clientEvs[0])
-	if err != nil {
-		panic("failed to marshal client event as json: " + err.Error())
-	}
-	jsonBytes, err := gomatrixserverlib.CanonicalJSON(b)
-	if err != nil {
-		panic("failed to turn event json into canonical json: " + err.Error())
-	}
-	return string(jsonBytes)
-}
-
-// startSyncServer creates the database and config file needed for the sync server to run and
-// then starts the sync server. The Cmd being executed is returned. A channel is also returned,
-// which will have any termination errors sent down it, followed immediately by the channel being closed.
-func startSyncServer() (*exec.Cmd, chan error) {
-
-	dir, err := ioutil.TempDir("", "syncapi-server-test")
-	if err != nil {
-		panic(err)
-	}
-
-	cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000)
-	if err != nil {
-		panic(err)
-	}
-	// TODO use the address assigned by the config generator rather than clobbering.
-	cfg.Global.ServerName = "localhost"
-	cfg.SyncAPI.InternalAPI.Listen = config.HTTPAddress("http://" + syncserverAddr)
-	cfg.SyncAPI.InternalAPI.Connect = cfg.SyncAPI.InternalAPI.Listen
-
-	if err := test.WriteConfig(cfg, dir); err != nil {
-		panic(err)
-	}
-
-	serverArgs := []string{
-		"--config", filepath.Join(dir, test.ConfigFile),
-	}
-
-	databases := []string{
-		testDatabaseName,
-	}
-
-	test.InitDatabase(
-		postgresDatabase,
-		postgresContainerName,
-		databases,
-	)
-
-	if err := createTestUser(testDatabase, "alice", "@alice:localhost"); err != nil {
-		panic(err)
-	}
-	if err := createTestUser(testDatabase, "bob", "@bob:localhost"); err != nil {
-		panic(err)
-	}
-	if err := createTestUser(testDatabase, "charlie", "@charlie:localhost"); err != nil {
-		panic(err)
-	}
-
-	cmd, cmdChan := test.CreateBackgroundCommand(
-		filepath.Join(filepath.Dir(os.Args[0]), "dendrite-sync-api-server"),
-		serverArgs,
-	)
-
-	return cmd, cmdChan
-}
-
-// prepareKafka creates the topics which will be written to by the tests.
-func prepareKafka() {
-	err := exe.DeleteTopic(inputTopic)
-	if err != nil {
-		panic(err)
-	}
-
-	if err = exe.CreateTopic(inputTopic); err != nil {
-		panic(err)
-	}
-
-	err = exe.DeleteTopic(clientTopic)
-	if err != nil {
-		panic(err)
-	}
-
-	if err = exe.CreateTopic(clientTopic); err != nil {
-		panic(err)
-	}
-}
-
-func testSyncServer(syncServerCmdChan chan error, userID, since, want string) {
-	fmt.Printf("==TESTING== testSyncServer(%s,%s)\n", userID, since)
-	sinceQuery := ""
-	if since != "" {
-		sinceQuery = "&since=" + since
-	}
-	req, err := http.NewRequest(
-		"GET",
-		"http://"+syncserverAddr+"/api/_matrix/client/r0/sync?timeout=100&access_token="+userID+sinceQuery,
-		nil,
-	)
-	if err != nil {
-		panic(err)
-	}
-	testReq := &test.Request{
-		Req:              req,
-		WantedStatusCode: 200,
-		WantedBody:       test.CanonicalJSONInput([]string{want})[0],
-	}
-	testReq.Run("sync-api", timeout, syncServerCmdChan)
-}
-
-func writeToRoomServerLog(indexes ...int) {
-	var roomEvents []string
-	for _, i := range indexes {
-		roomEvents = append(roomEvents, outputRoomEventTestData[i])
-	}
-	if err := exe.WriteToTopic(inputTopic, test.CanonicalJSONInput(roomEvents)); err != nil {
-		panic(err)
-	}
-}
-
-// Runs a battery of sync server tests against test data in testdata.go
-// testdata.go has a list of OutputRoomEvents which will be fed into the kafka log which the sync server will consume.
-// The tests will pause at various points in this list to conduct tests on the /sync responses before continuing.
-// For ease of understanding, the curl commands used to create the OutputRoomEvents are listed along with each write to kafka.
-func main() {
-	fmt.Println("==TESTING==", os.Args[0])
-	prepareKafka()
-	cmd, syncServerCmdChan := startSyncServer()
-	// ensure server is dead, only cleaning up so don't care about errors this returns.
-	defer cmd.Process.Kill() // nolint: errcheck
-
-	// $ curl -XPOST -d '{}' "http://localhost:8009/_matrix/client/r0/createRoom?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello world"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/1?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello world 2"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/2?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello world 3"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"name":"Custom Room Name"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.name?access_token=@alice:localhost"
-	writeToRoomServerLog(
-		i0StateRoomCreate, i1StateAliceJoin, i2StatePowerLevels, i3StateJoinRules, i4StateHistoryVisibility,
-		i5AliceMsg, i6AliceMsg, i7AliceMsg, i8StateAliceRoomName,
-	)
-
-	// Make sure initial sync works TODO: prev_batch
-	testSyncServer(syncServerCmdChan, "@alice:localhost", "", `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "9",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {},
-			"join": {
-				"!PjrbIMW2cIiaYF4t:localhost": {
-					"account_data": {
-						"events": []
-					},
-					"ephemeral": {
-						"events": []
-					},
-					"state": {
-						"events": []
-					},
-					"timeline": {
-						"events": [`+
-		clientEventTestData[i0StateRoomCreate]+","+
-		clientEventTestData[i1StateAliceJoin]+","+
-		clientEventTestData[i2StatePowerLevels]+","+
-		clientEventTestData[i3StateJoinRules]+","+
-		clientEventTestData[i4StateHistoryVisibility]+","+
-		clientEventTestData[i5AliceMsg]+","+
-		clientEventTestData[i6AliceMsg]+","+
-		clientEventTestData[i7AliceMsg]+","+
-		clientEventTestData[i8StateAliceRoomName]+`],
-						"limited": true,
-						"prev_batch": ""
-					}
-				}
-			},
-			"leave": {}
-		}
-	}`)
-	// Make sure alice's rooms don't leak to bob
-	testSyncServer(syncServerCmdChan, "@bob:localhost", "", `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "9",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {},
-			"join": {},
-			"leave": {}
-		}
-	}`)
-	// Make sure polling with an up-to-date token returns nothing new
-	testSyncServer(syncServerCmdChan, "@alice:localhost", "9", `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "9",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {},
-			"join": {},
-			"leave": {}
-		}
-	}`)
-
-	// $ curl -XPUT -d '{"membership":"join"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@bob:localhost?access_token=@bob:localhost"
-	writeToRoomServerLog(i9StateBobJoin)
-
-	// Make sure alice sees it TODO: prev_batch
-	testSyncServer(syncServerCmdChan, "@alice:localhost", "9", `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "10",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {},
-			"join": {
-				"!PjrbIMW2cIiaYF4t:localhost": {
-					"account_data": {
-						"events": []
-					},
-					"ephemeral": {
-						"events": []
-					},
-					"state": {
-						"events": []
-					},
-					"timeline": {
-						"limited": false,
-						"prev_batch": "",
-						"events": [`+clientEventTestData[i9StateBobJoin]+`]
-					}
-				}
-			},
-			"leave": {}
-		}
-	}`)
-
-	// Make sure bob sees the room AND all the current room state TODO: history visibility
-	testSyncServer(syncServerCmdChan, "@bob:localhost", "9", `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "10",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {},
-			"join": {
-				"!PjrbIMW2cIiaYF4t:localhost": {
-					"account_data": {
-						"events": []
-					},
-					"ephemeral": {
-						"events": []
-					},
-					"state": {
-						"events": [`+
-		clientEventTestData[i0StateRoomCreate]+","+
-		clientEventTestData[i1StateAliceJoin]+","+
-		clientEventTestData[i2StatePowerLevels]+","+
-		clientEventTestData[i3StateJoinRules]+","+
-		clientEventTestData[i4StateHistoryVisibility]+","+
-		clientEventTestData[i8StateAliceRoomName]+`]
-					},
-					"timeline": {
-						"limited": false,
-						"prev_batch": "",
-						"events": [`+
-		clientEventTestData[i9StateBobJoin]+`]
-					}
-				}
-			},
-			"leave": {}
-		}
-	}`)
-
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello alice"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/1?access_token=@bob:localhost"
-	writeToRoomServerLog(i10BobMsg)
-
-	// Make sure alice can see everything around the join point for bob TODO: prev_batch
-	testSyncServer(syncServerCmdChan, "@alice:localhost", "7", `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "11",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {},
-			"join": {
-				"!PjrbIMW2cIiaYF4t:localhost": {
-					"account_data": {
-						"events": []
-					},
-					"ephemeral": {
-						"events": []
-					},
-					"state": {
-						"events": []
-					},
-					"timeline": {
-						"limited": false,
-						"prev_batch": "",
-						"events": [`+
-		clientEventTestData[i7AliceMsg]+","+
-		clientEventTestData[i8StateAliceRoomName]+","+
-		clientEventTestData[i9StateBobJoin]+","+
-		clientEventTestData[i10BobMsg]+`]
-					}
-				}
-			},
-			"leave": {}
-		}
-	}`)
-
-	// $ curl -XPUT -d '{"name":"A Different Custom Room Name"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.name?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello bob"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/2?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"membership":"invite"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@charlie:localhost?access_token=@bob:localhost"
-	writeToRoomServerLog(i11StateAliceRoomName, i12AliceMsg, i13StateBobInviteCharlie)
-
-	// Make sure charlie sees the invite both with and without a ?since= token
-	// TODO: Invite state should include the invite event and the room name.
-	charlieInviteData := `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "14",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {
-				"!PjrbIMW2cIiaYF4t:localhost": {
-					"invite_state": {
-						"events": []
-					}
-				}
-			},
-			"join": {},
-			"leave": {}
-		}
-	}`
-	testSyncServer(syncServerCmdChan, "@charlie:localhost", "7", charlieInviteData)
-	testSyncServer(syncServerCmdChan, "@charlie:localhost", "", charlieInviteData)
-
-	// $ curl -XPUT -d '{"membership":"join"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@charlie:localhost?access_token=@charlie:localhost"
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"not charlie..."}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"membership":"leave"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@charlie:localhost?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"why did you kick charlie"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@bob:localhost"
-	writeToRoomServerLog(i14StateCharlieJoin, i15AliceMsg, i16StateAliceKickCharlie, i17BobMsg)
-
-	// Check transitions to leave work
-	testSyncServer(syncServerCmdChan, "@charlie:localhost", "15", `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "18",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {},
-			"join": {},
-			"leave": {
-				"!PjrbIMW2cIiaYF4t:localhost": {
-					"state": {
-						"events": []
-					},
-					"timeline": {
-						"limited": false,
-						"prev_batch": "",
-						"events": [`+
-		clientEventTestData[i15AliceMsg]+","+
-		clientEventTestData[i16StateAliceKickCharlie]+`]
-					}
-				}
-			}
-		}
-	}`)
-
-	// Test joining and leaving the same room in a single /sync request puts the room in the 'leave' section.
-	// TODO: Use an earlier since value to assert that the /sync response doesn't leak messages
-	//       from before charlie was joined to the room. Currently it does leak because RecentEvents doesn't
-	//       take membership into account.
-	testSyncServer(syncServerCmdChan, "@charlie:localhost", "14", `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "18",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {},
-			"join": {},
-			"leave": {
-				"!PjrbIMW2cIiaYF4t:localhost": {
-					"state": {
-						"events": []
-					},
-					"timeline": {
-						"limited": false,
-						"prev_batch": "",
-						"events": [`+
-		clientEventTestData[i14StateCharlieJoin]+","+
-		clientEventTestData[i15AliceMsg]+","+
-		clientEventTestData[i16StateAliceKickCharlie]+`]
-					}
-				}
-			}
-		}
-	}`)
-
-	// $ curl -XPUT -d '{"name":"No Charlies"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.name?access_token=@alice:localhost"
-	writeToRoomServerLog(i18StateAliceRoomName)
-
-	// Check that users don't see state changes in rooms after they have left
-	testSyncServer(syncServerCmdChan, "@charlie:localhost", "17", `{
-		"account_data": {
-			"events": []
-		},
-		"next_batch": "19",
-		"presence": {
-			"events": []
-		},
-		"rooms": {
-			"invite": {},
-			"join": {},
-			"leave": {}
-		}
-	}`)
-
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"whatever"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@bob:localhost"
-	// $ curl -XPUT -d '{"membership":"leave"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@bob:localhost?access_token=@bob:localhost"
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"im alone now"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"membership":"invite"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@bob:localhost?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"membership":"leave"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@bob:localhost?access_token=@bob:localhost"
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"so alone"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"name":"Everyone welcome"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.name?access_token=@alice:localhost"
-	// $ curl -XPUT -d '{"membership":"join"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@charlie:localhost?access_token=@charlie:localhost"
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hiiiii"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@charlie:localhost"
-}

cmd/syncserver-integration-tests/testdata.go

@@ -1,102 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-// nolint: varcheck, deadcode, unused, megacheck
-const (
-	i0StateRoomCreate = iota
-	i1StateAliceJoin
-	i2StatePowerLevels
-	i3StateJoinRules
-	i4StateHistoryVisibility
-	i5AliceMsg
-	i6AliceMsg
-	i7AliceMsg
-	i8StateAliceRoomName
-	i9StateBobJoin
-	i10BobMsg
-	i11StateAliceRoomName
-	i12AliceMsg
-	i13StateBobInviteCharlie
-	i14StateCharlieJoin
-	i15AliceMsg
-	i16StateAliceKickCharlie
-	i17BobMsg
-	i18StateAliceRoomName
-	i19BobMsg
-	i20StateBobLeave
-	i21AliceMsg
-	i22StateAliceInviteBob
-	i23StateBobRejectInvite
-	i24AliceMsg
-	i25StateAliceRoomName
-	i26StateCharlieJoin
-	i27CharlieMsg
-)
-
-var outputRoomEventTestData = []string{
-	// $ curl -XPOST -d '{}' "http://localhost:8009/_matrix/client/r0/createRoom?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[],"content":{"creator":"@alice:localhost"},"depth":1,"event_id":"$xz0fUB8zNMTGFh1W:localhost","hashes":{"sha256":"KKkpxS8NoH0igBbL3J+nJ39MRlmA7QgW4BGL7Fv4ASI"},"origin":"localhost","origin_server_ts":1494411218382,"prev_events":[],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"uZG5Q/Hs2Z611gFlZPdwomomRJKf70xV2FQV+gLWM1XgzkLDRlRF3cBZc9y3CnHKnV/upTcXs7Op2/GmgD3UBw"}},"state_key":"","type":"m.room.create"},"latest_event_ids":["$xz0fUB8zNMTGFh1W:localhost"],"adds_state_event_ids":["$xz0fUB8zNMTGFh1W:localhost"],"last_sent_event_id":""}}`,
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}]],"content":{"membership":"join"},"depth":2,"event_id":"$QTen1vksfcRTpUCk:localhost","hashes":{"sha256":"tTukc9ab1fJfzgc5EMA/UD3swqfl/ic9Y9Zkt4fJo0Q"},"origin":"localhost","origin_server_ts":1494411218385,"prev_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"OPysDn/wT7yHeALXLTcEgR+iaKjv0p7VPuR/Mzvyg2IMAwPUjSOw8SQZlhSioWRtVPUp9VHbhIhJxQaPUg9yBQ"}},"state_key":"@alice:localhost","type":"m.room.member"},"latest_event_ids":["$QTen1vksfcRTpUCk:localhost"],"adds_state_event_ids":["$QTen1vksfcRTpUCk:localhost"],"last_sent_event_id":"$xz0fUB8zNMTGFh1W:localhost"}}`,
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}]],"content":{"ban":50,"events":{"m.room.avatar":50,"m.room.canonical_alias":50,"m.room.history_visibility":100,"m.room.name":50,"m.room.power_levels":100},"events_default":0,"invite":0,"kick":50,"redact":50,"state_default":50,"users":{"@alice:localhost":100},"users_default":0},"depth":3,"event_id":"$RWsxGlfPHAcijTgu:localhost","hashes":{"sha256":"ueZWiL/Q8bagRQGFktpnYJAJV6V6U3QKcUEmWYeyaaM"},"origin":"localhost","origin_server_ts":1494411218385,"prev_events":[["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"hZwWx3lyW61zMYmqLOxLTlfW2CnbjJQsZPLjZFa97TVG4ISz8CixMPsnVAIu5is29UCmiHyP8RvLecJjbLCtAQ"}},"state_key":"","type":"m.room.power_levels"},"latest_event_ids":["$RWsxGlfPHAcijTgu:localhost"],"adds_state_event_ids":["$RWsxGlfPHAcijTgu:localhost"],"last_sent_event_id":"$QTen1vksfcRTpUCk:localhost"}}`,
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}]],"content":{"join_rule":"public"},"depth":4,"event_id":"$2O2DpHB37CuwwJOe:localhost","hashes":{"sha256":"3P3HxAXI8gc094i020EoV/gissYiMVWv8+JAbrakM4E"},"origin":"localhost","origin_server_ts":1494411218386,"prev_events":[["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"L2yZoBbG/6TNsRHz+UtHY0SK4FgrdAYPR1l7RBWaNFbm+k/7kVhnoGlJ9yptpdLJjPMR2InqKXH8BBxRC83BCg"}},"state_key":"","type":"m.room.join_rules"},"latest_event_ids":["$2O2DpHB37CuwwJOe:localhost"],"adds_state_event_ids":["$2O2DpHB37CuwwJOe:localhost"],"last_sent_event_id":"$RWsxGlfPHAcijTgu:localhost"}}`,
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}]],"content":{"history_visibility":"joined"},"depth":5,"event_id":"$5LRiBskVCROnL5WY:localhost","hashes":{"sha256":"341alVufcKSVKLPr9WsJNTnW33QkBTn9eTfVWbyoa0o"},"origin":"localhost","origin_server_ts":1494411218387,"prev_events":[["$2O2DpHB37CuwwJOe:localhost",{"sha256":"ulaRD63dbCyolLTwvInIQpcrtU2c7ex/BHmhpLXAUoE"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"kRyt68cstwYgK8NtYzf0V5CnAbqUO47ixCCWYzRCi0WNstEwUw4XW1GHc8BllQsXwSj+nNv9g/66zZgG0DtxCA"}},"state_key":"","type":"m.room.history_visibility"},"latest_event_ids":["$5LRiBskVCROnL5WY:localhost"],"adds_state_event_ids":["$5LRiBskVCROnL5WY:localhost"],"last_sent_event_id":"$2O2DpHB37CuwwJOe:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello world"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/1?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"body":"hello world","msgtype":"m.text"},"depth":0,"event_id":"$Z8ZJik7ghwzSYTH9:localhost","hashes":{"sha256":"ahN1T5aiSZCzllf0pqNWJkF+x2h2S3kic+40pQ1X6BE"},"origin":"localhost","origin_server_ts":1494411339207,"prev_events":[["$5LRiBskVCROnL5WY:localhost",{"sha256":"3jULNC9b9Q0AhvnDQqpjhbtYwmkioHzPzdTJZvn8vOI"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"ylEpahRwEfGpqk+UCv0IF8YAxmut7w7udgHy3sVDfdJhs/4uJ6EkFEsKLknpXRc1vTIy1etKCBQ63QbCmRC2Bw"}},"type":"m.room.message"},"latest_event_ids":["$Z8ZJik7ghwzSYTH9:localhost"],"last_sent_event_id":"$5LRiBskVCROnL5WY:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello world 2"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/2?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"body":"hello world 2","msgtype":"m.text"},"depth":0,"event_id":"$8382Ah682eL4hxjN:localhost","hashes":{"sha256":"hQElDGSYc6KOdylrbMMm3+LlvUiCKo6S9G9n58/qtns"},"origin":"localhost","origin_server_ts":1494411380282,"prev_events":[["$Z8ZJik7ghwzSYTH9:localhost",{"sha256":"FBDwP+2FeqDENe7AEa3iAFAVKl1/IVq43mCH0uPRn90"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"LFXi6jTG7qn9xzi4rhIiHbkLD+4AZ9Yg7UTS2gqm1gt2lXQsgTYH1wE4Fol2fq4lvGlQVpxhtEr2huAYSbT7DA"}},"type":"m.room.message"},"latest_event_ids":["$8382Ah682eL4hxjN:localhost"],"last_sent_event_id":"$Z8ZJik7ghwzSYTH9:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello world 3"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"body":"hello world 3","msgtype":"m.text"},"depth":0,"event_id":"$17SfHsvSeTQthSWF:localhost","hashes":{"sha256":"eS6VFQI0l2U8rA8U17jgSHr9lQ73SNSnlnZu+HD0IjE"},"origin":"localhost","origin_server_ts":1494411396560,"prev_events":[["$8382Ah682eL4hxjN:localhost",{"sha256":"c6I/PUY7WnvxQ+oUEp/w2HEEuD3g8Vq7QwPUOSUjuc8"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"dvu9bSHZmX+yZoEqHioK7YDMtLH9kol0DdFqc5aHsbhZe/fKRZpfJMrlf1iXQdXSCMhikvnboPAXN3guiZCUBQ"}},"type":"m.room.message"},"latest_event_ids":["$17SfHsvSeTQthSWF:localhost"],"last_sent_event_id":"$8382Ah682eL4hxjN:localhost"}}`,
-	// $ curl -XPUT -d '{"name":"Custom Room Name"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.name?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"name":"Custom Room Name"},"depth":0,"event_id":"$j7KtuOzM0K15h3Kr:localhost","hashes":{"sha256":"QIKj5Klr50ugll4EjaNUATJmrru4CDp6TvGPv0v15bo"},"origin":"localhost","origin_server_ts":1494411482625,"prev_events":[["$17SfHsvSeTQthSWF:localhost",{"sha256":"iMTefewJ4W5sKQy7osQv4ilJAi7X0NsK791kqEUmYX0"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"WU7lwSWUAk7bsyDnBs128PyXxPZZoD1sN4AiDcvk+W1mDezJbFvWHDWymclxWESlP7TDrFTZEumRWGGCakjyAg"}},"state_key":"","type":"m.room.name"},"latest_event_ids":["$j7KtuOzM0K15h3Kr:localhost"],"adds_state_event_ids":["$j7KtuOzM0K15h3Kr:localhost"],"last_sent_event_id":"$17SfHsvSeTQthSWF:localhost"}}`,
-	// $ curl -XPUT -d '{"membership":"join"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@bob:localhost?access_token=@bob:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$2O2DpHB37CuwwJOe:localhost",{"sha256":"ulaRD63dbCyolLTwvInIQpcrtU2c7ex/BHmhpLXAUoE"}]],"content":{"membership":"join"},"depth":0,"event_id":"$wPepDhIla765Odre:localhost","hashes":{"sha256":"KeKqWLvM+LTvyFbwx6y3Y4W5Pj6nBSFUQ6jpkSf1oTE"},"origin":"localhost","origin_server_ts":1494411534290,"prev_events":[["$j7KtuOzM0K15h3Kr:localhost",{"sha256":"oDrWG5/sy1Ea3hYDOSJZRuGKCcjaHQlDYPDn2gB0/L0"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@bob:localhost","signatures":{"localhost":{"ed25519:something":"oVtvjZbWFe+iJhoDvLcQKnFpSYQ94dOodM4gGsx26P6fs2sFJissYwSIqpoxlElCJnmBAgy5iv4JK/5x21R2CQ"}},"state_key":"@bob:localhost","type":"m.room.member"},"latest_event_ids":["$wPepDhIla765Odre:localhost"],"adds_state_event_ids":["$wPepDhIla765Odre:localhost"],"last_sent_event_id":"$j7KtuOzM0K15h3Kr:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello alice"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/1?access_token=@bob:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$wPepDhIla765Odre:localhost",{"sha256":"GqUhRiAkRvPrNBDyUxj+emRfK2P8j6iWtvsXDOUltiI"}]],"content":{"body":"hello alice","msgtype":"m.text"},"depth":0,"event_id":"$RHNjeYUvXVZfb93t:localhost","hashes":{"sha256":"Ic1QLxTWFrWt1o31DS93ftrNHkunf4O6ubFvdD4ydNI"},"origin":"localhost","origin_server_ts":1494411593196,"prev_events":[["$wPepDhIla765Odre:localhost",{"sha256":"GqUhRiAkRvPrNBDyUxj+emRfK2P8j6iWtvsXDOUltiI"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@bob:localhost","signatures":{"localhost":{"ed25519:something":"8BHHkiThWwiIZbXCegRjIKNVGIa2kqrZW8VuL7nASfJBORhZ9R9p34UsmhsxVwTs/2/dX7M2ogMB28gIGdLQCg"}},"type":"m.room.message"},"latest_event_ids":["$RHNjeYUvXVZfb93t:localhost"],"last_sent_event_id":"$wPepDhIla765Odre:localhost"}}`,
-	// $ curl -XPUT -d '{"name":"A Different Custom Room Name"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.name?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"name":"A Different Custom Room Name"},"depth":0,"event_id":"$1xoUuqOFjFFJgwA5:localhost","hashes":{"sha256":"2pNnLhoHxNeSUpqxrd3c0kZUA4I+cdWZgYcJ8V3e2tk"},"origin":"localhost","origin_server_ts":1494411643348,"prev_events":[["$RHNjeYUvXVZfb93t:localhost",{"sha256":"LqFmTIzULgUDSf5xM3REObvnsRGLQliWBUf1hEDT4+w"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"gsY4B6TIBdVvLyFAaXw0xez9N5/Cn/ZaJ4z+j9gJU/ZR8j1t3OYlcVQN6uln9JwEU1k20AsGnIqvOaayd+bfCg"}},"state_key":"","type":"m.room.name"},"latest_event_ids":["$1xoUuqOFjFFJgwA5:localhost"],"adds_state_event_ids":["$1xoUuqOFjFFJgwA5:localhost"],"removes_state_event_ids":["$j7KtuOzM0K15h3Kr:localhost"],"last_sent_event_id":"$RHNjeYUvXVZfb93t:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hello bob"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/2?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"body":"hello bob","msgtype":"m.text"},"depth":0,"event_id":"$4NBTdIwDxq5fDGpv:localhost","hashes":{"sha256":"msCIESAya8kD7nLCopxkEqrgVuGfrlr9YBIADH5czTA"},"origin":"localhost","origin_server_ts":1494411674630,"prev_events":[["$1xoUuqOFjFFJgwA5:localhost",{"sha256":"ZXj+kY6sqQpf5vsNqvCMSvNoXXKDKxRE4R7+gZD9Tkk"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"bZRT3NxVlfBWw1PxSlKlgfnJixG+NI5H9QmUK2AjECg+l887BZJNCvAK0eD27N8e9V+c2glyXWYje2wexP2CBw"}},"type":"m.room.message"},"latest_event_ids":["$4NBTdIwDxq5fDGpv:localhost"],"last_sent_event_id":"$1xoUuqOFjFFJgwA5:localhost"}}`,
-	// $ curl -XPUT -d '{"membership":"invite"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@charlie:localhost?access_token=@bob:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$wPepDhIla765Odre:localhost",{"sha256":"GqUhRiAkRvPrNBDyUxj+emRfK2P8j6iWtvsXDOUltiI"}]],"content":{"membership":"invite"},"depth":0,"event_id":"$zzLHVlHIWPrnE7DI:localhost","hashes":{"sha256":"LKk7tnYJAHsyffbi9CzfdP+TU4KQ5g6YTgYGKjJ7NxU"},"origin":"localhost","origin_server_ts":1494411709192,"prev_events":[["$4NBTdIwDxq5fDGpv:localhost",{"sha256":"EpqmxEoJP93Zb2Nt2fS95SJWTqqIutHm/Ne8OHqp6Ps"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@bob:localhost","signatures":{"localhost":{"ed25519:something":"GdUzkC+7YKl1XDi7kYuD39yi2L/+nv+YrecIQHS+0BLDQqnEj+iRXfNBuZfTk6lUBCJCHXZlk7MnEIjvWDlZCg"}},"state_key":"@charlie:localhost","type":"m.room.member"},"latest_event_ids":["$zzLHVlHIWPrnE7DI:localhost"],"adds_state_event_ids":["$zzLHVlHIWPrnE7DI:localhost"],"last_sent_event_id":"$4NBTdIwDxq5fDGpv:localhost"}}`,
-	// $ curl -XPUT -d '{"membership":"join"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@charlie:localhost?access_token=@charlie:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$2O2DpHB37CuwwJOe:localhost",{"sha256":"ulaRD63dbCyolLTwvInIQpcrtU2c7ex/BHmhpLXAUoE"}],["$zzLHVlHIWPrnE7DI:localhost",{"sha256":"Jw28x9W+GoZYw7sEynsi1fcRzqRQiLddolOa/p26PV0"}]],"content":{"membership":"join"},"unsigned":{"prev_content":{"membership":"invite"},"prev_sender":"@bob:localhost","replaces_state":"$zzLHVlHIWPrnE7DI:localhost"},"depth":0,"event_id":"$uJVKyzZi8ZX0kOd9:localhost","hashes":{"sha256":"9ZZs/Cg0ewpBiCB6iFXXYlmW8koFiesCNGFrOLDTolE"},"origin":"localhost","origin_server_ts":1494411745015,"prev_events":[["$zzLHVlHIWPrnE7DI:localhost",{"sha256":"Jw28x9W+GoZYw7sEynsi1fcRzqRQiLddolOa/p26PV0"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@charlie:localhost","signatures":{"localhost":{"ed25519:something":"+TM0gFPM/M3Ji2BjYuTUTgDyCOWlOq8aTMCxLg7EBvS62yPxJ558f13OWWTczUO5aRAt+PvXsMVM/bp8u6c8DQ"}},"state_key":"@charlie:localhost","type":"m.room.member"},"latest_event_ids":["$uJVKyzZi8ZX0kOd9:localhost"],"adds_state_event_ids":["$uJVKyzZi8ZX0kOd9:localhost"],"removes_state_event_ids":["$zzLHVlHIWPrnE7DI:localhost"],"last_sent_event_id":"$zzLHVlHIWPrnE7DI:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"not charlie..."}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"body":"not charlie...","msgtype":"m.text"},"depth":0,"event_id":"$Ixfn5WT9ocWTYxfy:localhost","hashes":{"sha256":"hRChdyMQ3AY4jvrPpI8PEX6Taux83Qo5hdSeHlhPxGo"},"origin":"localhost","origin_server_ts":1494411792737,"prev_events":[["$uJVKyzZi8ZX0kOd9:localhost",{"sha256":"BtesLFnHZOREQCeilFM+xvDU/Wdj+nyHMw7IGTh/9gU"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"LC/Zqwu/XdqjmLdTOp/NQaFaE0niSAGgEpa39gCxsnsqEX80P7P5WDn/Kzx6rjWTnhIszrLsnoycqkXQT0Z4DQ"}},"type":"m.room.message"},"latest_event_ids":["$Ixfn5WT9ocWTYxfy:localhost"],"last_sent_event_id":"$uJVKyzZi8ZX0kOd9:localhost"}}`,
-	// $ curl -XPUT -d '{"membership":"leave"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@charlie:localhost?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$uJVKyzZi8ZX0kOd9:localhost",{"sha256":"BtesLFnHZOREQCeilFM+xvDU/Wdj+nyHMw7IGTh/9gU"}]],"content":{"membership":"leave"},"unsigned":{"prev_content":{"membership":"join"},"prev_sender":"@charlie:localhost","replaces_state":"$uJVKyzZi8ZX0kOd9:localhost"},"depth":0,"event_id":"$om1F4AI8tCYlHUSp:localhost","hashes":{"sha256":"7JVI0uCxSUyEqDJ+o36/zUIlIZkXVK/R6wkrZGvQXDE"},"origin":"localhost","origin_server_ts":1494411855278,"prev_events":[["$Ixfn5WT9ocWTYxfy:localhost",{"sha256":"hOoPIDQFvvNqQJzA5ggjoQi4v1BOELnhnmwU4UArDOY"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"3sxoDLUPnKuDJgFgS3C647BbiXrozxhhxrZOlFP3KgJKzBYv/ht+Jd2V2iSZOvsv94wgRBf0A/lEcJRIqeLgDA"}},"state_key":"@charlie:localhost","type":"m.room.member"},"latest_event_ids":["$om1F4AI8tCYlHUSp:localhost"],"adds_state_event_ids":["$om1F4AI8tCYlHUSp:localhost"],"removes_state_event_ids":["$uJVKyzZi8ZX0kOd9:localhost"],"last_sent_event_id":"$Ixfn5WT9ocWTYxfy:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"why did you kick charlie"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@bob:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$wPepDhIla765Odre:localhost",{"sha256":"GqUhRiAkRvPrNBDyUxj+emRfK2P8j6iWtvsXDOUltiI"}]],"content":{"body":"why did you kick charlie","msgtype":"m.text"},"depth":0,"event_id":"$hgao5gTmr3r9TtK2:localhost","hashes":{"sha256":"Aa2ZCrvwjX5xhvkVqIOFUeEGqrnrQZjjNFiZRybjsPY"},"origin":"localhost","origin_server_ts":1494411912809,"prev_events":[["$om1F4AI8tCYlHUSp:localhost",{"sha256":"yVs+CW7AiJrJOYouL8xPIBrtIHAhnbxaegna8MxeCto"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@bob:localhost","signatures":{"localhost":{"ed25519:something":"sGkpbEXGsvAuCvE3wb5E9H5fjCVKpRdWNt6csj1bCB9Fmg4Rg4mvj3TAJ+91DjO8IPsgSxDKdqqRYF0OtcynBA"}},"type":"m.room.message"},"latest_event_ids":["$hgao5gTmr3r9TtK2:localhost"],"last_sent_event_id":"$om1F4AI8tCYlHUSp:localhost"}}`,
-	// $ curl -XPUT -d '{"name":"No Charlies"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.name?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"name":"No Charlies"},"depth":0,"event_id":"$CY4XDoxjbns3a4Pc:localhost","hashes":{"sha256":"chk72pVkp3AGR2FtdC0mORBWS1b9ePnRN4WK3BP0BiI"},"origin":"localhost","origin_server_ts":1494411959114,"prev_events":[["$hgao5gTmr3r9TtK2:localhost",{"sha256":"/4/OG4Q2YalIeBtN76BEPIieBKA/3UFshR9T+WJip4o"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"mapvA3KJYgw5FmzJMhSFa/+JSuNyv2eKAkiGomAeBB7LQ1e9nK9XhW/Fp7a5Z2Sy2ENwHyd3ij7FEGiLOnSIAw"}},"state_key":"","type":"m.room.name"},"latest_event_ids":["$CY4XDoxjbns3a4Pc:localhost"],"adds_state_event_ids":["$CY4XDoxjbns3a4Pc:localhost"],"removes_state_event_ids":["$1xoUuqOFjFFJgwA5:localhost"],"last_sent_event_id":"$hgao5gTmr3r9TtK2:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"whatever"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@bob:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$wPepDhIla765Odre:localhost",{"sha256":"GqUhRiAkRvPrNBDyUxj+emRfK2P8j6iWtvsXDOUltiI"}]],"content":{"body":"whatever","msgtype":"m.text"},"depth":0,"event_id":"$pl8VBHRPYDmsnDh4:localhost","hashes":{"sha256":"FYqY9+/cepwIxxjfFV3AjOFBXkTlyEI2jep87dUc+SU"},"origin":"localhost","origin_server_ts":1494411988548,"prev_events":[["$CY4XDoxjbns3a4Pc:localhost",{"sha256":"hCoV63fp8eiquVdEefsOqJtLmJhw4wTlRv+wNTS20Ac"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@bob:localhost","signatures":{"localhost":{"ed25519:something":"sQKwRzE59eZyb8rDySo/pVwZXBh0nA5zx+kjEyXglxIQrTre+8Gj3R7Prni+RE3Dq7oWfKYV7QklTLURAaSICQ"}},"type":"m.room.message"},"latest_event_ids":["$pl8VBHRPYDmsnDh4:localhost"],"last_sent_event_id":"$CY4XDoxjbns3a4Pc:localhost"}}`,
-	// $ curl -XPUT -d '{"membership":"leave"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@bob:localhost?access_token=@bob:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$wPepDhIla765Odre:localhost",{"sha256":"GqUhRiAkRvPrNBDyUxj+emRfK2P8j6iWtvsXDOUltiI"}]],"content":{"membership":"leave"},"depth":0,"event_id":"$acCW4IgnBo8YD3jw:localhost","hashes":{"sha256":"porP+E2yftBGjfS381+WpZeDM9gZHsM3UydlBcRKBLw"},"origin":"localhost","origin_server_ts":1494412037042,"prev_events":[["$pl8VBHRPYDmsnDh4:localhost",{"sha256":"b+qQ380JDFq7quVU9EbIJ2sbpUKM1LAUNX0ZZUoVMZw"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@bob:localhost","signatures":{"localhost":{"ed25519:something":"kxbjTIC0/UR4cOYUAOTNiUc0SSVIF4BY6Rq6IEgYJemq4jcU2fYqum4mFxIQTDKKXMSRHEoNPDmYMFIJwkrsCg"}},"state_key":"@bob:localhost","type":"m.room.member"},"latest_event_ids":["$acCW4IgnBo8YD3jw:localhost"],"adds_state_event_ids":["$acCW4IgnBo8YD3jw:localhost"],"removes_state_event_ids":["$wPepDhIla765Odre:localhost"],"last_sent_event_id":"$pl8VBHRPYDmsnDh4:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"im alone now"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"body":"im alone now","msgtype":"m.text"},"depth":0,"event_id":"$nYdEXrvTDeb7DfkC:localhost","hashes":{"sha256":"qibC5NmlJpSRMBWSWxy1pv73FXymhPDXQFMmGosfsV0"},"origin":"localhost","origin_server_ts":1494412084668,"prev_events":[["$acCW4IgnBo8YD3jw:localhost",{"sha256":"8h3uXoE6pnI9iLnXI6493qJ0HeuRQfenRIu9PcgH72g"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"EHRoZznhXywhYeIn83o4FSFm3No/aOdLQPHQ68YGtNgESWwpuWLkkGVjoISjz3QgXQ06Fl3cHt7nlTaAHpCNAg"}},"type":"m.room.message"},"latest_event_ids":["$nYdEXrvTDeb7DfkC:localhost"],"last_sent_event_id":"$acCW4IgnBo8YD3jw:localhost"}}`,
-	// $ curl -XPUT -d '{"membership":"invite"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@bob:localhost?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$acCW4IgnBo8YD3jw:localhost",{"sha256":"8h3uXoE6pnI9iLnXI6493qJ0HeuRQfenRIu9PcgH72g"}]],"content":{"membership":"invite"},"depth":0,"event_id":"$gKNfcXLlWvs2cFad:localhost","hashes":{"sha256":"iYDOUjYkaGSFbVp7TRVFvGJyGMEuBHMQrJ9XqwhzmPI"},"origin":"localhost","origin_server_ts":1494412135845,"prev_events":[["$nYdEXrvTDeb7DfkC:localhost",{"sha256":"83T5Q3+nDvtS0oJTEhHxIw02twBDa1A7QR2bHtnxv1Y"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"ofw009aMJMqVjww9eDXgeTjOQqSlJl/GN/AAb+6mZAPcUI8aVgRlXOSESfhu1ONEuV/yNUycxNXWfMwuvoWsDg"}},"state_key":"@bob:localhost","type":"m.room.member"},"latest_event_ids":["$gKNfcXLlWvs2cFad:localhost"],"adds_state_event_ids":["$gKNfcXLlWvs2cFad:localhost"],"removes_state_event_ids":["$acCW4IgnBo8YD3jw:localhost"],"last_sent_event_id":"$nYdEXrvTDeb7DfkC:localhost"}}`,
-	// $ curl -XPUT -d '{"membership":"leave"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@bob:localhost?access_token=@bob:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$gKNfcXLlWvs2cFad:localhost",{"sha256":"/TYIY+L9qjg516Bzl8sadu+Np21KkxE4KdPXALeJ9eE"}]],"content":{"membership":"leave"},"depth":0,"event_id":"$B2q9Tepb6Xc1Rku0:localhost","hashes":{"sha256":"RbHTVdceAEfTALQDZdGrOmakKeTYnChaKjlVuoNUdSY"},"origin":"localhost","origin_server_ts":1494412187614,"prev_events":[["$gKNfcXLlWvs2cFad:localhost",{"sha256":"/TYIY+L9qjg516Bzl8sadu+Np21KkxE4KdPXALeJ9eE"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@bob:localhost","signatures":{"localhost":{"ed25519:something":"dNtUL86j2zUe5+DkfOkil5VujvFZg4FeTjbtcpeF+3E4SUChCAG3lyR6YOAIYBnjtD0/kqT7OcP3pM6vMEp1Aw"}},"state_key":"@bob:localhost","type":"m.room.member"},"latest_event_ids":["$B2q9Tepb6Xc1Rku0:localhost"],"adds_state_event_ids":["$B2q9Tepb6Xc1Rku0:localhost"],"removes_state_event_ids":["$gKNfcXLlWvs2cFad:localhost"],"last_sent_event_id":"$gKNfcXLlWvs2cFad:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"so alone"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"body":"so alone","msgtype":"m.text"},"depth":0,"event_id":"$W1nrYHQIbCTTSJOV:localhost","hashes":{"sha256":"uUKSa4U1coDoT3LUcNF25dt+UpUa2pLXzRJ3ljgxXZs"},"origin":"localhost","origin_server_ts":1494412229742,"prev_events":[["$B2q9Tepb6Xc1Rku0:localhost",{"sha256":"0CLru7nGPgyF9AWlZnarCElscSVrXl2MMY2atrz80Uc"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"YlBJyDnE34UhaCB9hirQN5OySfTDoqiBDnNvxomXjU94z4a8g2CLWKjApwd/q/j4HamCUtjgkjJ2um6hNjsVBA"}},"type":"m.room.message"},"latest_event_ids":["$W1nrYHQIbCTTSJOV:localhost"],"last_sent_event_id":"$B2q9Tepb6Xc1Rku0:localhost"}}`,
-	// $ curl -XPUT -d '{"name":"Everyone welcome"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.name?access_token=@alice:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$QTen1vksfcRTpUCk:localhost",{"sha256":"znwhbYzdueh0grYkUX4jgXmP9AjKphzyesMZWMiF4IY"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}]],"content":{"name":"Everyone welcome"},"depth":0,"event_id":"$nLzxoBC4A0QRvJ1k:localhost","hashes":{"sha256":"PExCybjaMW1TfgFr57MdIRYJ642FY2jnrdW/tpPOf1Y"},"origin":"localhost","origin_server_ts":1494412294551,"prev_events":[["$W1nrYHQIbCTTSJOV:localhost",{"sha256":"HXk/ACcsiaZ/z1f2aZSIhJF8Ih3BWeh1vp+cV/fwoE0"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@alice:localhost","signatures":{"localhost":{"ed25519:something":"RK09L8sQv78y69PNbOLaX8asq5kp51mbqUuct5gd7ZNmaHKnVds6ew06QEn+gHSDAxqQo2tpcfoajp+yMj1HBw"}},"state_key":"","type":"m.room.name"},"latest_event_ids":["$nLzxoBC4A0QRvJ1k:localhost"],"adds_state_event_ids":["$nLzxoBC4A0QRvJ1k:localhost"],"removes_state_event_ids":["$CY4XDoxjbns3a4Pc:localhost"],"last_sent_event_id":"$W1nrYHQIbCTTSJOV:localhost"}}`,
-	// $ curl -XPUT -d '{"membership":"join"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/state/m.room.member/@charlie:localhost?access_token=@charlie:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$2O2DpHB37CuwwJOe:localhost",{"sha256":"ulaRD63dbCyolLTwvInIQpcrtU2c7ex/BHmhpLXAUoE"}],["$om1F4AI8tCYlHUSp:localhost",{"sha256":"yVs+CW7AiJrJOYouL8xPIBrtIHAhnbxaegna8MxeCto"}]],"content":{"membership":"join"},"depth":0,"event_id":"$Zo6P8r9bczF6kctV:localhost","hashes":{"sha256":"R3J2iUWnGxVdmly8ah+Dgb5VbJ2i/e8BLaWM0z9eZKU"},"origin":"localhost","origin_server_ts":1494412338689,"prev_events":[["$nLzxoBC4A0QRvJ1k:localhost",{"sha256":"TDcFaArAXpxIJ1noSubcFqkLXiQTrc1Dw1+kgCtx3XY"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@charlie:localhost","signatures":{"localhost":{"ed25519:something":"tVnjLVoJ9SLlMQIJSK/6zANWaEu8tVVkx3AEJiC3y5JmhPORb3PyG8eE+e/9hC4aJSQL8LGLaJNWXukMpb2SBg"}},"state_key":"@charlie:localhost","type":"m.room.member"},"latest_event_ids":["$Zo6P8r9bczF6kctV:localhost"],"adds_state_event_ids":["$Zo6P8r9bczF6kctV:localhost"],"removes_state_event_ids":["$om1F4AI8tCYlHUSp:localhost"],"last_sent_event_id":"$nLzxoBC4A0QRvJ1k:localhost"}}`,
-	// $ curl -XPUT -d '{"msgtype":"m.text","body":"hiiiii"}' "http://localhost:8009/_matrix/client/r0/rooms/%21PjrbIMW2cIiaYF4t:localhost/send/m.room.message/3?access_token=@charlie:localhost"
-	`{"type":"new_room_event","new_room_event":{"event":{"auth_events":[["$xz0fUB8zNMTGFh1W:localhost",{"sha256":"F4tTLtltC6f2XKeXq4ZKpMZ5EpditaW+RYQSnYzq3lI"}],["$RWsxGlfPHAcijTgu:localhost",{"sha256":"1zc+86U9vLK1BvTJbeLuYpw9dZqvX2fr8rc3pOF69f8"}],["$Zo6P8r9bczF6kctV:localhost",{"sha256":"mnjt3WTYqwtuyl2Fca+0cgm6moHaNL+W9BqRJTQzdEY"}]],"content":{"body":"hiiiii","msgtype":"m.text"},"depth":0,"event_id":"$YAEvK8u2zkTsjf5P:localhost","hashes":{"sha256":"6hKy61h1tuHjYdfpq2MnaPtGEBAZOUz8FLTtxLwjK5A"},"origin":"localhost","origin_server_ts":1494412375465,"prev_events":[["$Zo6P8r9bczF6kctV:localhost",{"sha256":"mnjt3WTYqwtuyl2Fca+0cgm6moHaNL+W9BqRJTQzdEY"}]],"room_id":"!PjrbIMW2cIiaYF4t:localhost","sender":"@charlie:localhost","signatures":{"localhost":{"ed25519:something":"BsSLaMM5U/YkyvBZ00J/+si9My+wAJZOcBhBeato0oHayiag7FW77ZpSTfADazPdNH62kjB0sdP9CN6vQA7yDg"}},"type":"m.room.message"},"latest_event_ids":["$YAEvK8u2zkTsjf5P:localhost"],"last_sent_event_id":"$Zo6P8r9bczF6kctV:localhost"}}`,
-}

dendrite-config.yaml

@@ -112,7 +112,7 @@ global:
     # Maximum number of entries to hold in the DNS cache, and
     # for how long those items should be considered valid in seconds.
     cache_size: 256
-    cache_lifetime: 300
+    cache_lifetime: "5m" # 5minutes; see https://pkg.go.dev/time@master#ParseDuration for more
 
 # Configuration for the Appservice API.
 app_service_api:

docs/INSTALL.md

@@ -25,7 +25,7 @@ use in production environments just yet!
 
 Dendrite requires:
 
-* Go 1.13 or higher
+* Go 1.15 or higher
 * Postgres 9.6 or higher (if using Postgres databases, not needed for SQLite)
 
 If you want to run a polylith deployment, you also need:

docs/nginx/monolith-sample.conf

@@ -1,6 +1,6 @@
 server {
     listen 443 ssl; # IPv4
-    listen [::]:443; # IPv6
+    listen [::]:443 ssl; # IPv6
     server_name my.hostname.com;
 
     ssl_certificate /path/to/fullchain.pem;
@@ -16,6 +16,9 @@ server {
     }
 
     location /.well-known/matrix/client {
+        # If your sever_name here doesn't match your matrix homeserver URL
+        # (e.g. hostname.com as server_name and matrix.hostname.com as homeserver URL)
+        # add_header Access-Control-Allow-Origin '*';
         return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }';
     }
 

docs/nginx/polylith-sample.conf

@@ -1,6 +1,6 @@
 server {
     listen 443 ssl; # IPv4
-    listen [::]:443; # IPv6
+    listen [::]:443 ssl; # IPv6
     server_name my.hostname.com;
 
     ssl_certificate /path/to/fullchain.pem;
@@ -16,6 +16,9 @@ server {
     }
 
     location /.well-known/matrix/client {
+        # If your sever_name here doesn't match your matrix homeserver URL
+        # (e.g. hostname.com as server_name and matrix.hostname.com as homeserver URL)
+        # add_header Access-Control-Allow-Origin '*';
         return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }';
     }
 

docs/p2p.md

@@ -2,14 +2,23 @@
 
 These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up.
 
-
 ### Dendrite
 
+#### Build
+
 - The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`.
 - Build it and copy assets to riot-web.
 ```
-$ GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs
-$ cp main.wasm ../riot-web/src/vector/dendrite.wasm
+$ ./build-dendritejs.sh
+$ cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm
+```
+
+#### Test
+
+To check that the Dendrite side is working well as Wasm, you can run the
+Wasm-specific tests:
+```
+$ ./test-dendritejs.sh
 ```
 
 ### Rendezvous

eduserver/api/input.go

@@ -75,6 +75,12 @@ type InputReceiptEventRequest struct {
 // InputReceiptEventResponse is a response to InputReceiptEventRequest
 type InputReceiptEventResponse struct{}
 
+type InputCrossSigningKeyUpdateRequest struct {
+	CrossSigningKeyUpdate `json:"signing_keys"`
+}
+
+type InputCrossSigningKeyUpdateResponse struct{}
+
 // EDUServerInputAPI is used to write events to the typing server.
 type EDUServerInputAPI interface {
 	InputTypingEvent(
@@ -94,4 +100,10 @@ type EDUServerInputAPI interface {
 		request *InputReceiptEventRequest,
 		response *InputReceiptEventResponse,
 	) error
+
+	InputCrossSigningKeyUpdate(
+		ctx context.Context,
+		request *InputCrossSigningKeyUpdateRequest,
+		response *InputCrossSigningKeyUpdateResponse,
+	) error
 }

eduserver/api/output.go

@@ -33,14 +33,6 @@ type OutputTypingEvent struct {
 	ExpireTime *time.Time
 }
 
-// TypingEvent represents a matrix edu event of type 'm.typing'.
-type TypingEvent struct {
-	Type   string `json:"type"`
-	RoomID string `json:"room_id"`
-	UserID string `json:"user_id"`
-	Typing bool   `json:"typing"`
-}
-
 // OutputSendToDeviceEvent is an entry in the send-to-device output kafka log.
 // This contains the full event content, along with the user ID and device ID
 // to which it is destined.
@@ -50,14 +42,6 @@ type OutputSendToDeviceEvent struct {
 	gomatrixserverlib.SendToDeviceEvent
 }
 
-type ReceiptEvent struct {
-	UserID    string                      `json:"user_id"`
-	RoomID    string                      `json:"room_id"`
-	EventID   string                      `json:"event_id"`
-	Type      string                      `json:"type"`
-	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
-}
-
 // OutputReceiptEvent is an entry in the receipt output kafka log
 type OutputReceiptEvent struct {
 	UserID    string                      `json:"user_id"`
@@ -67,21 +51,7 @@ type OutputReceiptEvent struct {
 	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
 }
 
-// Helper structs for receipts json creation
-type ReceiptMRead struct {
-	User map[string]ReceiptTS `json:"m.read"`
-}
-
-type ReceiptTS struct {
-	TS gomatrixserverlib.Timestamp `json:"ts"`
-}
-
-// FederationSender output
-type FederationReceiptMRead struct {
-	User map[string]FederationReceiptData `json:"m.read"`
-}
-
-type FederationReceiptData struct {
-	Data     ReceiptTS `json:"data"`
-	EventIDs []string  `json:"event_ids"`
+// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log
+type OutputCrossSigningKeyUpdate struct {
+	CrossSigningKeyUpdate `json:"signing_keys"`
 }

eduserver/api/types.go

@@ -0,0 +1,59 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import "github.com/matrix-org/gomatrixserverlib"
+
+const (
+	MSigningKeyUpdate = "m.signing_key_update"
+)
+
+type TypingEvent struct {
+	Type   string `json:"type"`
+	RoomID string `json:"room_id"`
+	UserID string `json:"user_id"`
+	Typing bool   `json:"typing"`
+}
+
+type ReceiptEvent struct {
+	UserID    string                      `json:"user_id"`
+	RoomID    string                      `json:"room_id"`
+	EventID   string                      `json:"event_id"`
+	Type      string                      `json:"type"`
+	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
+}
+
+type FederationReceiptMRead struct {
+	User map[string]FederationReceiptData `json:"m.read"`
+}
+
+type FederationReceiptData struct {
+	Data     ReceiptTS `json:"data"`
+	EventIDs []string  `json:"event_ids"`
+}
+
+type ReceiptMRead struct {
+	User map[string]ReceiptTS `json:"m.read"`
+}
+
+type ReceiptTS struct {
+	TS gomatrixserverlib.Timestamp `json:"ts"`
+}
+
+type CrossSigningKeyUpdate struct {
+	MasterKey      *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"`
+	SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"`
+	UserID         string                             `json:"user_id"`
+}

eduserver/eduserver.go

@@ -52,6 +52,7 @@ func NewInternalAPI(
 		OutputTypingEventTopic:       cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent),
 		OutputSendToDeviceEventTopic: cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent),
 		OutputReceiptEventTopic:      cfg.Matrix.Kafka.TopicFor(config.TopicOutputReceiptEvent),
+		OutputKeyChangeEventTopic:    cfg.Matrix.Kafka.TopicFor(config.TopicOutputKeyChangeEvent),
 		ServerName:                   cfg.Matrix.ServerName,
 	}
 }

eduserver/input/input.go

@@ -24,6 +24,7 @@ import (
 	"github.com/Shopify/sarama"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/eduserver/cache"
+	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
@@ -39,6 +40,8 @@ type EDUServerInputAPI struct {
 	OutputSendToDeviceEventTopic string
 	// The kafka topic to output new receipt events to
 	OutputReceiptEventTopic string
+	// The kafka topic to output new key change events to
+	OutputKeyChangeEventTopic string
 	// kafka producer
 	Producer sarama.SyncProducer
 	// Internal user query API
@@ -77,6 +80,36 @@ func (t *EDUServerInputAPI) InputSendToDeviceEvent(
 	return t.sendToDeviceEvent(ise)
 }
 
+// InputCrossSigningKeyUpdate implements api.EDUServerInputAPI
+func (t *EDUServerInputAPI) InputCrossSigningKeyUpdate(
+	ctx context.Context,
+	request *api.InputCrossSigningKeyUpdateRequest,
+	response *api.InputCrossSigningKeyUpdateResponse,
+) error {
+	eventJSON, err := json.Marshal(&keyapi.DeviceMessage{
+		Type: keyapi.TypeCrossSigningUpdate,
+		OutputCrossSigningKeyUpdate: &api.OutputCrossSigningKeyUpdate{
+			CrossSigningKeyUpdate: request.CrossSigningKeyUpdate,
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	logrus.WithFields(logrus.Fields{
+		"user_id": request.UserID,
+	}).Infof("Producing to topic '%s'", t.OutputKeyChangeEventTopic)
+
+	m := &sarama.ProducerMessage{
+		Topic: string(t.OutputKeyChangeEventTopic),
+		Key:   sarama.StringEncoder(request.UserID),
+		Value: sarama.ByteEncoder(eventJSON),
+	}
+
+	_, _, err = t.Producer.SendMessage(m)
+	return err
+}
+
 func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error {
 	ev := &api.TypingEvent{
 		Type:   gomatrixserverlib.MTyping,

eduserver/inthttp/client.go

@@ -12,9 +12,10 @@ import (
 
 // HTTP paths for the internal HTTP APIs
 const (
-	EDUServerInputTypingEventPath       = "/eduserver/input"
-	EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice"
-	EDUServerInputReceiptEventPath      = "/eduserver/receipt"
+	EDUServerInputTypingEventPath           = "/eduserver/input"
+	EDUServerInputSendToDeviceEventPath     = "/eduserver/sendToDevice"
+	EDUServerInputReceiptEventPath          = "/eduserver/receipt"
+	EDUServerInputCrossSigningKeyUpdatePath = "/eduserver/crossSigningKeyUpdate"
 )
 
 // NewEDUServerClient creates a EDUServerInputAPI implemented by talking to a HTTP POST API.
@@ -68,3 +69,16 @@ func (h *httpEDUServerInputAPI) InputReceiptEvent(
 	apiURL := h.eduServerURL + EDUServerInputReceiptEventPath
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
+
+// InputCrossSigningKeyUpdate implements EDUServerInputAPI
+func (h *httpEDUServerInputAPI) InputCrossSigningKeyUpdate(
+	ctx context.Context,
+	request *api.InputCrossSigningKeyUpdateRequest,
+	response *api.InputCrossSigningKeyUpdateResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "InputCrossSigningKeyUpdate")
+	defer span.Finish()
+
+	apiURL := h.eduServerURL + EDUServerInputCrossSigningKeyUpdatePath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

eduserver/inthttp/server.go

@@ -51,4 +51,17 @@ func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(EDUServerInputCrossSigningKeyUpdatePath,
+		httputil.MakeInternalAPI("inputCrossSigningKeyUpdate", func(req *http.Request) util.JSONResponse {
+			var request api.InputCrossSigningKeyUpdateRequest
+			var response api.InputCrossSigningKeyUpdateResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := t.InputCrossSigningKeyUpdate(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }

federationapi/api/servers.go

@@ -0,0 +1,11 @@
+package api
+
+import (
+	"context"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type ServersInRoomProvider interface {
+	GetServersForRoom(ctx context.Context, roomID string, event *gomatrixserverlib.Event) []gomatrixserverlib.ServerName
+}

federationapi/federationapi.go

@@ -17,6 +17,7 @@ package federationapi
 import (
 	"github.com/gorilla/mux"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
+	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
@@ -39,10 +40,12 @@ func AddPublicRoutes(
 	eduAPI eduserverAPI.EDUServerInputAPI,
 	keyAPI keyserverAPI.KeyInternalAPI,
 	mscCfg *config.MSCs,
+	servers federationAPI.ServersInRoomProvider,
 ) {
 	routing.Setup(
 		fedRouter, keyRouter, cfg, rsAPI,
 		eduAPI, federationSenderAPI, keyRing,
 		federation, userAPI, keyAPI, mscCfg,
+		servers,
 	)
 }

federationapi/federationapi_test.go

@@ -31,7 +31,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	fsAPI := base.FederationSenderHTTPClient()
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, &cfg.MSCs)
+	federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, &cfg.MSCs, nil)
 	baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true)
 	defer cancel()
 	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))

federationapi/routing/devices.go

@@ -37,12 +37,27 @@ func GetUserDevices(
 		return jsonerror.InternalServerError()
 	}
 
+	sigReq := &keyapi.QuerySignaturesRequest{
+		TargetIDs: map[string][]gomatrixserverlib.KeyID{
+			userID: {},
+		},
+	}
+	sigRes := &keyapi.QuerySignaturesResponse{}
+	keyAPI.QuerySignatures(req.Context(), sigReq, sigRes)
+
 	response := gomatrixserverlib.RespUserDevices{
 		UserID:   userID,
 		StreamID: res.StreamID,
 		Devices:  []gomatrixserverlib.RespUserDevice{},
 	}
 
+	if masterKey, ok := sigRes.MasterKeys[userID]; ok {
+		response.MasterKey = &masterKey
+	}
+	if selfSigningKey, ok := sigRes.SelfSigningKeys[userID]; ok {
+		response.SelfSigningKey = &selfSigningKey
+	}
+
 	for _, dev := range res.Devices {
 		var key gomatrixserverlib.RespUserDeviceKeys
 		err := json.Unmarshal(dev.DeviceKeys.KeyJSON, &key)
@@ -56,6 +71,20 @@ func GetUserDevices(
 			DisplayName: dev.DisplayName,
 			Keys:        key,
 		}
+
+		if targetUser, ok := sigRes.Signatures[userID]; ok {
+			if targetKey, ok := targetUser[gomatrixserverlib.KeyID(dev.DeviceID)]; ok {
+				for sourceUserID, forSourceUser := range targetKey {
+					for sourceKeyID, sourceKey := range forSourceUser {
+						if _, ok := device.Keys.Signatures[sourceUserID]; !ok {
+							device.Keys.Signatures[sourceUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{}
+						}
+						device.Keys.Signatures[sourceUserID][sourceKeyID] = sourceKey
+					}
+				}
+			}
+		}
+
 		response.Devices = append(response.Devices, device)
 	}
 

federationapi/routing/invite.go

@@ -40,22 +40,29 @@ func InviteV2(
 ) util.JSONResponse {
 	inviteReq := gomatrixserverlib.InviteV2Request{}
 	err := json.Unmarshal(request.Content(), &inviteReq)
-	switch err.(type) {
+	switch e := err.(type) {
+	case gomatrixserverlib.UnsupportedRoomVersionError:
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnsupportedRoomVersion(
+				fmt.Sprintf("Room version %q is not supported by this server.", e.Version),
+			),
+		}
 	case gomatrixserverlib.BadJSONError:
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.BadJSON(err.Error()),
 		}
 	case nil:
+		return processInvite(
+			httpReq.Context(), true, inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys,
+		)
 	default:
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.NotJSON("The request body could not be decoded into an invite request. " + err.Error()),
 		}
 	}
-	return processInvite(
-		httpReq.Context(), true, inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys,
-	)
 }
 
 // InviteV1 implements /_matrix/federation/v1/invite/{roomID}/{eventID}