Merge branch 'master' into matthew/peeking-over-fed

CHANGES.md

@@ -1,5 +1,49 @@
 # Changelog
 
+## Dendrite 0.3.1 (2020-11-20)
+
+### Features
+
+* Memory optimisation by reference passing, significantly reducing the number of allocations and duplication in memory
+* A hook API has been added for experimental MSCs, with an early implementation of MSC2836
+* The last seen timestamp and IP address are now updated automatically when calling `/sync`
+* The last seen timestamp and IP address are now reported in `/_matrix/client/r0/devices` (contributed by [alexkursell](https://github.com/alexkursell))
+* An optional configuration option `sync_api.real_ip_header` has been added for specifying which HTTP header contains the real client IP address (for if Dendrite is running behind a reverse HTTP proxy)
+* Partial implementation of `/_matrix/client/r0/admin/whois` (contributed by [DavidSpenler](https://github.com/DavidSpenler))
+
+### Fixes
+
+* A concurrency bug has been fixed in the federation API that could cause Dendrite to crash
+* The error when registering a username with invalid characters has been corrected (contributed by [bodqhrohro](https://github.com/bodqhrohro))
+
+## Dendrite 0.3.0 (2020-11-16)
+
+### Features
+
+* Read receipts (both inbound and outbound) are now supported (contributed by [S7evinK](https://github.com/S7evinK))
+* Forgetting rooms is now supported (contributed by [S7evinK](https://github.com/S7evinK))
+* The `-version` command line flag has been added (contributed by [S7evinK](https://github.com/S7evinK))
+
+### Fixes
+
+* User accounts that contain the `=` character can now be registered
+* Backfilling should now work properly on rooms with world-readable history visibility (contributed by [MayeulC](https://github.com/MayeulC))
+* The `gjson` dependency has been updated for correct JSON integer ranges
+* Some more client event fields have been marked as omit-when-empty (contributed by [S7evinK](https://github.com/S7evinK))
+* The `build.sh` script has been updated to work properly on all POSIX platforms (contributed by [felix](https://github.com/felix))
+
+## Dendrite 0.2.1 (2020-10-22)
+
+### Fixes
+
+* Forward extremities are now calculated using only references from other extremities, rather than including outliers, which should fix cases where state can become corrupted ([#1556](https://github.com/matrix-org/dendrite/pull/1556))
+* Old state events will no longer be processed by the sync API as new, which should fix some cases where clients incorrectly believe they have joined or left rooms ([#1548](https://github.com/matrix-org/dendrite/pull/1548))
+* More SQLite database locking issues have been resolved in the latest events updater ([#1554](https://github.com/matrix-org/dendrite/pull/1554))
+* Internal HTTP API calls are now made using H2C (HTTP/2) in polylith mode, mitigating some potential head-of-line blocking issues ([#1541](https://github.com/matrix-org/dendrite/pull/1541))
+* Roomserver output events no longer incorrectly flag state rewrites ([#1557](https://github.com/matrix-org/dendrite/pull/1557))
+* Notification levels are now parsed correctly in power level events ([gomatrixserverlib#228](https://github.com/matrix-org/gomatrixserverlib/pull/228), contributed by [Pestdoktor](https://github.com/Pestdoktor))
+* Invalid UTF-8 is now correctly rejected when making federation requests ([gomatrixserverlib#229](https://github.com/matrix-org/gomatrixserverlib/pull/229), contributed by [Pestdoktor](https://github.com/Pestdoktor))
+
 ## Dendrite 0.2.0 (2020-10-20)
 
 ### Important

README.md

@@ -18,7 +18,7 @@ As of October 2020, Dendrite has now entered **beta** which means:
 This does not mean:
  - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
  - All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular,
-   read receipts, presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
+   presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
  - Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
@@ -54,31 +54,32 @@ The following instructions are enough to get Dendrite started as a non-federatin
 ```bash
 $ git clone https://github.com/matrix-org/dendrite
 $ cd dendrite
+$ ./build.sh
 
-# generate self-signed certificate and an event signing key for federation
-$ go build ./cmd/generate-keys
-$ ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
+# Generate a Matrix signing key for federation (required)
+$ ./bin/generate-keys --private-key matrix_key.pem
 
-# Copy and modify the config file:
-# you'll need to set a server name and paths to the keys at the very least, along with setting
-# up the database filenames
+# Generate a self-signed certificate (optional, but a valid TLS certificate is normally
+# needed for Matrix federation/clients to work properly!)
+$ ./bin/generate-keys --tls-cert server.crt --tls-key server.key
+
+# Copy and modify the config file - you'll need to set a server name and paths to the keys
+# at the very least, along with setting up the database connection strings.
 $ cp dendrite-config.yaml dendrite.yaml
 
-# build and run the server
-$ go build ./cmd/dendrite-monolith-server
-$ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
+# Build and run the server:
+$ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
 ```
 
-Then point your favourite Matrix client at `http://localhost:8008`.
+Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
 
 ## Progress
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of October 2020 we're at around 57% CS API coverage and 81% Federation coverage, though check
+updates with CI. As of November 2020 we're at around 58% CS API coverage and 83% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably:
- - Receipts
  - Push
  - Search and Context
  - User Directory
@@ -98,6 +99,7 @@ This means Dendrite supports amongst others:
  - Redaction
  - Tagging
  - E2E keys and device lists
+ - Receipts
 
 
 ## Contributing

appservice/consumers/roomserver.go

@@ -88,7 +88,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 		return nil
 	}
 
-	events := []gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
+	events := []*gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
 	events = append(events, output.NewRoomEvent.AddStateEvents...)
 
 	// Send event to any relevant application services
@@ -102,14 +102,14 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 // application service.
 func (s *OutputRoomEventConsumer) filterRoomserverEvents(
 	ctx context.Context,
-	events []gomatrixserverlib.HeaderedEvent,
+	events []*gomatrixserverlib.HeaderedEvent,
 ) error {
 	for _, ws := range s.workerStates {
 		for _, event := range events {
 			// Check if this event is interesting to this application service
 			if s.appserviceIsInterestedInEvent(ctx, event, ws.AppService) {
 				// Queue this event to be sent off to the application service
-				if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, &event); err != nil {
+				if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, event); err != nil {
 					log.WithError(err).Warn("failed to insert incoming event into appservices database")
 				} else {
 					// Tell our worker to send out new messages by updating remaining message
@@ -125,7 +125,7 @@ func (s *OutputRoomEventConsumer) filterRoomserverEvents(
 
 // appserviceIsInterestedInEvent returns a boolean depending on whether a given
 // event falls within one of a given application service's namespaces.
-func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
+func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
 	// No reason to queue events if they'll never be sent to the application
 	// service
 	if appservice.URL == "" {

appservice/workers/transaction_scheduler.go

@@ -185,7 +185,7 @@ func createTransaction(
 		}
 	}
 
-	var ev []gomatrixserverlib.Event
+	var ev []*gomatrixserverlib.Event
 	for _, e := range events {
 		ev = append(ev, e.Event)
 	}

build-dendritejs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -eu
+#!/bin/sh -eu
 
 export GIT_COMMIT=$(git rev-list -1 HEAD) && \
-GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs
+GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs

build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -eu
+#!/bin/sh -eu
 
 # Put installed packages into ./bin
 export GOBIN=$PWD/`dirname $0`/bin
@@ -7,7 +7,7 @@ if [ -d ".git" ]
 then
     export BUILD=`git rev-parse --short HEAD || ""`
     export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""`
-    if [[ $BRANCH == "master" ]]
+    if [ "$BRANCH" = master ]
     then
         export BRANCH=""
     fi

build/docker/README.md

@@ -2,6 +2,11 @@
 
 These are Docker images for Dendrite!
 
+They can be found on Docker Hub:
+
+- [matrixdotorg/dendrite-monolith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-monolith) for monolith deployments
+- [matrixdotorg/dendrite-polylith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-polylith) for polylith deployments
+
 ## Dockerfiles
 
 The `Dockerfile` builds the base image which contains all of the Dendrite

build/docker/docker-compose.deps.yml

@@ -1,5 +1,6 @@
 version: "3.4"
 services:
+  # PostgreSQL is needed for both polylith and monolith modes.
   postgres:
     hostname: postgres
     image: postgres:9.6
@@ -15,12 +16,14 @@ services:
     networks:
       - internal
 
+  # Zookeeper is only needed for polylith mode!
   zookeeper:
     hostname: zookeeper
     image: zookeeper
     networks:
       - internal
 
+  # Kafka is only needed for polylith mode!
   kafka:
     container_name: dendrite_kafka
     hostname: kafka
@@ -29,8 +32,6 @@ services:
       KAFKA_ADVERTISED_HOST_NAME: "kafka"
       KAFKA_DELETE_TOPIC_ENABLE: "true"
       KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
-    ports:
-      - 9092:9092
     depends_on:
       - zookeeper
     networks:

build/docker/docker-compose.monolith.yml

@@ -7,6 +7,9 @@ services:
       "--tls-cert=server.crt",
       "--tls-key=server.key"
     ]
+    ports:
+      - 8008:8008
+      - 8448:8448
     volumes:
       - ./config:/etc/dendrite
     networks:

build/gobind/monolith.go

@@ -112,22 +112,24 @@ func (m *DendriteMonolith) Start() {
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
-	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
-	keyAPI.SetUserAPI(userAPI)
 
 	rsAPI := roomserver.NewInternalAPI(
 		base, keyRing,
 	)
 
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing,
+	)
+
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
+	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
+	keyAPI.SetUserAPI(userAPI)
+
 	eduInputAPI := eduserver.NewInternalAPI(
 		base, cache.New(), userAPI,
 	)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	fsAPI := federationsender.NewInternalAPI(
-		base, federation, rsAPI, keyRing,
-	)
 
 	ygg.SetSessionFunc(func(address string) {
 		req := &api.PerformServersAliveRequest{

clientapi/routing/account_data.go

@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
+	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
 
@@ -148,7 +149,8 @@ type fullyReadEvent struct {
 
 // SaveReadMarker implements POST /rooms/{roomId}/read_markers
 func SaveReadMarker(
-	req *http.Request, userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
+	req *http.Request,
+	userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI,
 	syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
 ) util.JSONResponse {
 	// Verify that the user is a member of this room
@@ -192,8 +194,10 @@ func SaveReadMarker(
 		return jsonerror.InternalServerError()
 	}
 
-	// TODO handle the read receipt that may be included in the read marker
-	// See https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-rooms-roomid-read-markers
+	// Handle the read receipt that may be included in the read marker
+	if r.Read != "" {
+		return SetReceipt(req, eduAPI, device, roomID, "m.read", r.Read)
+	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,

clientapi/routing/admin_whois.go

@@ -0,0 +1,88 @@
+// Copyright 2020 David Spenler
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+
+	"github.com/matrix-org/util"
+)
+
+type adminWhoisResponse struct {
+	UserID  string                `json:"user_id"`
+	Devices map[string]deviceInfo `json:"devices"`
+}
+
+type deviceInfo struct {
+	Sessions []sessionInfo `json:"sessions"`
+}
+
+type sessionInfo struct {
+	Connections []connectionInfo `json:"connections"`
+}
+
+type connectionInfo struct {
+	IP        string `json:"ip"`
+	LastSeen  int64  `json:"last_seen"`
+	UserAgent string `json:"user_agent"`
+}
+
+// GetAdminWhois implements GET /admin/whois/{userId}
+func GetAdminWhois(
+	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	userID string,
+) util.JSONResponse {
+	if userID != device.UserID {
+		// TODO: Still allow if user is admin
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("userID does not match the current user"),
+		}
+	}
+
+	var queryRes api.QueryDevicesResponse
+	err := userAPI.QueryDevices(req.Context(), &api.QueryDevicesRequest{
+		UserID: userID,
+	}, &queryRes)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("GetAdminWhois failed to query user devices")
+		return jsonerror.InternalServerError()
+	}
+
+	devices := make(map[string]deviceInfo)
+	for _, device := range queryRes.Devices {
+		connInfo := connectionInfo{
+			IP:        device.LastSeenIP,
+			LastSeen:  device.LastSeenTS,
+			UserAgent: device.UserAgent,
+		}
+		dev, ok := devices[device.ID]
+		if !ok {
+			dev.Sessions = []sessionInfo{{}}
+		}
+		dev.Sessions[0].Connections = append(dev.Sessions[0].Connections, connInfo)
+		devices[device.ID] = dev
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: adminWhoisResponse{
+			UserID:  userID,
+			Devices: devices,
+		},
+	}
+}

clientapi/routing/createroom.go

@@ -255,7 +255,7 @@ func createRoom(
 		historyVisibility = historyVisibilityShared
 	}
 
-	var builtEvents []gomatrixserverlib.HeaderedEvent
+	var builtEvents []*gomatrixserverlib.HeaderedEvent
 
 	// send events into the room in order of:
 	//  1- m.room.create
@@ -327,13 +327,13 @@ func createRoom(
 			return jsonerror.InternalServerError()
 		}
 
-		if err = gomatrixserverlib.Allowed(*ev, &authEvents); err != nil {
+		if err = gomatrixserverlib.Allowed(ev, &authEvents); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.Allowed failed")
 			return jsonerror.InternalServerError()
 		}
 
 		// Add the event to the list of auth events
-		builtEvents = append(builtEvents, (*ev).Headered(roomVersion))
+		builtEvents = append(builtEvents, ev.Headered(roomVersion))
 		err = authEvents.AddEvent(ev)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("authEvents.AddEvent failed")
@@ -397,7 +397,7 @@ func createRoom(
 				ev := event.Event
 				globalStrippedState = append(
 					globalStrippedState,
-					gomatrixserverlib.NewInviteV2StrippedState(&ev),
+					gomatrixserverlib.NewInviteV2StrippedState(ev),
 				)
 			}
 		}
@@ -415,7 +415,7 @@ func createRoom(
 			}
 			inviteStrippedState := append(
 				globalStrippedState,
-				gomatrixserverlib.NewInviteV2StrippedState(&inviteEvent.Event),
+				gomatrixserverlib.NewInviteV2StrippedState(inviteEvent.Event),
 			)
 			// Send the invite event to the roomserver.
 			err = roomserverAPI.SendInvite(
@@ -488,5 +488,5 @@ func buildEvent(
 	if err != nil {
 		return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err)
 	}
-	return &event, nil
+	return event, nil
 }

clientapi/routing/device.go

@@ -16,6 +16,7 @@ package routing
 
 import (
 	"io/ioutil"
+	"net"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
@@ -32,7 +33,7 @@ type deviceJSON struct {
 	DeviceID    string `json:"device_id"`
 	DisplayName string `json:"display_name"`
 	LastSeenIP  string `json:"last_seen_ip"`
-	LastSeenTS  uint64 `json:"last_seen_ts"`
+	LastSeenTS  int64  `json:"last_seen_ts"`
 }
 
 type devicesJSON struct {
@@ -79,6 +80,8 @@ func GetDeviceByID(
 		JSON: deviceJSON{
 			DeviceID:    targetDevice.ID,
 			DisplayName: targetDevice.DisplayName,
+			LastSeenIP:  stripIPPort(targetDevice.LastSeenIP),
+			LastSeenTS:  targetDevice.LastSeenTS,
 		},
 	}
 }
@@ -102,6 +105,8 @@ func GetDevicesByLocalpart(
 		res.Devices = append(res.Devices, deviceJSON{
 			DeviceID:    dev.ID,
 			DisplayName: dev.DisplayName,
+			LastSeenIP:  stripIPPort(dev.LastSeenIP),
+			LastSeenTS:  dev.LastSeenTS,
 		})
 	}
 
@@ -230,3 +235,20 @@ func DeleteDevices(
 		JSON: struct{}{},
 	}
 }
+
+// stripIPPort converts strings like "[::1]:12345" to "::1"
+func stripIPPort(addr string) string {
+	ip := net.ParseIP(addr)
+	if ip != nil {
+		return addr
+	}
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return ""
+	}
+	ip = net.ParseIP(host)
+	if ip != nil {
+		return host
+	}
+	return ""
+}

clientapi/routing/getevent.go

@@ -32,7 +32,7 @@ type getEventRequest struct {
 	eventID        string
 	cfg            *config.ClientAPI
 	federation     *gomatrixserverlib.FederationClient
-	requestedEvent gomatrixserverlib.Event
+	requestedEvent *gomatrixserverlib.Event
 }
 
 // GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}

clientapi/routing/membership.go

@@ -77,7 +77,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us
 	if err = roomserverAPI.SendEvents(
 		ctx, rsAPI,
 		api.KindNew,
-		[]gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)},
+		[]*gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)},
 		cfg.Matrix.ServerName,
 		nil,
 	); err != nil {
@@ -214,7 +214,7 @@ func SendInvite(
 
 	err = roomserverAPI.SendInvite(
 		req.Context(), rsAPI,
-		*event,
+		event,
 		nil, // ask the roomserver to draw up invite room state for us
 		cfg.Matrix.ServerName,
 		nil,
@@ -407,3 +407,47 @@ func checkMemberInRoom(ctx context.Context, rsAPI api.RoomserverInternalAPI, use
 	}
 	return nil
 }
+
+func SendForget(
+	req *http.Request, device *userapi.Device,
+	roomID string, rsAPI roomserverAPI.RoomserverInternalAPI,
+) util.JSONResponse {
+	ctx := req.Context()
+	logger := util.GetLogger(ctx).WithField("roomID", roomID).WithField("userID", device.UserID)
+	var membershipRes api.QueryMembershipForUserResponse
+	membershipReq := api.QueryMembershipForUserRequest{
+		RoomID: roomID,
+		UserID: device.UserID,
+	}
+	err := rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes)
+	if err != nil {
+		logger.WithError(err).Error("QueryMembershipForUser: could not query membership for user")
+		return jsonerror.InternalServerError()
+	}
+	if membershipRes.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Forbidden("user is still a member of the room"),
+		}
+	}
+	if !membershipRes.HasBeenInRoom {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Forbidden("user did not belong to room"),
+		}
+	}
+
+	request := api.PerformForgetRequest{
+		RoomID: roomID,
+		UserID: device.UserID,
+	}
+	response := api.PerformForgetResponse{}
+	if err := rsAPI.PerformForget(ctx, &request, &response); err != nil {
+		logger.WithError(err).Error("PerformForget: unable to forget room")
+		return jsonerror.InternalServerError()
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/profile.go

@@ -346,14 +346,14 @@ func buildMembershipEvents(
 	roomIDs []string,
 	newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
 	evTime time.Time, rsAPI api.RoomserverInternalAPI,
-) ([]gomatrixserverlib.HeaderedEvent, error) {
-	evs := []gomatrixserverlib.HeaderedEvent{}
+) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	evs := []*gomatrixserverlib.HeaderedEvent{}
 
 	for _, roomID := range roomIDs {
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
 		if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
-			return []gomatrixserverlib.HeaderedEvent{}, err
+			return nil, err
 		}
 
 		builder := gomatrixserverlib.EventBuilder{
@@ -379,7 +379,7 @@ func buildMembershipEvents(
 			return nil, err
 		}
 
-		evs = append(evs, (*event).Headered(verRes.RoomVersion))
+		evs = append(evs, event.Headered(verRes.RoomVersion))
 	}
 
 	return evs, nil

clientapi/routing/receipt.go

@@ -0,0 +1,54 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/eduserver/api"
+
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+)
+
+func SetReceipt(req *http.Request, eduAPI api.EDUServerInputAPI, device *userapi.Device, roomId, receiptType, eventId string) util.JSONResponse {
+	timestamp := gomatrixserverlib.AsTimestamp(time.Now())
+	logrus.WithFields(logrus.Fields{
+		"roomId":      roomId,
+		"receiptType": receiptType,
+		"eventId":     eventId,
+		"userId":      device.UserID,
+		"timestamp":   timestamp,
+	}).Debug("Setting receipt")
+
+	// currently only m.read is accepted
+	if receiptType != "m.read" {
+		return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType))
+	}
+
+	if err := api.SendReceipt(req.Context(), eduAPI, device.UserID, roomId, eventId, receiptType, timestamp); err != nil {
+		return util.ErrorResponse(err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/redaction.go

@@ -121,7 +121,7 @@ func SendRedaction(
 			JSON: jsonerror.NotFound("Room does not exist"),
 		}
 	}
-	if err = roomserverAPI.SendEvents(context.Background(), rsAPI, api.KindNew, []gomatrixserverlib.HeaderedEvent{*e}, cfg.Matrix.ServerName, nil); err != nil {
+	if err = roomserverAPI.SendEvents(context.Background(), rsAPI, api.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, cfg.Matrix.ServerName, nil); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/register.go

@@ -113,7 +113,7 @@ var (
 	// TODO: Remove old sessions. Need to do so on a session-specific timeout.
 	// sessions stores the completed flow stages for all sessions. Referenced using their sessionID.
 	sessions           = newSessionsDict()
-	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-./]+$`)
+	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 )
 
 // registerRequest represents the submitted registration request.
@@ -209,7 +209,7 @@ func validateUsername(username string) *util.JSONResponse {
 	} else if !validUsernameRegex.MatchString(username) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./'"),
+			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
 		}
 	} else if username[0] == '_' { // Regex checks its not a zero length string
 		return &util.JSONResponse{
@@ -230,7 +230,7 @@ func validateApplicationServiceUsername(username string) *util.JSONResponse {
 	} else if !validUsernameRegex.MatchString(username) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./'"),
+			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
 		}
 	}
 	return nil

clientapi/routing/routing.go

@@ -651,6 +651,16 @@ func Setup(
 		}),
 	).Methods(http.MethodGet)
 
+	r0mux.Handle("/admin/whois/{userID}",
+		httputil.MakeAuthAPI("admin_whois", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetAdminWhois(req, userAPI, device, vars["userID"])
+		}),
+	).Methods(http.MethodGet)
+
 	r0mux.Handle("/user_directory/search",
 		httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.rateLimit(req); r != nil {
@@ -705,7 +715,20 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SaveReadMarker(req, userAPI, rsAPI, syncProducer, device, vars["roomID"])
+			return SaveReadMarker(req, userAPI, rsAPI, eduAPI, syncProducer, device, vars["roomID"])
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/forget",
+		httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			if r := rateLimits.rateLimit(req); r != nil {
+				return *r
+			}
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendForget(req, device, vars["roomID"], rsAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -830,4 +853,17 @@ func Setup(
 			return ClaimKeys(req, keyAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			if r := rateLimits.rateLimit(req); r != nil {
+				return *r
+			}
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+
+			return SetReceipt(req, eduAPI, device, vars["roomId"], vars["receiptType"], vars["eventId"])
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
 }

clientapi/routing/sendevent.go

@@ -93,7 +93,7 @@ func SendEvent(
 	if err := api.SendEvents(
 		req.Context(), rsAPI,
 		api.KindNew,
-		[]gomatrixserverlib.HeaderedEvent{
+		[]*gomatrixserverlib.HeaderedEvent{
 			e.Headered(verRes.RoomVersion),
 		},
 		cfg.Matrix.ServerName,
@@ -189,7 +189,7 @@ func generateSendEvent(
 	// check to see if this user can perform this operation
 	stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents))
 	for i := range queryRes.StateEvents {
-		stateEvents[i] = &queryRes.StateEvents[i].Event
+		stateEvents[i] = queryRes.StateEvents[i].Event
 	}
 	provider := gomatrixserverlib.NewAuthEvents(stateEvents)
 	if err = gomatrixserverlib.Allowed(e.Event, &provider); err != nil {
@@ -198,5 +198,5 @@ func generateSendEvent(
 			JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
 		}
 	}
-	return &e.Event, nil
+	return e.Event, nil
 }

clientapi/routing/state.go

@@ -267,7 +267,7 @@ func OnIncomingStateTypeRequest(
 		// to find the state event, if provided.
 		for _, ev := range stateRes.StateEvents {
 			if ev.Type() == evType && ev.StateKeyEquals(stateKey) {
-				event = &ev
+				event = ev
 				break
 			}
 		}
@@ -290,7 +290,7 @@ func OnIncomingStateTypeRequest(
 			return jsonerror.InternalServerError()
 		}
 		if len(stateAfterRes.StateEvents) > 0 {
-			event = &stateAfterRes.StateEvents[0]
+			event = stateAfterRes.StateEvents[0]
 		}
 	}
 
@@ -304,7 +304,7 @@ func OnIncomingStateTypeRequest(
 	}
 
 	stateEvent := stateEventInStateResp{
-		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(*event, gomatrixserverlib.FormatAll),
+		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(event, gomatrixserverlib.FormatAll),
 	}
 
 	var res interface{}

clientapi/threepid/invites.go

@@ -362,8 +362,8 @@ func emit3PIDInviteEvent(
 	return api.SendEvents(
 		ctx, rsAPI,
 		api.KindNew,
-		[]gomatrixserverlib.HeaderedEvent{
-			(*event).Headered(queryRes.RoomVersion),
+		[]*gomatrixserverlib.HeaderedEvent{
+			event.Headered(queryRes.RoomVersion),
 		},
 		cfg.Matrix.ServerName,
 		nil,

cmd/create-room-events/main.go

@@ -123,7 +123,7 @@ func buildAndOutput() gomatrixserverlib.EventReference {
 }
 
 // Write an event to the output.
-func writeEvent(event gomatrixserverlib.Event) {
+func writeEvent(event *gomatrixserverlib.Event) {
 	encoder := json.NewEncoder(os.Stdout)
 	if *format == "InputRoomEvent" {
 		var ire api.InputRoomEvent

cmd/dendrite-monolith-server/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/mscs"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
@@ -148,6 +149,12 @@ func main() {
 		base.PublicMediaAPIMux,
 	)
 
+	if len(base.Cfg.MSCs.MSCs) > 0 {
+		if err := mscs.Enable(base, &monolith); err != nil {
+			logrus.WithError(err).Fatalf("Failed to enable MSCs")
+		}
+	}
+
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {
 		base.SetupAndServeHTTP(

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -21,7 +21,8 @@ import (
 )
 
 func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
+	fsAPI := base.FederationSenderHTTPClient()
+	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI)
 	intAPI.SetUserAPI(base.UserAPIClient())
 
 	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)

cmd/resolve-state/main.go

@@ -80,9 +80,9 @@ func main() {
 	}
 
 	authEventIDMap := make(map[string]struct{})
-	eventPtrs := make([]*gomatrixserverlib.Event, len(eventEntries))
+	events := make([]*gomatrixserverlib.Event, len(eventEntries))
 	for i := range eventEntries {
-		eventPtrs[i] = &eventEntries[i].Event
+		events[i] = eventEntries[i].Event
 		for _, authEventID := range eventEntries[i].AuthEventIDs() {
 			authEventIDMap[authEventID] = struct{}{}
 		}
@@ -99,18 +99,9 @@ func main() {
 		panic(err)
 	}
 
-	authEventPtrs := make([]*gomatrixserverlib.Event, len(authEventEntries))
+	authEvents := make([]*gomatrixserverlib.Event, len(authEventEntries))
 	for i := range authEventEntries {
-		authEventPtrs[i] = &authEventEntries[i].Event
-	}
-
-	events := make([]gomatrixserverlib.Event, len(eventEntries))
-	authEvents := make([]gomatrixserverlib.Event, len(authEventEntries))
-	for i, ptr := range eventPtrs {
-		events[i] = *ptr
-	}
-	for i, ptr := range authEventPtrs {
-		authEvents[i] = *ptr
+		authEvents[i] = authEventEntries[i].Event
 	}
 
 	fmt.Println("Resolving state")

cmd/syncserver-integration-tests/main.go

@@ -103,7 +103,7 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string {
 	if err := json.Unmarshal([]byte(outputRoomEvent), &out); err != nil {
 		panic("failed to unmarshal output room event: " + err.Error())
 	}
-	clientEvs := gomatrixserverlib.ToClientEvents([]gomatrixserverlib.Event{
+	clientEvs := gomatrixserverlib.ToClientEvents([]*gomatrixserverlib.Event{
 		out.NewRoomEvent.Event.Event,
 	}, gomatrixserverlib.FormatSync)
 	b, err := json.Marshal(clientEvs[0])

dendrite-config.yaml

@@ -36,6 +36,8 @@ global:
   server_name: localhost
 
   # The path to the signing private key file, used to sign requests and events.
+  # Note that this is NOT the same private key as used for TLS! To generate a
+  # signing key, use "./bin/generate-keys --private-key matrix_key.pem".
   private_key: matrix_key.pem
 
   # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
@@ -74,6 +76,12 @@ global:
     # Kafka.
     use_naffka: true
 
+    # The max size a Kafka message is allowed to use.
+    # You only need to change this value, if you encounter issues with too large messages.
+    # Must be less than/equal to "max.message.bytes" configured in Kafka.
+    # Defaults to 8388608 bytes.
+    # max_message_bytes: 8388608
+
     # Naffka database options. Not required when using Kafka.
     naffka_database:
       connection_string: file:naffka.db
@@ -292,6 +300,11 @@ sync_api:
     max_idle_conns: 2
     conn_max_lifetime: -1
 
+  # This option controls which HTTP header to inspect to find the real remote IP
+  # address of the client. This is likely required if Dendrite is running behind
+  # a reverse proxy server.
+  # real_ip_header: X-Real-IP
+
 # Configuration for the User API.
 user_api:
   internal_api:

docs/INSTALL.md

@@ -12,6 +12,8 @@ Dendrite can be run in one of two configurations:
    lightweight implementation called [Naffka](https://github.com/matrix-org/naffka). This
    will usually be the preferred model for low-volume, low-user or experimental deployments.
 
+For most deployments, it is **recommended to run in monolith mode with PostgreSQL databases**.
+
 Regardless of whether you are running in polylith or monolith mode, each Dendrite component that
 requires storage has its own database. Both Postgres and SQLite are supported and can be
 mixed-and-matched across components as needed in the configuration file.
@@ -30,23 +32,9 @@ If you want to run a polylith deployment, you also need:
 
 * Apache Kafka 0.10.2+
 
-## Building up a monolith deploment
+Please note that Kafka is **not required** for a monolith deployment.
 
-Start by cloning the code:
-
-```bash
-git clone https://github.com/matrix-org/dendrite
-cd dendrite
-```
-
-Then build it:
-
-```bash
-go build -o bin/dendrite-monolith-server ./cmd/dendrite-monolith-server
-go build -o bin/generate-keys ./cmd/generate-keys
-```
-
-## Building up a polylith deployment
+## Building Dendrite
 
 Start by cloning the code:
 
@@ -61,6 +49,8 @@ Then build it:
 ./build.sh
 ```
 
+## Install Kafka (polylith only)
+
 Install and start Kafka (c.f. [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh)):
 
 ```bash
@@ -96,9 +86,9 @@ Dendrite can use the built-in SQLite database engine for small setups.
 The SQLite databases do not need to be pre-built - Dendrite will
 create them automatically at startup.
 
-### Postgres database setup
+### PostgreSQL database setup
 
-Assuming that Postgres 9.6 (or later) is installed:
+Assuming that PostgreSQL 9.6 (or later) is installed:
 
 * Create role, choosing a new password when prompted:
 
@@ -118,17 +108,31 @@ Assuming that Postgres 9.6 (or later) is installed:
 
 ### Server key generation
 
-Each Dendrite server requires unique server keys.
+Each Dendrite installation requires:
 
-In order for an instance to federate correctly, you should have a valid
-certificate issued by a trusted authority, and private key to match. If you
-don't and just want to test locally, generate the self-signed SSL certificate
-for federation and the server signing key:
+- A unique Matrix signing private key
+- A valid and trusted TLS certificate and private key
+
+To generate a Matrix signing private key:
 
 ```bash
-./bin/generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
+./bin/generate-keys --private-key matrix_key.pem
 ```
 
+**Warning:** Make sure take a safe backup of this key! You will likely need it if you want to reinstall Dendrite, or
+any other Matrix homeserver, on the same domain name in the future. If you lose this key, you may have trouble joining
+federated rooms.
+
+For testing, you can generate a self-signed certificate and key, although this will not work for public federation:
+
+```bash
+./bin/generate-keys --tls-cert server.crt --tls-key server.key
+```
+
+If you have server keys from an older Synapse instance, 
+[convert them](serverkeyformat.md#converting-synapse-keys) to Dendrite's PEM 
+format and configure them as `old_private_keys` in your config.
+
 ### Configuration file
 
 Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Things that will need editing include *at least*:
@@ -136,9 +140,9 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th
 * The `server_name` entry to reflect the hostname of your Dendrite server
 * The `database` lines with an updated connection string based on your
   desired setup, e.g. replacing `database` with the name of the database:
-  * For Postgres: `postgres://dendrite:password@localhost/database`
-  * For SQLite on disk: `file:component.db` or `file:///path/to/component.db`
-  * Postgres and SQLite can be mixed and matched.
+  * For Postgres: `postgres://dendrite:password@localhost/database`, e.g. `postgres://dendrite:password@localhost/dendrite_userapi_account.db`
+  * For SQLite on disk: `file:component.db` or `file:///path/to/component.db`, e.g. `file:userapi_account.db`
+  * Postgres and SQLite can be mixed and matched on different components as desired.
 * The `use_naffka` option if using Naffka in a monolith deployment
 
 There are other options which may be useful so review them all. In particular,
@@ -148,7 +152,7 @@ help to improve reliability considerably by allowing your homeserver to fetch
 public keys for dead homeservers from somewhere else.
 
 **WARNING:** Dendrite supports running all components from the same database in
-Postgres mode, but this is **NOT** a supported configuration with SQLite. When
+PostgreSQL mode, but this is **NOT** a supported configuration with SQLite. When
 using SQLite, all components **MUST** use their own database file.
 
 ## Starting a monolith server
@@ -160,8 +164,14 @@ Be sure to update the database username and password if needed.
 
 The monolith server can be started as shown below. By default it listens for
 HTTP connections on port 8008, so you can configure your Matrix client to use
-`http://localhost:8008` as the server. If you set `--tls-cert` and `--tls-key`
-as shown below, it will also listen for HTTPS connections on port 8448.
+`http://servername:8008` as the server:
+
+```bash
+./bin/dendrite-monolith-server
+```
+
+If you set `--tls-cert` and `--tls-key` as shown below, it will also listen
+for HTTPS connections on port 8448:
 
 ```bash
 ./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key

docs/hiawatha/polylith-sample.conf

@@ -0,0 +1,16 @@
+VirtualHost {
+        ...
+        # route requests to:
+        # /_matrix/client/.*/sync
+        # /_matrix/client/.*/user/{userId}/filter
+        # /_matrix/client/.*/user/{userId}/filter/{filterID}
+        # /_matrix/client/.*/keys/changes
+        # /_matrix/client/.*/rooms/{roomId}/messages
+        # to sync_api
+        ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073
+        ReverseProxy = /_matrix/client http://localhost:8071
+        ReverseProxy = /_matrix/federation http://localhost:8072
+        ReverseProxy = /_matrix/key http://localhost:8072
+        ReverseProxy = /_matrix/media http://localhost:8074
+        ...
+}

docs/nginx/monolith-sample.conf

@@ -1,5 +1,6 @@
 server {
-    listen 443 ssl;
+    listen 443 ssl; # IPv4
+    listen [::]:443; # IPv6
     server_name my.hostname.com;
 
     ssl_certificate /path/to/fullchain.pem;

docs/nginx/polylith-sample.conf

@@ -1,5 +1,6 @@
 server {
-    listen 443 ssl;
+    listen 443 ssl; # IPv4
+    listen [::]:443; # IPv6
     server_name my.hostname.com;
 
     ssl_certificate /path/to/fullchain.pem;

docs/serverkeyformat.md

@@ -0,0 +1,29 @@
+# Server Key Format
+
+Dendrite stores the server signing key in the PEM format with the following structure.
+
+```
+-----BEGIN MATRIX PRIVATE KEY-----
+Key-ID: ed25519:<Key Handle>
+
+<Base64 Encoded Key Data>
+-----END MATRIX PRIVATE KEY-----
+```
+
+## Converting Synapse Keys
+
+If you have signing keys from a previous synapse server, you should ideally configure them as `old_private_keys` in your Dendrite config file. Synapse stores signing keys in the following format.
+
+```
+ed25519 <Key Handle> <Base64 Encoded Key Data>
+```
+
+To convert this key to Dendrite's PEM format, use the following template. **It is important to include the equals sign, as the key data needs to be padded to 32 bytes.**
+
+```
+-----BEGIN MATRIX PRIVATE KEY-----
+Key-ID: ed25519:<Key Handle>
+
+<Base64 Encoded Key Data>=
+-----END MATRIX PRIVATE KEY-----
+```

docs/sytest.md

@@ -85,6 +85,7 @@ Set up the database:
 
 ```sh
 sudo -u postgres psql -c "CREATE USER dendrite PASSWORD 'itsasecret'"
+sudo -u postgres psql -c "ALTER USER dendrite CREATEDB"
 for i in dendrite0 dendrite1 sytest_template; do sudo -u postgres psql -c "CREATE DATABASE $i OWNER dendrite;"; done
 mkdir -p "server-0"
 cat > "server-0/database.yaml" << EOF

eduserver/api/input.go

@@ -59,6 +59,22 @@ type InputSendToDeviceEventRequest struct {
 // InputSendToDeviceEventResponse is a response to InputSendToDeviceEventRequest
 type InputSendToDeviceEventResponse struct{}
 
+type InputReceiptEvent struct {
+	UserID    string                      `json:"user_id"`
+	RoomID    string                      `json:"room_id"`
+	EventID   string                      `json:"event_id"`
+	Type      string                      `json:"type"`
+	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
+}
+
+// InputReceiptEventRequest is a request to EDUServerInputAPI
+type InputReceiptEventRequest struct {
+	InputReceiptEvent InputReceiptEvent `json:"input_receipt_event"`
+}
+
+// InputReceiptEventResponse is a response to InputReceiptEventRequest
+type InputReceiptEventResponse struct{}
+
 // EDUServerInputAPI is used to write events to the typing server.
 type EDUServerInputAPI interface {
 	InputTypingEvent(
@@ -72,4 +88,10 @@ type EDUServerInputAPI interface {
 		request *InputSendToDeviceEventRequest,
 		response *InputSendToDeviceEventResponse,
 	) error
+
+	InputReceiptEvent(
+		ctx context.Context,
+		request *InputReceiptEventRequest,
+		response *InputReceiptEventResponse,
+	) error
 }

eduserver/api/output.go

@@ -49,3 +49,39 @@ type OutputSendToDeviceEvent struct {
 	DeviceID string `json:"device_id"`
 	gomatrixserverlib.SendToDeviceEvent
 }
+
+type ReceiptEvent struct {
+	UserID    string                      `json:"user_id"`
+	RoomID    string                      `json:"room_id"`
+	EventID   string                      `json:"event_id"`
+	Type      string                      `json:"type"`
+	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
+}
+
+// OutputReceiptEvent is an entry in the receipt output kafka log
+type OutputReceiptEvent struct {
+	UserID    string                      `json:"user_id"`
+	RoomID    string                      `json:"room_id"`
+	EventID   string                      `json:"event_id"`
+	Type      string                      `json:"type"`
+	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
+}
+
+// Helper structs for receipts json creation
+type ReceiptMRead struct {
+	User map[string]ReceiptTS `json:"m.read"`
+}
+
+type ReceiptTS struct {
+	TS gomatrixserverlib.Timestamp `json:"ts"`
+}
+
+// FederationSender output
+type FederationReceiptMRead struct {
+	User map[string]FederationReceiptData `json:"m.read"`
+}
+
+type FederationReceiptData struct {
+	Data     ReceiptTS `json:"data"`
+	EventIDs []string  `json:"event_ids"`
+}

eduserver/api/wrapper.go

@@ -67,3 +67,22 @@ func SendToDevice(
 	response := InputSendToDeviceEventResponse{}
 	return eduAPI.InputSendToDeviceEvent(ctx, &request, &response)
 }
+
+// SendReceipt sends a receipt event to EDU Server
+func SendReceipt(
+	ctx context.Context,
+	eduAPI EDUServerInputAPI, userID, roomID, eventID, receiptType string,
+	timestamp gomatrixserverlib.Timestamp,
+) error {
+	request := InputReceiptEventRequest{
+		InputReceiptEvent: InputReceiptEvent{
+			UserID:    userID,
+			RoomID:    roomID,
+			EventID:   eventID,
+			Type:      receiptType,
+			Timestamp: timestamp,
+		},
+	}
+	response := InputReceiptEventResponse{}
+	return eduAPI.InputReceiptEvent(ctx, &request, &response)
+}

eduserver/eduserver.go

@@ -49,8 +49,9 @@ func NewInternalAPI(
 		Cache:                        eduCache,
 		UserAPI:                      userAPI,
 		Producer:                     producer,
-		OutputTypingEventTopic:       string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent)),
-		OutputSendToDeviceEventTopic: string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent)),
+		OutputTypingEventTopic:       cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent),
+		OutputSendToDeviceEventTopic: cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent),
+		OutputReceiptEventTopic:      cfg.Matrix.Kafka.TopicFor(config.TopicOutputReceiptEvent),
 		ServerName:                   cfg.Matrix.ServerName,
 	}
 }

eduserver/input/input.go

@@ -37,6 +37,8 @@ type EDUServerInputAPI struct {
 	OutputTypingEventTopic string
 	// The kafka topic to output new send to device events to.
 	OutputSendToDeviceEventTopic string
+	// The kafka topic to output new receipt events to
+	OutputReceiptEventTopic string
 	// kafka producer
 	Producer sarama.SyncProducer
 	// Internal user query API
@@ -173,3 +175,31 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e
 
 	return nil
 }
+
+// InputReceiptEvent implements api.EDUServerInputAPI
+// TODO: Intelligently batch requests sent by the same user (e.g wait a few milliseconds before emitting output events)
+func (t *EDUServerInputAPI) InputReceiptEvent(
+	ctx context.Context,
+	request *api.InputReceiptEventRequest,
+	response *api.InputReceiptEventResponse,
+) error {
+	logrus.WithFields(logrus.Fields{}).Infof("Producing to topic '%s'", t.OutputReceiptEventTopic)
+	output := &api.OutputReceiptEvent{
+		UserID:    request.InputReceiptEvent.UserID,
+		RoomID:    request.InputReceiptEvent.RoomID,
+		EventID:   request.InputReceiptEvent.EventID,
+		Type:      request.InputReceiptEvent.Type,
+		Timestamp: request.InputReceiptEvent.Timestamp,
+	}
+	js, err := json.Marshal(output)
+	if err != nil {
+		return err
+	}
+	m := &sarama.ProducerMessage{
+		Topic: t.OutputReceiptEventTopic,
+		Key:   sarama.StringEncoder(request.InputReceiptEvent.RoomID + ":" + request.InputReceiptEvent.UserID),
+		Value: sarama.ByteEncoder(js),
+	}
+	_, _, err = t.Producer.SendMessage(m)
+	return err
+}

eduserver/inthttp/client.go

@@ -14,6 +14,7 @@ import (
 const (
 	EDUServerInputTypingEventPath       = "/eduserver/input"
 	EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice"
+	EDUServerInputReceiptEventPath      = "/eduserver/receipt"
 )
 
 // NewEDUServerClient creates a EDUServerInputAPI implemented by talking to a HTTP POST API.
@@ -54,3 +55,16 @@ func (h *httpEDUServerInputAPI) InputSendToDeviceEvent(
 	apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
+
+// InputSendToDeviceEvent implements EDUServerInputAPI
+func (h *httpEDUServerInputAPI) InputReceiptEvent(
+	ctx context.Context,
+	request *api.InputReceiptEventRequest,
+	response *api.InputReceiptEventResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "InputReceiptEventPath")
+	defer span.Finish()
+
+	apiURL := h.eduServerURL + EDUServerInputReceiptEventPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

eduserver/inthttp/server.go

@@ -38,4 +38,17 @@ func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(EDUServerInputReceiptEventPath,
+		httputil.MakeInternalAPI("inputReceiptEvent", func(req *http.Request) util.JSONResponse {
+			var request api.InputReceiptEventRequest
+			var response api.InputReceiptEventResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := t.InputReceiptEvent(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }

federationapi/federationapi_test.go

@@ -72,7 +72,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 			t.Errorf("failed to parse event: %s", err)
 		}
 		he := ev.Headered(tc.roomVer)
-		invReq, err := gomatrixserverlib.NewInviteV2Request(&he, nil)
+		invReq, err := gomatrixserverlib.NewInviteV2Request(he, nil)
 		if err != nil {
 			t.Errorf("failed to create invite v2 request: %s", err)
 			continue

federationapi/routing/backfill.go

@@ -93,9 +93,9 @@ func Backfill(
 	}
 
 	// Filter any event that's not from the requested room out.
-	evs := make([]gomatrixserverlib.Event, 0)
+	evs := make([]*gomatrixserverlib.Event, 0)
 
-	var ev gomatrixserverlib.HeaderedEvent
+	var ev *gomatrixserverlib.HeaderedEvent
 	for _, ev = range res.Events {
 		if ev.RoomID() == roomID {
 			evs = append(evs, ev.Event)

federationapi/routing/events.go

@@ -98,5 +98,5 @@ func fetchEvent(ctx context.Context, rsAPI api.RoomserverInternalAPI, eventID st
 		return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
 	}
 
-	return &eventsResponse.Events[0].Event, nil
+	return eventsResponse.Events[0].Event, nil
 }

federationapi/routing/invite.go

@@ -97,7 +97,7 @@ func InviteV1(
 func processInvite(
 	ctx context.Context,
 	isInviteV2 bool,
-	event gomatrixserverlib.Event,
+	event *gomatrixserverlib.Event,
 	roomVer gomatrixserverlib.RoomVersion,
 	strippedState []gomatrixserverlib.InviteV2StrippedState,
 	roomID string,
@@ -171,12 +171,12 @@ func processInvite(
 		if isInviteV2 {
 			return util.JSONResponse{
 				Code: http.StatusOK,
-				JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent},
+				JSON: gomatrixserverlib.RespInviteV2{Event: &signedEvent},
 			}
 		} else {
 			return util.JSONResponse{
 				Code: http.StatusOK,
-				JSON: gomatrixserverlib.RespInvite{Event: signedEvent},
+				JSON: gomatrixserverlib.RespInvite{Event: &signedEvent},
 			}
 		}
 	default:

federationapi/routing/join.go

@@ -138,7 +138,7 @@ func MakeJoin(
 	// Check that the join is allowed or not
 	stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents))
 	for i := range queryRes.StateEvents {
-		stateEvents[i] = &queryRes.StateEvents[i].Event
+		stateEvents[i] = queryRes.StateEvents[i].Event
 	}
 
 	provider := gomatrixserverlib.NewAuthEvents(stateEvents)
@@ -291,7 +291,7 @@ func SendJoin(
 		if err = api.SendEvents(
 			httpReq.Context(), rsAPI,
 			api.KindNew,
-			[]gomatrixserverlib.HeaderedEvent{
+			[]*gomatrixserverlib.HeaderedEvent{
 				event.Headered(stateAndAuthChainResponse.RoomVersion),
 			},
 			cfg.Matrix.ServerName,
@@ -319,7 +319,7 @@ func SendJoin(
 	}
 }
 
-type eventsByDepth []gomatrixserverlib.HeaderedEvent
+type eventsByDepth []*gomatrixserverlib.HeaderedEvent
 
 func (e eventsByDepth) Len() int {
 	return len(e)

federationapi/routing/leave.go

@@ -98,7 +98,7 @@ func MakeLeave(
 	// Check that the leave is allowed or not
 	stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents))
 	for i := range queryRes.StateEvents {
-		stateEvents[i] = &queryRes.StateEvents[i].Event
+		stateEvents[i] = queryRes.StateEvents[i].Event
 	}
 	provider := gomatrixserverlib.NewAuthEvents(stateEvents)
 	if err = gomatrixserverlib.Allowed(event.Event, &provider); err != nil {
@@ -257,7 +257,7 @@ func SendLeave(
 	if err = api.SendEvents(
 		httpReq.Context(), rsAPI,
 		api.KindNew,
-		[]gomatrixserverlib.HeaderedEvent{
+		[]*gomatrixserverlib.HeaderedEvent{
 			event.Headered(verRes.RoomVersion),
 		},
 		cfg.Matrix.ServerName,

federationapi/routing/missingevents.go

@@ -73,8 +73,8 @@ func GetMissingEvents(
 
 // filterEvents returns only those events with matching roomID
 func filterEvents(
-	events []gomatrixserverlib.HeaderedEvent, roomID string,
-) []gomatrixserverlib.HeaderedEvent {
+	events []*gomatrixserverlib.HeaderedEvent, roomID string,
+) []*gomatrixserverlib.HeaderedEvent {
 	ref := events[:0]
 	for _, ev := range events {
 		if ev.RoomID() == roomID {

federationapi/routing/send.go

@@ -111,7 +111,8 @@ type txnReq struct {
 	// which the roomserver is unaware of.
 	haveEvents map[string]*gomatrixserverlib.HeaderedEvent
 	// new events which the roomserver does not know about
-	newEvents map[string]bool
+	newEvents      map[string]bool
+	newEventsMutex sync.RWMutex
 }
 
 // A subset of FederationClient functionality that txn requires. Useful for testing.
@@ -128,7 +129,7 @@ type txnFederationClient interface {
 func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.RespSend, *util.JSONResponse) {
 	results := make(map[string]gomatrixserverlib.PDUResult)
 
-	pdus := []gomatrixserverlib.HeaderedEvent{}
+	pdus := []*gomatrixserverlib.HeaderedEvent{}
 	for _, pdu := range t.PDUs {
 		var header struct {
 			RoomID string `json:"room_id"`
@@ -171,7 +172,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res
 			}
 			continue
 		}
-		if err = gomatrixserverlib.VerifyAllEventSignatures(ctx, []gomatrixserverlib.Event{event}, t.keys); err != nil {
+		if err = gomatrixserverlib.VerifyAllEventSignatures(ctx, []*gomatrixserverlib.Event{event}, t.keys); err != nil {
 			util.GetLogger(ctx).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID())
 			results[event.EventID()] = gomatrixserverlib.PDUResult{
 				Error: err.Error(),
@@ -264,6 +265,8 @@ func (e missingPrevEventsError) Error() string {
 }
 
 func (t *txnReq) haveEventIDs() map[string]bool {
+	t.newEventsMutex.RLock()
+	defer t.newEventsMutex.RUnlock()
 	result := make(map[string]bool, len(t.haveEvents))
 	for eventID := range t.haveEvents {
 		if t.newEvents[eventID] {
@@ -322,12 +325,69 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			}
 		case gomatrixserverlib.MDeviceListUpdate:
 			t.processDeviceListUpdate(ctx, e)
+		case gomatrixserverlib.MReceipt:
+			// https://matrix.org/docs/spec/server_server/r0.1.4#receipts
+			payload := map[string]eduserverAPI.FederationReceiptMRead{}
+
+			if err := json.Unmarshal(e.Content, &payload); err != nil {
+				util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal receipt event")
+				continue
+			}
+
+			for roomID, receipt := range payload {
+				for userID, mread := range receipt.User {
+					_, domain, err := gomatrixserverlib.SplitID('@', userID)
+					if err != nil {
+						util.GetLogger(ctx).WithError(err).Error("Failed to split domain from receipt event sender")
+						continue
+					}
+					if t.Origin != domain {
+						util.GetLogger(ctx).Warnf("Dropping receipt event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin)
+						continue
+					}
+					if err := t.processReceiptEvent(ctx, userID, roomID, "m.read", mread.Data.TS, mread.EventIDs); err != nil {
+						util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
+							"sender":  t.Origin,
+							"user_id": userID,
+							"room_id": roomID,
+							"events":  mread.EventIDs,
+						}).Error("Failed to send receipt event to edu server")
+						continue
+					}
+				}
+			}
 		default:
 			util.GetLogger(ctx).WithField("type", e.Type).Debug("Unhandled EDU")
 		}
 	}
 }
 
+// processReceiptEvent sends receipt events to the edu server
+func (t *txnReq) processReceiptEvent(ctx context.Context,
+	userID, roomID, receiptType string,
+	timestamp gomatrixserverlib.Timestamp,
+	eventIDs []string,
+) error {
+	// store every event
+	for _, eventID := range eventIDs {
+		req := eduserverAPI.InputReceiptEventRequest{
+			InputReceiptEvent: eduserverAPI.InputReceiptEvent{
+				UserID:    userID,
+				RoomID:    roomID,
+				EventID:   eventID,
+				Type:      receiptType,
+				Timestamp: timestamp,
+			},
+		}
+		resp := eduserverAPI.InputReceiptEventResponse{}
+		if err := t.eduAPI.InputReceiptEvent(ctx, &req, &resp); err != nil {
+			return fmt.Errorf("unable to set receipt event: %w", err)
+		}
+	}
+
+	return nil
+}
+
 func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverlib.EDU) {
 	var payload gomatrixserverlib.DeviceListUpdateEvent
 	if err := json.Unmarshal(e.Content, &payload); err != nil {
@@ -356,7 +416,7 @@ func (t *txnReq) getServers(ctx context.Context, roomID string) []gomatrixserver
 	return servers
 }
 
-func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) error {
+func (t *txnReq) processEvent(ctx context.Context, e *gomatrixserverlib.Event) error {
 	logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID())
 
 	// Work out if the roomserver knows everything it needs to know to auth
@@ -404,7 +464,7 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) er
 		context.Background(),
 		t.rsAPI,
 		api.KindNew,
-		[]gomatrixserverlib.HeaderedEvent{
+		[]*gomatrixserverlib.HeaderedEvent{
 			e.Headered(stateResp.RoomVersion),
 		},
 		api.DoNotSendToOtherServers,
@@ -413,7 +473,7 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) er
 }
 
 func (t *txnReq) retrieveMissingAuthEvents(
-	ctx context.Context, e gomatrixserverlib.Event, stateResp *api.QueryMissingAuthPrevEventsResponse,
+	ctx context.Context, e *gomatrixserverlib.Event, stateResp *api.QueryMissingAuthPrevEventsResponse,
 ) error {
 	logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID())
 
@@ -466,10 +526,10 @@ withNextEvent:
 	return nil
 }
 
-func checkAllowedByState(e gomatrixserverlib.Event, stateEvents []gomatrixserverlib.Event) error {
+func checkAllowedByState(e *gomatrixserverlib.Event, stateEvents []*gomatrixserverlib.Event) error {
 	authUsingState := gomatrixserverlib.NewAuthEvents(nil)
 	for i := range stateEvents {
-		err := authUsingState.AddEvent(&stateEvents[i])
+		err := authUsingState.AddEvent(stateEvents[i])
 		if err != nil {
 			return err
 		}
@@ -478,7 +538,7 @@ func checkAllowedByState(e gomatrixserverlib.Event, stateEvents []gomatrixserver
 }
 
 // nolint:gocyclo
-func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) error {
+func (t *txnReq) processEventWithMissingState(ctx context.Context, e *gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) error {
 	// Do this with a fresh context, so that we keep working even if the
 	// original request times out. With any luck, by the time the remote
 	// side retries, we'll have fetched the missing state.
@@ -512,7 +572,7 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser
 		return nil
 	}
 
-	backwardsExtremity := &newEvents[0]
+	backwardsExtremity := newEvents[0]
 	newEvents = newEvents[1:]
 
 	type respState struct {
@@ -600,7 +660,7 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser
 	// than the backward extremity, into the roomserver without state. This way
 	// they will automatically fast-forward based on the room state at the
 	// extremity in the last step.
-	headeredNewEvents := make([]gomatrixserverlib.HeaderedEvent, len(newEvents))
+	headeredNewEvents := make([]*gomatrixserverlib.HeaderedEvent, len(newEvents))
 	for i, newEvent := range newEvents {
 		headeredNewEvents[i] = newEvent.Headered(roomVersion)
 	}
@@ -677,9 +737,9 @@ func (t *txnReq) lookupStateAfterEventLocally(ctx context.Context, roomID, event
 		return nil
 	}
 	for i, ev := range res.StateEvents {
-		t.haveEvents[ev.EventID()] = &res.StateEvents[i]
+		t.haveEvents[ev.EventID()] = res.StateEvents[i]
 	}
-	var authEvents []gomatrixserverlib.Event
+	var authEvents []*gomatrixserverlib.Event
 	missingAuthEvents := make(map[string]bool)
 	for _, ev := range res.StateEvents {
 		for _, ae := range ev.AuthEventIDs() {
@@ -707,7 +767,7 @@ func (t *txnReq) lookupStateAfterEventLocally(ctx context.Context, roomID, event
 	}
 	for i := range queryRes.Events {
 		evID := queryRes.Events[i].EventID()
-		t.haveEvents[evID] = &queryRes.Events[i]
+		t.haveEvents[evID] = queryRes.Events[i]
 		authEvents = append(authEvents, queryRes.Events[i].Unwrap())
 	}
 
@@ -730,8 +790,8 @@ func (t *txnReq) lookupStateBeforeEvent(ctx context.Context, roomVersion gomatri
 }
 
 func (t *txnReq) resolveStatesAndCheck(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, states []*gomatrixserverlib.RespState, backwardsExtremity *gomatrixserverlib.Event) (*gomatrixserverlib.RespState, error) {
-	var authEventList []gomatrixserverlib.Event
-	var stateEventList []gomatrixserverlib.Event
+	var authEventList []*gomatrixserverlib.Event
+	var stateEventList []*gomatrixserverlib.Event
 	for _, state := range states {
 		authEventList = append(authEventList, state.AuthEvents...)
 		stateEventList = append(stateEventList, state.StateEvents...)
@@ -742,7 +802,7 @@ func (t *txnReq) resolveStatesAndCheck(ctx context.Context, roomVersion gomatrix
 	}
 	// apply the current event
 retryAllowedState:
-	if err = checkAllowedByState(*backwardsExtremity, resolvedStateEvents); err != nil {
+	if err = checkAllowedByState(backwardsExtremity, resolvedStateEvents); err != nil {
 		switch missing := err.(type) {
 		case gomatrixserverlib.MissingAuthEventError:
 			servers := t.getServers(ctx, backwardsExtremity.RoomID())
@@ -779,9 +839,9 @@ retryAllowedState:
 // This function recursively calls txnReq.processEvent with the missing events, which will be processed before this function returns.
 // This means that we may recursively call this function, as we spider back up prev_events.
 // nolint:gocyclo
-func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) (newEvents []gomatrixserverlib.Event, err error) {
+func (t *txnReq) getMissingEvents(ctx context.Context, e *gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) (newEvents []*gomatrixserverlib.Event, err error) {
 	logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID())
-	needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{e})
+	needed := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{e})
 	// query latest events (our trusted forward extremities)
 	req := api.QueryLatestEventsAndStateRequest{
 		RoomID:       e.RoomID(),
@@ -922,7 +982,7 @@ func (t *txnReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, even
 	}
 	for i := range queryRes.Events {
 		evID := queryRes.Events[i].EventID()
-		t.haveEvents[evID] = &queryRes.Events[i]
+		t.haveEvents[evID] = queryRes.Events[i]
 		if missing[evID] {
 			delete(missing, evID)
 		}
@@ -1059,10 +1119,10 @@ func (t *txnReq) lookupEvent(ctx context.Context, roomVersion gomatrixserverlib.
 		if err := t.rsAPI.QueryEventsByID(ctx, &queryReq, &queryRes); err != nil {
 			util.GetLogger(ctx).Warnf("Failed to query roomserver for missing event %s: %s - falling back to remote", missingEventID, err)
 		} else if len(queryRes.Events) == 1 {
-			return &queryRes.Events[0], nil
+			return queryRes.Events[0], nil
 		}
 	}
-	var event gomatrixserverlib.Event
+	var event *gomatrixserverlib.Event
 	found := false
 	for _, serverName := range servers {
 		txn, err := t.federation.GetEvent(ctx, serverName, missingEventID)
@@ -1082,11 +1142,13 @@ func (t *txnReq) lookupEvent(ctx context.Context, roomVersion gomatrixserverlib.
 		util.GetLogger(ctx).WithField("event_id", missingEventID).Warnf("Failed to get missing /event for event ID from %d server(s)", len(servers))
 		return nil, fmt.Errorf("wasn't able to find event via %d server(s)", len(servers))
 	}
-	if err := gomatrixserverlib.VerifyAllEventSignatures(ctx, []gomatrixserverlib.Event{event}, t.keys); err != nil {
+	if err := gomatrixserverlib.VerifyAllEventSignatures(ctx, []*gomatrixserverlib.Event{event}, t.keys); err != nil {
 		util.GetLogger(ctx).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID())
 		return nil, verifySigError{event.EventID(), err}
 	}
 	h := event.Headered(roomVersion)
+	t.newEventsMutex.Lock()
 	t.newEvents[h.EventID()] = true
-	return &h, nil
+	t.newEventsMutex.Unlock()
+	return h, nil
 }

federationapi/routing/send_test.go

@@ -33,8 +33,8 @@ var (
 		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":6,"event_id":"$MYSbs8m4rEbsCWXD:kaer.morhen","hashes":{"sha256":"kgbYM7v4Ud2YaBsjBTolM4ySg6rHcJNYI6nWhMSdFUA"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$gl2T9l3qm0kUbiIJ:kaer.morhen",{"sha256":"C/rD04h9wGxRdN2G/IBfrgoE1UovzLZ+uskwaKZ37/Q"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"x0UoKh968jj/F5l1/R7Ew0T6CTKuew3PLNHASNxqck/bkNe8yYQiDHXRr+kZxObeqPZZTpaF1+EI+bLU9W8GDQ"}},"type":"m.room.message"}`),
 		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":7,"event_id":"$N5x9WJkl9ClPrAEg:kaer.morhen","hashes":{"sha256":"FWM8oz4yquTunRZ67qlW2gzPDzdWfBP6RPHXhK1I/x8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$MYSbs8m4rEbsCWXD:kaer.morhen",{"sha256":"fatqgW+SE8mb2wFn3UN+drmluoD4UJ/EcSrL6Ur9q1M"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"Y+LX/xcyufoXMOIoqQBNOzy6lZfUGB1ffgXIrSugk6obMiyAsiRejHQN/pciZXsHKxMJLYRFAz4zSJoS/LGPAA"}},"type":"m.room.message"}`),
 	}
-	testEvents      = []gomatrixserverlib.HeaderedEvent{}
-	testStateEvents = make(map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent)
+	testEvents      = []*gomatrixserverlib.HeaderedEvent{}
+	testStateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent)
 )
 
 func init() {
@@ -76,6 +76,14 @@ func (p *testEDUProducer) InputSendToDeviceEvent(
 	return nil
 }
 
+func (o *testEDUProducer) InputReceiptEvent(
+	ctx context.Context,
+	request *eduAPI.InputReceiptEventRequest,
+	response *eduAPI.InputReceiptEventResponse,
+) error {
+	return nil
+}
+
 type testRoomserverAPI struct {
 	inputRoomEvents            []api.InputRoomEvent
 	queryMissingAuthPrevEvents func(*api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse
@@ -84,6 +92,10 @@ type testRoomserverAPI struct {
 	queryLatestEventsAndState  func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse
 }
 
+func (t *testRoomserverAPI) PerformForget(ctx context.Context, req *api.PerformForgetRequest, resp *api.PerformForgetResponse) error {
+	return nil
+}
+
 func (t *testRoomserverAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) {}
 
 func (t *testRoomserverAPI) InputRoomEvents(
@@ -433,7 +445,7 @@ NextPDU:
 	}
 }
 
-func fromStateTuples(tuples []gomatrixserverlib.StateKeyTuple, omitTuples []gomatrixserverlib.StateKeyTuple) (result []gomatrixserverlib.HeaderedEvent) {
+func fromStateTuples(tuples []gomatrixserverlib.StateKeyTuple, omitTuples []gomatrixserverlib.StateKeyTuple) (result []*gomatrixserverlib.HeaderedEvent) {
 NextTuple:
 	for _, t := range tuples {
 		for _, o := range omitTuples {
@@ -449,7 +461,7 @@ NextTuple:
 	return
 }
 
-func assertInputRoomEvents(t *testing.T, got []api.InputRoomEvent, want []gomatrixserverlib.HeaderedEvent) {
+func assertInputRoomEvents(t *testing.T, got []api.InputRoomEvent, want []*gomatrixserverlib.HeaderedEvent) {
 	for _, g := range got {
 		fmt.Println("GOT ", g.Event.EventID())
 	}
@@ -481,7 +493,7 @@ func TestBasicTransaction(t *testing.T) {
 	}
 	txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus)
 	mustProcessTransaction(t, txn, nil)
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]})
+	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]})
 }
 
 // The purpose of this test is to check that if the event received fails auth checks the event is still sent to the roomserver
@@ -502,7 +514,7 @@ func TestTransactionFailAuthChecks(t *testing.T) {
 	txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus)
 	mustProcessTransaction(t, txn, []string{})
 	// expect message to be sent to the roomserver
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]})
+	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]})
 }
 
 // The purpose of this test is to make sure that when an event is received for which we do not know the prev_events,
@@ -574,7 +586,7 @@ func TestTransactionFetchMissingPrevEvents(t *testing.T) {
 				t.Errorf("call to /get_missing_events wrong latest events: got %v want %v", missing.LatestEvents, inputEvent.EventID())
 			}
 			return gomatrixserverlib.RespMissingEvents{
-				Events: []gomatrixserverlib.Event{
+				Events: []*gomatrixserverlib.Event{
 					prevEvent.Unwrap(),
 				},
 			}, nil
@@ -586,7 +598,7 @@ func TestTransactionFetchMissingPrevEvents(t *testing.T) {
 	}
 	txn := mustCreateTransaction(rsAPI, cli, pdus)
 	mustProcessTransaction(t, txn, nil)
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{prevEvent, inputEvent})
+	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{prevEvent, inputEvent})
 }
 
 // The purpose of this test is to check that when there are missing prev_events and we still haven't been able to fill
@@ -641,7 +653,7 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) {
 			} else if askingForEvent == eventB.EventID() {
 				prevEventExists = haveEventB
 			}
-			var stateEvents []gomatrixserverlib.HeaderedEvent
+			var stateEvents []*gomatrixserverlib.HeaderedEvent
 			if prevEventExists {
 				stateEvents = fromStateTuples(req.StateToFetch, omitTuples)
 			}
@@ -759,7 +771,7 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) {
 			}
 			// just return event C, not event B so /state_ids logic kicks in as there will STILL be missing prev_events
 			return gomatrixserverlib.RespMissingEvents{
-				Events: []gomatrixserverlib.Event{
+				Events: []*gomatrixserverlib.Event{
 					eventC.Unwrap(),
 				},
 			}, nil
@@ -771,5 +783,5 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) {
 	}
 	txn := mustCreateTransaction(rsAPI, cli, pdus)
 	mustProcessTransaction(t, txn, nil)
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{eventB, eventC, eventD})
+	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{eventB, eventC, eventD})
 }

federationapi/routing/state.go

@@ -136,7 +136,7 @@ func getState(
 	}, nil
 }
 
-func getIDsFromEvent(events []gomatrixserverlib.Event) []string {
+func getIDsFromEvent(events []*gomatrixserverlib.Event) []string {
 	IDs := make([]string, len(events))
 	for i := range events {
 		IDs[i] = events[i].EventID()

federationapi/routing/threepid.go

@@ -65,7 +65,7 @@ func CreateInvitesFrom3PIDInvites(
 		return *reqErr
 	}
 
-	evs := []gomatrixserverlib.HeaderedEvent{}
+	evs := []*gomatrixserverlib.HeaderedEvent{}
 	for _, inv := range body.Invites {
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
@@ -84,7 +84,7 @@ func CreateInvitesFrom3PIDInvites(
 			return jsonerror.InternalServerError()
 		}
 		if event != nil {
-			evs = append(evs, (*event).Headered(verRes.RoomVersion))
+			evs = append(evs, event.Headered(verRes.RoomVersion))
 		}
 	}
 
@@ -165,7 +165,7 @@ func ExchangeThirdPartyInvite(
 
 	// Ask the requesting server to sign the newly created event so we know it
 	// acknowledged it
-	signedEvent, err := federation.SendInvite(httpReq.Context(), request.Origin(), *event)
+	signedEvent, err := federation.SendInvite(httpReq.Context(), request.Origin(), event)
 	if err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("federation.SendInvite failed")
 		return jsonerror.InternalServerError()
@@ -175,7 +175,7 @@ func ExchangeThirdPartyInvite(
 	if err = api.SendEvents(
 		httpReq.Context(), rsAPI,
 		api.KindNew,
-		[]gomatrixserverlib.HeaderedEvent{
+		[]*gomatrixserverlib.HeaderedEvent{
 			signedEvent.Event.Headered(verRes.RoomVersion),
 		},
 		cfg.Matrix.ServerName,
@@ -297,7 +297,7 @@ func buildMembershipEvent(
 	authEvents := gomatrixserverlib.NewAuthEvents(nil)
 
 	for i := range queryRes.StateEvents {
-		err = authEvents.AddEvent(&queryRes.StateEvents[i].Event)
+		err = authEvents.AddEvent(queryRes.StateEvents[i].Event)
 		if err != nil {
 			return nil, err
 		}
@@ -318,7 +318,7 @@ func buildMembershipEvent(
 		cfg.Matrix.PrivateKey, queryRes.RoomVersion,
 	)
 
-	return &event, err
+	return event, err
 }
 
 // sendToRemoteServer uses federation to send an invite provided by an identity

federationsender/api/api.go

@@ -48,6 +48,7 @@ type FederationSenderInternalAPI interface {
 	// Query the server names of the joined hosts in a room.
 	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
 	// containing only the server names (without information for membership events).
+	// The response will include this server if they are joined to the room.
 	QueryJoinedHostServerNamesInRoom(
 		ctx context.Context,
 		request *QueryJoinedHostServerNamesInRoomRequest,
@@ -110,6 +111,7 @@ type PerformJoinRequest struct {
 }
 
 type PerformJoinResponse struct {
+	JoinedVia gomatrixserverlib.ServerName
 	LastError *gomatrix.HTTPError
 }
 
@@ -134,12 +136,12 @@ type PerformLeaveResponse struct {
 
 type PerformInviteRequest struct {
 	RoomVersion     gomatrixserverlib.RoomVersion             `json:"room_version"`
-	Event           gomatrixserverlib.HeaderedEvent           `json:"event"`
+	Event           *gomatrixserverlib.HeaderedEvent          `json:"event"`
 	InviteRoomState []gomatrixserverlib.InviteV2StrippedState `json:"invite_room_state"`
 }
 
 type PerformInviteResponse struct {
-	Event gomatrixserverlib.HeaderedEvent `json:"event"`
+	Event *gomatrixserverlib.HeaderedEvent `json:"event"`
 }
 
 type PerformServersAliveRequest struct {

federationsender/consumers/eduserver.go

@@ -34,6 +34,7 @@ import (
 type OutputEDUConsumer struct {
 	typingConsumer       *internal.ContinualConsumer
 	sendToDeviceConsumer *internal.ContinualConsumer
+	receiptConsumer      *internal.ContinualConsumer
 	db                   storage.Database
 	queues               *queue.OutgoingQueues
 	ServerName           gomatrixserverlib.ServerName
@@ -51,24 +52,31 @@ func NewOutputEDUConsumer(
 	c := &OutputEDUConsumer{
 		typingConsumer: &internal.ContinualConsumer{
 			ComponentName:  "eduserver/typing",
-			Topic:          string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent)),
+			Topic:          cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent),
 			Consumer:       kafkaConsumer,
 			PartitionStore: store,
 		},
 		sendToDeviceConsumer: &internal.ContinualConsumer{
 			ComponentName:  "eduserver/sendtodevice",
-			Topic:          string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent)),
+			Topic:          cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent),
+			Consumer:       kafkaConsumer,
+			PartitionStore: store,
+		},
+		receiptConsumer: &internal.ContinualConsumer{
+			ComponentName:  "eduserver/receipt",
+			Topic:          cfg.Matrix.Kafka.TopicFor(config.TopicOutputReceiptEvent),
 			Consumer:       kafkaConsumer,
 			PartitionStore: store,
 		},
 		queues:            queues,
 		db:                store,
 		ServerName:        cfg.Matrix.ServerName,
-		TypingTopic:       string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent)),
-		SendToDeviceTopic: string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent)),
+		TypingTopic:       cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent),
+		SendToDeviceTopic: cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent),
 	}
 	c.typingConsumer.ProcessMessage = c.onTypingEvent
 	c.sendToDeviceConsumer.ProcessMessage = c.onSendToDeviceEvent
+	c.receiptConsumer.ProcessMessage = c.onReceiptEvent
 
 	return c
 }
@@ -81,6 +89,9 @@ func (t *OutputEDUConsumer) Start() error {
 	if err := t.sendToDeviceConsumer.Start(); err != nil {
 		return fmt.Errorf("t.sendToDeviceConsumer.Start: %w", err)
 	}
+	if err := t.receiptConsumer.Start(); err != nil {
+		return fmt.Errorf("t.receiptConsumer.Start: %w", err)
+	}
 	return nil
 }
 
@@ -177,3 +188,58 @@ func (t *OutputEDUConsumer) onTypingEvent(msg *sarama.ConsumerMessage) error {
 
 	return t.queues.SendEDU(edu, t.ServerName, names)
 }
+
+// onReceiptEvent is called in response to a message received on the receipt
+// events topic from the EDU server.
+func (t *OutputEDUConsumer) onReceiptEvent(msg *sarama.ConsumerMessage) error {
+	// Extract the typing event from msg.
+	var receipt api.OutputReceiptEvent
+	if err := json.Unmarshal(msg.Value, &receipt); err != nil {
+		// Skip this msg but continue processing messages.
+		log.WithError(err).Errorf("eduserver output log: message parse failed (expected receipt)")
+		return nil
+	}
+
+	// only send receipt events which originated from us
+	_, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID)
+	if err != nil {
+		log.WithError(err).WithField("user_id", receipt.UserID).Error("Failed to extract domain from receipt sender")
+		return nil
+	}
+	if receiptServerName != t.ServerName {
+		log.WithField("other_server", receiptServerName).Info("Suppressing receipt notif: originated elsewhere")
+		return nil
+	}
+
+	joined, err := t.db.GetJoinedHosts(context.TODO(), receipt.RoomID)
+	if err != nil {
+		return err
+	}
+
+	names := make([]gomatrixserverlib.ServerName, len(joined))
+	for i := range joined {
+		names[i] = joined[i].ServerName
+	}
+
+	content := map[string]api.FederationReceiptMRead{}
+	content[receipt.RoomID] = api.FederationReceiptMRead{
+		User: map[string]api.FederationReceiptData{
+			receipt.UserID: {
+				Data: api.ReceiptTS{
+					TS: receipt.Timestamp,
+				},
+				EventIDs: []string{receipt.EventID},
+			},
+		},
+	}
+
+	edu := &gomatrixserverlib.EDU{
+		Type:   gomatrixserverlib.MReceipt,
+		Origin: string(t.ServerName),
+	}
+	if edu.Content, err = json.Marshal(content); err != nil {
+		return err
+	}
+
+	return t.queues.SendEDU(edu, t.ServerName, names)
+}

federationsender/consumers/roomserver.go

@@ -85,7 +85,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 
 	switch output.Type {
 	case api.OutputTypeNewRoomEvent:
-		ev := &output.NewRoomEvent.Event
+		ev := output.NewRoomEvent.Event
 
 		if output.NewRoomEvent.RewritesState {
 			if err := s.db.PurgeRoomState(context.TODO(), ev.RoomID()); err != nil {
@@ -187,7 +187,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err
 
 	// Send the event.
 	return s.queues.SendEvent(
-		&ore.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent,
+		ore.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent,
 	)
 }
 
@@ -264,7 +264,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 // joinedHostsFromEvents turns a list of state events into a list of joined hosts.
 // This errors if one of the events was invalid.
 // It should be impossible for an invalid event to get this far in the pipeline.
-func joinedHostsFromEvents(evs []gomatrixserverlib.Event) ([]types.JoinedHost, error) {
+func joinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost, error) {
 	var joinedHosts []types.JoinedHost
 	for _, ev := range evs {
 		if ev.Type() != "m.room.member" || ev.StateKey() == nil {
@@ -329,8 +329,8 @@ func combineDeltas(adds1, removes1, adds2, removes2 []string) (adds, removes []s
 
 // lookupStateEvents looks up the state events that are added by a new event.
 func (s *OutputRoomEventConsumer) lookupStateEvents(
-	addsStateEventIDs []string, event gomatrixserverlib.Event,
-) ([]gomatrixserverlib.Event, error) {
+	addsStateEventIDs []string, event *gomatrixserverlib.Event,
+) ([]*gomatrixserverlib.Event, error) {
 	// Fast path if there aren't any new state events.
 	if len(addsStateEventIDs) == 0 {
 		return nil, nil
@@ -338,11 +338,11 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 
 	// Fast path if the only state event added is the event itself.
 	if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() {
-		return []gomatrixserverlib.Event{event}, nil
+		return []*gomatrixserverlib.Event{event}, nil
 	}
 
 	missing := addsStateEventIDs
-	var result []gomatrixserverlib.Event
+	var result []*gomatrixserverlib.Event
 
 	// Check if event itself is being added.
 	for _, eventID := range missing {
@@ -381,7 +381,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 	return result, nil
 }
 
-func missingEventsFrom(events []gomatrixserverlib.Event, required []string) []string {
+func missingEventsFrom(events []*gomatrixserverlib.Event, required []string) []string {
 	have := map[string]bool{}
 	for _, event := range events {
 		have[event.EventID()] = true

federationsender/internal/perform.go

@@ -105,6 +105,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 		}
 
 		// We're all good.
+		response.JoinedVia = serverName
 		return
 	}
 
@@ -534,7 +535,7 @@ func (r *FederationSenderInternalAPI) PerformInvite(
 		"destination":  destination,
 	}).Info("Sending invite")
 
-	inviteReq, err := gomatrixserverlib.NewInviteV2Request(&request.Event, request.InviteRoomState)
+	inviteReq, err := gomatrixserverlib.NewInviteV2Request(request.Event, request.InviteRoomState)
 	if err != nil {
 		return fmt.Errorf("gomatrixserverlib.NewInviteV2Request: %w", err)
 	}

federationsender/internal/perform/join.go

@@ -40,19 +40,19 @@ func JoinContext(f *gomatrixserverlib.FederationClient, k *gomatrixserverlib.Key
 // and that the join is allowed by the supplied state.
 func (r joinContext) CheckSendJoinResponse(
 	ctx context.Context,
-	event gomatrixserverlib.Event,
+	event *gomatrixserverlib.Event,
 	server gomatrixserverlib.ServerName,
 	respSendJoin gomatrixserverlib.RespSendJoin,
 ) (*gomatrixserverlib.RespState, error) {
 	// A list of events that we have retried, if they were not included in
 	// the auth events supplied in the send_join.
-	retries := map[string][]gomatrixserverlib.Event{}
+	retries := map[string][]*gomatrixserverlib.Event{}
 
 	// Define a function which we can pass to Check to retrieve missing
 	// auth events inline. This greatly increases our chances of not having
 	// to repeat the entire set of checks just for a missing event or two.
-	missingAuth := func(roomVersion gomatrixserverlib.RoomVersion, eventIDs []string) ([]gomatrixserverlib.Event, error) {
-		returning := []gomatrixserverlib.Event{}
+	missingAuth := func(roomVersion gomatrixserverlib.RoomVersion, eventIDs []string) ([]*gomatrixserverlib.Event, error) {
+		returning := []*gomatrixserverlib.Event{}
 
 		// See if we have retry entries for each of the supplied event IDs.
 		for _, eventID := range eventIDs {
@@ -88,7 +88,7 @@ func (r joinContext) CheckSendJoinResponse(
 				}
 
 				// Check the signatures of the event.
-				if res, err := gomatrixserverlib.VerifyEventSignatures(ctx, []gomatrixserverlib.Event{ev}, r.keyRing); err != nil {
+				if res, err := gomatrixserverlib.VerifyEventSignatures(ctx, []*gomatrixserverlib.Event{ev}, r.keyRing); err != nil {
 					return nil, fmt.Errorf("missingAuth VerifyEventSignatures: %w", err)
 				} else {
 					for _, err := range res {

federationsender/internal/query.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI
@@ -13,17 +12,11 @@ func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom(
 	request *api.QueryJoinedHostServerNamesInRoomRequest,
 	response *api.QueryJoinedHostServerNamesInRoomResponse,
 ) (err error) {
-	joinedHosts, err := f.db.GetJoinedHosts(ctx, request.RoomID)
+	joinedHosts, err := f.db.GetJoinedHostsForRooms(ctx, []string{request.RoomID})
 	if err != nil {
 		return
 	}
-
-	response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(joinedHosts))
-	for _, host := range joinedHosts {
-		response.ServerNames = append(response.ServerNames, host.ServerName)
-	}
-
-	// TODO: remove duplicates?
+	response.ServerNames = joinedHosts
 
 	return
 }

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20201116151724-6e7b24e4956c
 	github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.2
@@ -33,7 +33,7 @@ require (
 	github.com/pressly/goose v2.7.0-rc5+incompatible
 	github.com/prometheus/client_golang v1.7.1
 	github.com/sirupsen/logrus v1.6.0
-	github.com/tidwall/gjson v1.6.1
+	github.com/tidwall/gjson v1.6.3
 	github.com/tidwall/sjson v1.1.1
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
 	github.com/uber/jaeger-lib v2.2.0+incompatible

go.sum

@@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7 h1:YPuewGCKaJh08NslYAhyGiLw2tg6ew9LtkW7Xr+4uTU=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201116151724-6e7b24e4956c h1:iiloytJig9EmlKwuSulIbNvoPz1BFZ1QdyPWpuy85XM=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201116151724-6e7b24e4956c/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
@@ -812,8 +812,8 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc=
 github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
-github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
-github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
+github.com/tidwall/gjson v1.6.3 h1:aHoiiem0dr7GHkW001T1SMTJ7X5PvyekH5WX0whWGnI=
+github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
 github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
 github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

internal/caching/cache_roomservernids.go

@@ -1,6 +1,8 @@
 package caching
 
 import (
+	"strconv"
+
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
 
@@ -83,11 +85,11 @@ func (c Caches) GetRoomServerRoomNID(roomID string) (types.RoomNID, bool) {
 
 func (c Caches) StoreRoomServerRoomNID(roomID string, roomNID types.RoomNID) {
 	c.RoomServerRoomNIDs.Set(roomID, roomNID)
-	c.RoomServerRoomIDs.Set(string(roomNID), roomID)
+	c.RoomServerRoomIDs.Set(strconv.Itoa(int(roomNID)), roomID)
 }
 
 func (c Caches) GetRoomServerRoomID(roomNID types.RoomNID) (string, bool) {
-	val, found := c.RoomServerRoomIDs.Get(string(roomNID))
+	val, found := c.RoomServerRoomIDs.Get(strconv.Itoa(int(roomNID)))
 	if found && val != nil {
 		if roomID, ok := val.(string); ok {
 			return roomID, true

internal/config/config.go

@@ -66,6 +66,8 @@ type Dendrite struct {
 	SyncAPI          SyncAPI          `yaml:"sync_api"`
 	UserAPI          UserAPI          `yaml:"user_api"`
 
+	MSCs MSCs `yaml:"mscs"`
+
 	// The config for tracing the dendrite servers.
 	Tracing struct {
 		// Set to true to enable tracer hooks. If false, no tracing is set up.
@@ -306,6 +308,7 @@ func (c *Dendrite) Defaults() {
 	c.SyncAPI.Defaults()
 	c.UserAPI.Defaults()
 	c.AppServiceAPI.Defaults()
+	c.MSCs.Defaults()
 
 	c.Wiring()
 }
@@ -319,7 +322,7 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) {
 		&c.EDUServer, &c.FederationAPI, &c.FederationSender,
 		&c.KeyServer, &c.MediaAPI, &c.RoomServer,
 		&c.SigningKeyServer, &c.SyncAPI, &c.UserAPI,
-		&c.AppServiceAPI,
+		&c.AppServiceAPI, &c.MSCs,
 	} {
 		c.Verify(configErrs, isMonolith)
 	}
@@ -337,6 +340,7 @@ func (c *Dendrite) Wiring() {
 	c.SyncAPI.Matrix = &c.Global
 	c.UserAPI.Matrix = &c.Global
 	c.AppServiceAPI.Matrix = &c.Global
+	c.MSCs.Matrix = &c.Global
 
 	c.ClientAPI.Derived = &c.Derived
 	c.AppServiceAPI.Derived = &c.Derived

internal/config/config_kafka.go

@@ -9,6 +9,7 @@ const (
 	TopicOutputKeyChangeEvent    = "OutputKeyChangeEvent"
 	TopicOutputRoomEvent         = "OutputRoomEvent"
 	TopicOutputClientData        = "OutputClientData"
+	TopicOutputReceiptEvent      = "OutputReceiptEvent"
 )
 
 type Kafka struct {
@@ -24,6 +25,9 @@ type Kafka struct {
 	UseNaffka bool `yaml:"use_naffka"`
 	// The Naffka database is used internally by the naffka library, if used.
 	Database DatabaseOptions `yaml:"naffka_database"`
+	// The max size a Kafka message passed between consumer/producer can have
+	// Equals roughly max.message.bytes / fetch.message.max.bytes in Kafka
+	MaxMessageBytes *int `yaml:"max_message_bytes"`
 }
 
 func (k *Kafka) TopicFor(name string) string {
@@ -36,6 +40,9 @@ func (c *Kafka) Defaults() {
 	c.Addresses = []string{"localhost:2181"}
 	c.Database.ConnectionString = DataSource("file:naffka.db")
 	c.TopicPrefix = "Dendrite"
+
+	maxBytes := 1024 * 1024 * 8 // about 8MB
+	c.MaxMessageBytes = &maxBytes
 }
 
 func (c *Kafka) Verify(configErrs *ConfigErrors, isMonolith bool) {
@@ -50,4 +57,5 @@ func (c *Kafka) Verify(configErrs *ConfigErrors, isMonolith bool) {
 		checkNotZero(configErrs, "global.kafka.addresses", int64(len(c.Addresses)))
 	}
 	checkNotEmpty(configErrs, "global.kafka.topic_prefix", string(c.TopicPrefix))
+	checkPositive(configErrs, "global.kafka.max_message_bytes", int64(*c.MaxMessageBytes))
 }

internal/config/config_mscs.go

@@ -0,0 +1,19 @@
+package config
+
+type MSCs struct {
+	Matrix *Global `yaml:"-"`
+
+	// The MSCs to enable, currently only `msc2836` is supported.
+	MSCs []string `yaml:"mscs"`
+
+	Database DatabaseOptions `yaml:"database"`
+}
+
+func (c *MSCs) Defaults() {
+	c.Database.Defaults()
+	c.Database.ConnectionString = "file:mscs.db"
+}
+
+func (c *MSCs) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "mscs.database.connection_string", string(c.Database.ConnectionString))
+}

internal/config/config_syncapi.go

@@ -7,6 +7,8 @@ type SyncAPI struct {
 	ExternalAPI ExternalAPIOptions `yaml:"external_api"`
 
 	Database DatabaseOptions `yaml:"database"`
+
+	RealIPHeader string `yaml:"real_ip_header"`
 }
 
 func (c *SyncAPI) Defaults() {

internal/eventutil/events.go

@@ -73,8 +73,7 @@ func BuildEvent(
 		return nil, err
 	}
 
-	h := event.Headered(queryRes.RoomVersion)
-	return &h, nil
+	return event.Headered(queryRes.RoomVersion), nil
 }
 
 // queryRequiredEventsForBuilder queries the roomserver for auth/prev events needed for this builder.
@@ -120,7 +119,7 @@ func addPrevEventsToEvent(
 	authEvents := gomatrixserverlib.NewAuthEvents(nil)
 
 	for i := range queryRes.StateEvents {
-		err = authEvents.AddEvent(&queryRes.StateEvents[i].Event)
+		err = authEvents.AddEvent(queryRes.StateEvents[i].Event)
 		if err != nil {
 			return fmt.Errorf("authEvents.AddEvent: %w", err)
 		}
@@ -186,5 +185,5 @@ func RedactEvent(redactionEvent, redactedEvent *gomatrixserverlib.Event) (*gomat
 	if err != nil {
 		return nil, err
 	}
-	return &r, nil
+	return r, nil
 }

internal/hooks/hooks.go

@@ -0,0 +1,74 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package hooks exposes places in Dendrite where custom code can be executed, useful for MSCs.
+// Hooks can only be run in monolith mode.
+package hooks
+
+import "sync"
+
+const (
+	// KindNewEventPersisted is a hook which is called with *gomatrixserverlib.HeaderedEvent
+	// It is run when a new event is persisted in the roomserver.
+	// Usage:
+	//   hooks.Attach(hooks.KindNewEventPersisted, func(headeredEvent interface{}) { ... })
+	KindNewEventPersisted = "new_event_persisted"
+	// KindNewEventReceived is a hook which is called with *gomatrixserverlib.HeaderedEvent
+	// It is run before a new event is processed by the roomserver. This hook can be used
+	// to modify the event before it is persisted by adding data to `unsigned`.
+	// Usage:
+	//   hooks.Attach(hooks.KindNewEventReceived, func(headeredEvent interface{}) {
+	//     ev := headeredEvent.(*gomatrixserverlib.HeaderedEvent)
+	//     _ = ev.SetUnsignedField("key", "val")
+	//   })
+	KindNewEventReceived = "new_event_received"
+)
+
+var (
+	hookMap = make(map[string][]func(interface{}))
+	hookMu  = sync.Mutex{}
+	enabled = false
+)
+
+// Enable all hooks. This may slow down the server slightly. Required for MSCs to work.
+func Enable() {
+	enabled = true
+}
+
+// Run any hooks
+func Run(kind string, data interface{}) {
+	if !enabled {
+		return
+	}
+	cbs := callbacks(kind)
+	for _, cb := range cbs {
+		cb(data)
+	}
+}
+
+// Attach a hook
+func Attach(kind string, callback func(interface{})) {
+	if !enabled {
+		return
+	}
+	hookMu.Lock()
+	defer hookMu.Unlock()
+	hookMap[kind] = append(hookMap[kind], callback)
+}
+
+func callbacks(kind string) []func(interface{}) {
+	hookMu.Lock()
+	defer hookMu.Unlock()
+	return hookMap[kind]
+}

internal/mscs/msc2836/msc2836.go

@@ -0,0 +1,530 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package msc2836 'Threading' implements https://github.com/matrix-org/matrix-doc/pull/2836
+package msc2836
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	fs "github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/internal/hooks"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/setup"
+	roomserver "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+const (
+	constRelType     = "m.reference"
+	constRoomIDKey   = "relationship_room_id"
+	constRoomServers = "relationship_servers"
+)
+
+type EventRelationshipRequest struct {
+	EventID         string `json:"event_id"`
+	MaxDepth        int    `json:"max_depth"`
+	MaxBreadth      int    `json:"max_breadth"`
+	Limit           int    `json:"limit"`
+	DepthFirst      bool   `json:"depth_first"`
+	RecentFirst     bool   `json:"recent_first"`
+	IncludeParent   bool   `json:"include_parent"`
+	IncludeChildren bool   `json:"include_children"`
+	Direction       string `json:"direction"`
+	Batch           string `json:"batch"`
+	AutoJoin        bool   `json:"auto_join"`
+}
+
+func NewEventRelationshipRequest(body io.Reader) (*EventRelationshipRequest, error) {
+	var relation EventRelationshipRequest
+	relation.Defaults()
+	if err := json.NewDecoder(body).Decode(&relation); err != nil {
+		return nil, err
+	}
+	return &relation, nil
+}
+
+func (r *EventRelationshipRequest) Defaults() {
+	r.Limit = 100
+	r.MaxBreadth = 10
+	r.MaxDepth = 3
+	r.DepthFirst = false
+	r.RecentFirst = true
+	r.IncludeParent = false
+	r.IncludeChildren = false
+	r.Direction = "down"
+}
+
+type EventRelationshipResponse struct {
+	Events    []gomatrixserverlib.ClientEvent `json:"events"`
+	NextBatch string                          `json:"next_batch"`
+	Limited   bool                            `json:"limited"`
+}
+
+// Enable this MSC
+// nolint:gocyclo
+func Enable(
+	base *setup.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI,
+	userAPI userapi.UserInternalAPI, keyRing gomatrixserverlib.JSONVerifier,
+) error {
+	db, err := NewDatabase(&base.Cfg.MSCs.Database)
+	if err != nil {
+		return fmt.Errorf("Cannot enable MSC2836: %w", err)
+	}
+	hooks.Enable()
+	hooks.Attach(hooks.KindNewEventPersisted, func(headeredEvent interface{}) {
+		he := headeredEvent.(*gomatrixserverlib.HeaderedEvent)
+		hookErr := db.StoreRelation(context.Background(), he)
+		if hookErr != nil {
+			util.GetLogger(context.Background()).WithError(hookErr).Error(
+				"failed to StoreRelation",
+			)
+		}
+	})
+	hooks.Attach(hooks.KindNewEventReceived, func(headeredEvent interface{}) {
+		he := headeredEvent.(*gomatrixserverlib.HeaderedEvent)
+		ctx := context.Background()
+		// we only inject metadata for events our server sends
+		userID := he.Sender()
+		_, domain, err := gomatrixserverlib.SplitID('@', userID)
+		if err != nil {
+			return
+		}
+		if domain != base.Cfg.Global.ServerName {
+			return
+		}
+		// if this event has an m.relationship, add on the room_id and servers to unsigned
+		parent, child, relType := parentChildEventIDs(he)
+		if parent == "" || child == "" || relType == "" {
+			return
+		}
+		event, joinedToRoom := getEventIfVisible(ctx, rsAPI, parent, userID)
+		if !joinedToRoom {
+			return
+		}
+		err = he.SetUnsignedField(constRoomIDKey, event.RoomID())
+		if err != nil {
+			util.GetLogger(context.Background()).WithError(err).Warn("Failed to SetUnsignedField")
+			return
+		}
+
+		var servers []gomatrixserverlib.ServerName
+		if fsAPI != nil {
+			var res fs.QueryJoinedHostServerNamesInRoomResponse
+			err = fsAPI.QueryJoinedHostServerNamesInRoom(ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{
+				RoomID: event.RoomID(),
+			}, &res)
+			if err != nil {
+				util.GetLogger(context.Background()).WithError(err).Warn("Failed to QueryJoinedHostServerNamesInRoom")
+				return
+			}
+			servers = res.ServerNames
+		} else {
+			servers = []gomatrixserverlib.ServerName{
+				base.Cfg.Global.ServerName,
+			}
+		}
+		err = he.SetUnsignedField(constRoomServers, servers)
+		if err != nil {
+			util.GetLogger(context.Background()).WithError(err).Warn("Failed to SetUnsignedField")
+			return
+		}
+	})
+
+	base.PublicClientAPIMux.Handle("/unstable/event_relationships",
+		httputil.MakeAuthAPI("eventRelationships", userAPI, eventRelationshipHandler(db, rsAPI)),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	base.PublicFederationAPIMux.Handle("/unstable/event_relationships", httputil.MakeExternalAPI(
+		"msc2836_event_relationships", func(req *http.Request) util.JSONResponse {
+			fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
+				req, time.Now(), base.Cfg.Global.ServerName, keyRing,
+			)
+			if fedReq == nil {
+				return errResp
+			}
+			return federatedEventRelationship(req.Context(), fedReq, db, rsAPI)
+		},
+	)).Methods(http.MethodPost, http.MethodOptions)
+	return nil
+}
+
+type reqCtx struct {
+	ctx                context.Context
+	rsAPI              roomserver.RoomserverInternalAPI
+	db                 Database
+	req                *EventRelationshipRequest
+	userID             string
+	isFederatedRequest bool
+}
+
+func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAPI) func(*http.Request, *userapi.Device) util.JSONResponse {
+	return func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		relation, err := NewEventRelationshipRequest(req.Body)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("failed to decode HTTP request as JSON")
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.BadJSON(fmt.Sprintf("invalid json: %s", err)),
+			}
+		}
+		rc := reqCtx{
+			ctx:                req.Context(),
+			req:                relation,
+			userID:             device.UserID,
+			rsAPI:              rsAPI,
+			isFederatedRequest: false,
+			db:                 db,
+		}
+		res, resErr := rc.process()
+		if resErr != nil {
+			return *resErr
+		}
+
+		return util.JSONResponse{
+			Code: 200,
+			JSON: res,
+		}
+	}
+}
+
+func federatedEventRelationship(ctx context.Context, fedReq *gomatrixserverlib.FederationRequest, db Database, rsAPI roomserver.RoomserverInternalAPI) util.JSONResponse {
+	relation, err := NewEventRelationshipRequest(bytes.NewBuffer(fedReq.Content()))
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("failed to decode HTTP request as JSON")
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON(fmt.Sprintf("invalid json: %s", err)),
+		}
+	}
+	rc := reqCtx{
+		ctx:                ctx,
+		req:                relation,
+		userID:             "",
+		rsAPI:              rsAPI,
+		isFederatedRequest: true,
+		db:                 db,
+	}
+	res, resErr := rc.process()
+	if resErr != nil {
+		return *resErr
+	}
+
+	return util.JSONResponse{
+		Code: 200,
+		JSON: res,
+	}
+}
+
+func (rc *reqCtx) process() (*EventRelationshipResponse, *util.JSONResponse) {
+	var res EventRelationshipResponse
+	var returnEvents []*gomatrixserverlib.HeaderedEvent
+	// Can the user see (according to history visibility) event_id? If no, reject the request, else continue.
+	// We should have the event being referenced so don't give any claimed room ID / servers
+	event := rc.getEventIfVisible(rc.req.EventID, "", nil)
+	if event == nil {
+		return nil, &util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Forbidden("Event does not exist or you are not authorised to see it"),
+		}
+	}
+
+	// Retrieve the event. Add it to response array.
+	returnEvents = append(returnEvents, event)
+
+	if rc.req.IncludeParent {
+		if parentEvent := rc.includeParent(event); parentEvent != nil {
+			returnEvents = append(returnEvents, parentEvent)
+		}
+	}
+
+	if rc.req.IncludeChildren {
+		remaining := rc.req.Limit - len(returnEvents)
+		if remaining > 0 {
+			children, resErr := rc.includeChildren(rc.db, event.EventID(), remaining, rc.req.RecentFirst)
+			if resErr != nil {
+				return nil, resErr
+			}
+			returnEvents = append(returnEvents, children...)
+		}
+	}
+
+	remaining := rc.req.Limit - len(returnEvents)
+	var walkLimited bool
+	if remaining > 0 {
+		included := make(map[string]bool, len(returnEvents))
+		for _, ev := range returnEvents {
+			included[ev.EventID()] = true
+		}
+		var events []*gomatrixserverlib.HeaderedEvent
+		events, walkLimited = walkThread(
+			rc.ctx, rc.db, rc, included, remaining,
+		)
+		returnEvents = append(returnEvents, events...)
+	}
+	res.Events = make([]gomatrixserverlib.ClientEvent, len(returnEvents))
+	for i, ev := range returnEvents {
+		res.Events[i] = gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll)
+	}
+	res.Limited = remaining == 0 || walkLimited
+	return &res, nil
+}
+
+// If include_parent: true and there is a valid m.relationship field in the event,
+// retrieve the referenced event. Apply history visibility check to that event and if it passes, add it to the response array.
+func (rc *reqCtx) includeParent(event *gomatrixserverlib.HeaderedEvent) (parent *gomatrixserverlib.HeaderedEvent) {
+	parentID, _, _ := parentChildEventIDs(event)
+	if parentID == "" {
+		return nil
+	}
+	claimedRoomID, claimedServers := roomIDAndServers(event)
+	return rc.getEventIfVisible(parentID, claimedRoomID, claimedServers)
+}
+
+// If include_children: true, lookup all events which have event_id as an m.relationship
+// Apply history visibility checks to all these events and add the ones which pass into the response array,
+// honouring the recent_first flag and the limit.
+func (rc *reqCtx) includeChildren(db Database, parentID string, limit int, recentFirst bool) ([]*gomatrixserverlib.HeaderedEvent, *util.JSONResponse) {
+	children, err := db.ChildrenForParent(rc.ctx, parentID, constRelType, recentFirst)
+	if err != nil {
+		util.GetLogger(rc.ctx).WithError(err).Error("failed to get ChildrenForParent")
+		resErr := jsonerror.InternalServerError()
+		return nil, &resErr
+	}
+	var childEvents []*gomatrixserverlib.HeaderedEvent
+	for _, child := range children {
+		// in order for us to even know about the children the server must be joined to those rooms, hence pass no claimed room ID or servers.
+		childEvent := rc.getEventIfVisible(child.EventID, "", nil)
+		if childEvent != nil {
+			childEvents = append(childEvents, childEvent)
+		}
+	}
+	if len(childEvents) > limit {
+		return childEvents[:limit], nil
+	}
+	return childEvents, nil
+}
+
+// Begin to walk the thread DAG in the direction specified, either depth or breadth first according to the depth_first flag,
+// honouring the limit, max_depth and max_breadth values according to the following rules
+// nolint: unparam
+func walkThread(
+	ctx context.Context, db Database, rc *reqCtx, included map[string]bool, limit int,
+) ([]*gomatrixserverlib.HeaderedEvent, bool) {
+	if rc.req.Direction != "down" {
+		util.GetLogger(ctx).Error("not implemented: direction=up")
+		return nil, false
+	}
+	var result []*gomatrixserverlib.HeaderedEvent
+	eventWalker := walker{
+		ctx: ctx,
+		req: rc.req,
+		db:  db,
+		fn: func(wi *walkInfo) bool {
+			// If already processed event, skip.
+			if included[wi.EventID] {
+				return false
+			}
+
+			// If the response array is >= limit, stop.
+			if len(result) >= limit {
+				return true
+			}
+
+			// Process the event.
+			// TODO: Include edge information: room ID and servers
+			event := rc.getEventIfVisible(wi.EventID, "", nil)
+			if event != nil {
+				result = append(result, event)
+			}
+			included[wi.EventID] = true
+			return false
+		},
+	}
+	limited, err := eventWalker.WalkFrom(rc.req.EventID)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Errorf("Failed to WalkFrom %s", rc.req.EventID)
+	}
+	return result, limited
+}
+
+func (rc *reqCtx) getEventIfVisible(eventID string, claimedRoomID string, claimedServers []string) *gomatrixserverlib.HeaderedEvent {
+	event, joinedToRoom := getEventIfVisible(rc.ctx, rc.rsAPI, eventID, rc.userID)
+	if event != nil && joinedToRoom {
+		return event
+	}
+	// either we don't have the event or we aren't joined to the room, regardless we should try joining if auto join is enabled
+	if !rc.req.AutoJoin {
+		return nil
+	}
+	// if we're doing this on behalf of a random server don't auto-join rooms regardless of what the request says
+	if rc.isFederatedRequest {
+		return nil
+	}
+	roomID := claimedRoomID
+	var servers []gomatrixserverlib.ServerName
+	if event != nil {
+		roomID = event.RoomID()
+	}
+	for _, s := range claimedServers {
+		servers = append(servers, gomatrixserverlib.ServerName(s))
+	}
+	var joinRes roomserver.PerformJoinResponse
+	rc.rsAPI.PerformJoin(rc.ctx, &roomserver.PerformJoinRequest{
+		UserID:        rc.userID,
+		Content:       map[string]interface{}{},
+		RoomIDOrAlias: roomID,
+		ServerNames:   servers,
+	}, &joinRes)
+	if joinRes.Error != nil {
+		util.GetLogger(rc.ctx).WithError(joinRes.Error).WithField("room_id", roomID).Error("Failed to auto-join room")
+		return nil
+	}
+	if event != nil {
+		return event
+	}
+	// TODO: hit /event_relationships on the server we joined via
+	util.GetLogger(rc.ctx).Infof("joined room but need to fetch event TODO")
+	return nil
+}
+
+func getEventIfVisible(ctx context.Context, rsAPI roomserver.RoomserverInternalAPI, eventID, userID string) (*gomatrixserverlib.HeaderedEvent, bool) {
+	var queryEventsRes roomserver.QueryEventsByIDResponse
+	err := rsAPI.QueryEventsByID(ctx, &roomserver.QueryEventsByIDRequest{
+		EventIDs: []string{eventID},
+	}, &queryEventsRes)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("getEventIfVisible: failed to QueryEventsByID")
+		return nil, false
+	}
+	if len(queryEventsRes.Events) == 0 {
+		util.GetLogger(ctx).Infof("event does not exist")
+		return nil, false // event does not exist
+	}
+	event := queryEventsRes.Events[0]
+
+	// Allow events if the member is in the room
+	// TODO: This does not honour history_visibility
+	// TODO: This does not honour m.room.create content
+	var queryMembershipRes roomserver.QueryMembershipForUserResponse
+	err = rsAPI.QueryMembershipForUser(ctx, &roomserver.QueryMembershipForUserRequest{
+		RoomID: event.RoomID(),
+		UserID: userID,
+	}, &queryMembershipRes)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("getEventIfVisible: failed to QueryMembershipForUser")
+		return nil, false
+	}
+	return event, queryMembershipRes.IsInRoom
+}
+
+type walkInfo struct {
+	eventInfo
+	SiblingNumber int
+	Depth         int
+}
+
+type walker struct {
+	ctx context.Context
+	req *EventRelationshipRequest
+	db  Database
+	fn  func(wi *walkInfo) bool // callback invoked for each event walked, return true to terminate the walk
+}
+
+// WalkFrom the event ID given
+func (w *walker) WalkFrom(eventID string) (limited bool, err error) {
+	children, err := w.db.ChildrenForParent(w.ctx, eventID, constRelType, w.req.RecentFirst)
+	if err != nil {
+		util.GetLogger(w.ctx).WithError(err).Error("WalkFrom() ChildrenForParent failed, cannot walk")
+		return false, err
+	}
+	var next *walkInfo
+	toWalk := w.addChildren(nil, children, 1)
+	next, toWalk = w.nextChild(toWalk)
+	for next != nil {
+		stop := w.fn(next)
+		if stop {
+			return true, nil
+		}
+		// find the children's children
+		children, err = w.db.ChildrenForParent(w.ctx, next.EventID, constRelType, w.req.RecentFirst)
+		if err != nil {
+			util.GetLogger(w.ctx).WithError(err).Error("WalkFrom() ChildrenForParent failed, cannot walk")
+			return false, err
+		}
+		toWalk = w.addChildren(toWalk, children, next.Depth+1)
+		next, toWalk = w.nextChild(toWalk)
+	}
+
+	return false, nil
+}
+
+// addChildren adds an event's children to the to walk data structure
+func (w *walker) addChildren(toWalk []walkInfo, children []eventInfo, depthOfChildren int) []walkInfo {
+	// Check what number child this event is (ordered by recent_first) compared to its parent, does it exceed (greater than) max_breadth? If yes, skip.
+	if len(children) > w.req.MaxBreadth {
+		children = children[:w.req.MaxBreadth]
+	}
+	// Check how deep the event is compared to event_id, does it exceed (greater than) max_depth? If yes, skip.
+	if depthOfChildren > w.req.MaxDepth {
+		return toWalk
+	}
+
+	if w.req.DepthFirst {
+		// the slice is a stack so push them in reverse order so we pop them in the correct order
+		// e.g [3,2,1] => [3,2] , 1 => [3] , 2 => [] , 3
+		for i := len(children) - 1; i >= 0; i-- {
+			toWalk = append(toWalk, walkInfo{
+				eventInfo:     children[i],
+				SiblingNumber: i + 1, // index from 1
+				Depth:         depthOfChildren,
+			})
+		}
+	} else {
+		// the slice is a queue so push them in normal order to we dequeue them in the correct order
+		// e.g [1,2,3] => 1, [2, 3] => 2 , [3] => 3, []
+		for i := range children {
+			toWalk = append(toWalk, walkInfo{
+				eventInfo:     children[i],
+				SiblingNumber: i + 1, // index from 1
+				Depth:         depthOfChildren,
+			})
+		}
+	}
+	return toWalk
+}
+
+func (w *walker) nextChild(toWalk []walkInfo) (*walkInfo, []walkInfo) {
+	if len(toWalk) == 0 {
+		return nil, nil
+	}
+	var child walkInfo
+	if w.req.DepthFirst {
+		// toWalk is a stack so pop the child off
+		child, toWalk = toWalk[len(toWalk)-1], toWalk[:len(toWalk)-1]
+		return &child, toWalk
+	}
+	// toWalk is a queue so shift the child off
+	child, toWalk = toWalk[0], toWalk[1:]
+	return &child, toWalk
+}

internal/mscs/msc2836/msc2836_test.go

@@ -0,0 +1,577 @@
+package msc2836_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/ed25519"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/hooks"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/mscs/msc2836"
+	"github.com/matrix-org/dendrite/internal/setup"
+	roomserver "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+var (
+	client = &http.Client{
+		Timeout: 10 * time.Second,
+	}
+)
+
+// Basic sanity check of MSC2836 logic. Injects a thread that looks like:
+//   A
+//   |
+//   B
+//  / \
+// C   D
+//    /|\
+//   E F G
+//   |
+//   H
+// And makes sure POST /event_relationships works with various parameters
+func TestMSC2836(t *testing.T) {
+	alice := "@alice:localhost"
+	bob := "@bob:localhost"
+	charlie := "@charlie:localhost"
+	roomIDA := "!alice:localhost"
+	roomIDB := "!bob:localhost"
+	roomIDC := "!charlie:localhost"
+	// give access tokens to all three users
+	nopUserAPI := &testUserAPI{
+		accessTokens: make(map[string]userapi.Device),
+	}
+	nopUserAPI.accessTokens["alice"] = userapi.Device{
+		AccessToken: "alice",
+		DisplayName: "Alice",
+		UserID:      alice,
+	}
+	nopUserAPI.accessTokens["bob"] = userapi.Device{
+		AccessToken: "bob",
+		DisplayName: "Bob",
+		UserID:      bob,
+	}
+	nopUserAPI.accessTokens["charlie"] = userapi.Device{
+		AccessToken: "charlie",
+		DisplayName: "Charles",
+		UserID:      charlie,
+	}
+	eventA := mustCreateEvent(t, fledglingEvent{
+		RoomID: roomIDA,
+		Sender: alice,
+		Type:   "m.room.message",
+		Content: map[string]interface{}{
+			"body": "[A] Do you know shelties?",
+		},
+	})
+	eventB := mustCreateEvent(t, fledglingEvent{
+		RoomID: roomIDB,
+		Sender: bob,
+		Type:   "m.room.message",
+		Content: map[string]interface{}{
+			"body": "[B] I <3 shelties",
+			"m.relationship": map[string]string{
+				"rel_type": "m.reference",
+				"event_id": eventA.EventID(),
+			},
+		},
+	})
+	eventC := mustCreateEvent(t, fledglingEvent{
+		RoomID: roomIDB,
+		Sender: bob,
+		Type:   "m.room.message",
+		Content: map[string]interface{}{
+			"body": "[C] like so much",
+			"m.relationship": map[string]string{
+				"rel_type": "m.reference",
+				"event_id": eventB.EventID(),
+			},
+		},
+	})
+	eventD := mustCreateEvent(t, fledglingEvent{
+		RoomID: roomIDA,
+		Sender: alice,
+		Type:   "m.room.message",
+		Content: map[string]interface{}{
+			"body": "[D] but what are shelties???",
+			"m.relationship": map[string]string{
+				"rel_type": "m.reference",
+				"event_id": eventB.EventID(),
+			},
+		},
+	})
+	eventE := mustCreateEvent(t, fledglingEvent{
+		RoomID: roomIDB,
+		Sender: bob,
+		Type:   "m.room.message",
+		Content: map[string]interface{}{
+			"body": "[E] seriously???",
+			"m.relationship": map[string]string{
+				"rel_type": "m.reference",
+				"event_id": eventD.EventID(),
+			},
+		},
+	})
+	eventF := mustCreateEvent(t, fledglingEvent{
+		RoomID: roomIDC,
+		Sender: charlie,
+		Type:   "m.room.message",
+		Content: map[string]interface{}{
+			"body": "[F] omg how do you not know what shelties are",
+			"m.relationship": map[string]string{
+				"rel_type": "m.reference",
+				"event_id": eventD.EventID(),
+			},
+		},
+	})
+	eventG := mustCreateEvent(t, fledglingEvent{
+		RoomID: roomIDA,
+		Sender: alice,
+		Type:   "m.room.message",
+		Content: map[string]interface{}{
+			"body": "[G] looked it up, it's a sheltered person?",
+			"m.relationship": map[string]string{
+				"rel_type": "m.reference",
+				"event_id": eventD.EventID(),
+			},
+		},
+	})
+	eventH := mustCreateEvent(t, fledglingEvent{
+		RoomID: roomIDB,
+		Sender: bob,
+		Type:   "m.room.message",
+		Content: map[string]interface{}{
+			"body": "[H] it's a dog!!!!!",
+			"m.relationship": map[string]string{
+				"rel_type": "m.reference",
+				"event_id": eventE.EventID(),
+			},
+		},
+	})
+	// make everyone joined to each other's rooms
+	nopRsAPI := &testRoomserverAPI{
+		userToJoinedRooms: map[string][]string{
+			alice:   []string{roomIDA, roomIDB, roomIDC},
+			bob:     []string{roomIDA, roomIDB, roomIDC},
+			charlie: []string{roomIDA, roomIDB, roomIDC},
+		},
+		events: map[string]*gomatrixserverlib.HeaderedEvent{
+			eventA.EventID(): eventA,
+			eventB.EventID(): eventB,
+			eventC.EventID(): eventC,
+			eventD.EventID(): eventD,
+			eventE.EventID(): eventE,
+			eventF.EventID(): eventF,
+			eventG.EventID(): eventG,
+			eventH.EventID(): eventH,
+		},
+	}
+	router := injectEvents(t, nopUserAPI, nopRsAPI, []*gomatrixserverlib.HeaderedEvent{
+		eventA, eventB, eventC, eventD, eventE, eventF, eventG, eventH,
+	})
+	cancel := runServer(t, router)
+	defer cancel()
+
+	t.Run("returns 403 on invalid event IDs", func(t *testing.T) {
+		_ = postRelationships(t, 403, "alice", newReq(t, map[string]interface{}{
+			"event_id": "$invalid",
+		}))
+	})
+	t.Run("returns 403 if not joined to the room of specified event in request", func(t *testing.T) {
+		nopUserAPI.accessTokens["frank"] = userapi.Device{
+			AccessToken: "frank",
+			DisplayName: "Frank Not In Room",
+			UserID:      "@frank:localhost",
+		}
+		_ = postRelationships(t, 403, "frank", newReq(t, map[string]interface{}{
+			"event_id":       eventB.EventID(),
+			"limit":          1,
+			"include_parent": true,
+		}))
+	})
+	t.Run("omits parent if not joined to the room of parent of event", func(t *testing.T) {
+		nopUserAPI.accessTokens["frank2"] = userapi.Device{
+			AccessToken: "frank2",
+			DisplayName: "Frank2 Not In Room",
+			UserID:      "@frank2:localhost",
+		}
+		// Event B is in roomB, Event A is in roomA, so make frank2 joined to roomB
+		nopRsAPI.userToJoinedRooms["@frank2:localhost"] = []string{roomIDB}
+		body := postRelationships(t, 200, "frank2", newReq(t, map[string]interface{}{
+			"event_id":       eventB.EventID(),
+			"limit":          1,
+			"include_parent": true,
+		}))
+		assertContains(t, body, []string{eventB.EventID()})
+	})
+	t.Run("returns the parent if include_parent is true", func(t *testing.T) {
+		body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":       eventB.EventID(),
+			"include_parent": true,
+			"limit":          2,
+		}))
+		assertContains(t, body, []string{eventB.EventID(), eventA.EventID()})
+	})
+	t.Run("returns the children in the right order if include_children is true", func(t *testing.T) {
+		body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":         eventD.EventID(),
+			"include_children": true,
+			"recent_first":     true,
+			"limit":            4,
+		}))
+		assertContains(t, body, []string{eventD.EventID(), eventG.EventID(), eventF.EventID(), eventE.EventID()})
+		body = postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":         eventD.EventID(),
+			"include_children": true,
+			"recent_first":     false,
+			"limit":            4,
+		}))
+		assertContains(t, body, []string{eventD.EventID(), eventE.EventID(), eventF.EventID(), eventG.EventID()})
+	})
+	t.Run("walks the graph depth first", func(t *testing.T) {
+		body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":     eventB.EventID(),
+			"recent_first": false,
+			"depth_first":  true,
+			"limit":        6,
+		}))
+		// Oldest first so:
+		//   A
+		//   |
+		//   B1
+		//  / \
+		// C2  D3
+		//    /| \
+		//  4E 6F G
+		//   |
+		//  5H
+		assertContains(t, body, []string{eventB.EventID(), eventC.EventID(), eventD.EventID(), eventE.EventID(), eventH.EventID(), eventF.EventID()})
+		body = postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":     eventB.EventID(),
+			"recent_first": true,
+			"depth_first":  true,
+			"limit":        6,
+		}))
+		// Recent first so:
+		//   A
+		//   |
+		//   B1
+		//  / \
+		// C   D2
+		//    /| \
+		//  E5 F4 G3
+		//   |
+		//  H6
+		assertContains(t, body, []string{eventB.EventID(), eventD.EventID(), eventG.EventID(), eventF.EventID(), eventE.EventID(), eventH.EventID()})
+	})
+	t.Run("walks the graph breadth first", func(t *testing.T) {
+		body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":     eventB.EventID(),
+			"recent_first": false,
+			"depth_first":  false,
+			"limit":        6,
+		}))
+		// Oldest first so:
+		//   A
+		//   |
+		//   B1
+		//  / \
+		// C2  D3
+		//    /| \
+		//  E4 F5 G6
+		//   |
+		//   H
+		assertContains(t, body, []string{eventB.EventID(), eventC.EventID(), eventD.EventID(), eventE.EventID(), eventF.EventID(), eventG.EventID()})
+		body = postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":     eventB.EventID(),
+			"recent_first": true,
+			"depth_first":  false,
+			"limit":        6,
+		}))
+		// Recent first so:
+		//   A
+		//   |
+		//   B1
+		//  / \
+		// C3  D2
+		//    /| \
+		//  E6 F5 G4
+		//   |
+		//   H
+		assertContains(t, body, []string{eventB.EventID(), eventD.EventID(), eventC.EventID(), eventG.EventID(), eventF.EventID(), eventE.EventID()})
+	})
+	t.Run("caps via max_breadth", func(t *testing.T) {
+		body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":     eventB.EventID(),
+			"recent_first": false,
+			"depth_first":  false,
+			"max_breadth":  2,
+			"limit":        10,
+		}))
+		// Event G gets omitted because of max_breadth
+		assertContains(t, body, []string{eventB.EventID(), eventC.EventID(), eventD.EventID(), eventE.EventID(), eventF.EventID(), eventH.EventID()})
+	})
+	t.Run("caps via max_depth", func(t *testing.T) {
+		body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":     eventB.EventID(),
+			"recent_first": false,
+			"depth_first":  false,
+			"max_depth":    2,
+			"limit":        10,
+		}))
+		// Event H gets omitted because of max_depth
+		assertContains(t, body, []string{eventB.EventID(), eventC.EventID(), eventD.EventID(), eventE.EventID(), eventF.EventID(), eventG.EventID()})
+	})
+	t.Run("terminates when reaching the limit", func(t *testing.T) {
+		body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":     eventB.EventID(),
+			"recent_first": false,
+			"depth_first":  false,
+			"limit":        4,
+		}))
+		assertContains(t, body, []string{eventB.EventID(), eventC.EventID(), eventD.EventID(), eventE.EventID()})
+	})
+	t.Run("returns all events with a high enough limit", func(t *testing.T) {
+		body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{
+			"event_id":     eventB.EventID(),
+			"recent_first": false,
+			"depth_first":  false,
+			"limit":        400,
+		}))
+		assertContains(t, body, []string{eventB.EventID(), eventC.EventID(), eventD.EventID(), eventE.EventID(), eventF.EventID(), eventG.EventID(), eventH.EventID()})
+	})
+}
+
+// TODO: TestMSC2836TerminatesLoops (short and long)
+// TODO: TestMSC2836UnknownEventsSkipped
+// TODO: TestMSC2836SkipEventIfNotInRoom
+
+func newReq(t *testing.T, jsonBody map[string]interface{}) *msc2836.EventRelationshipRequest {
+	t.Helper()
+	b, err := json.Marshal(jsonBody)
+	if err != nil {
+		t.Fatalf("Failed to marshal request: %s", err)
+	}
+	r, err := msc2836.NewEventRelationshipRequest(bytes.NewBuffer(b))
+	if err != nil {
+		t.Fatalf("Failed to NewEventRelationshipRequest: %s", err)
+	}
+	return r
+}
+
+func runServer(t *testing.T, router *mux.Router) func() {
+	t.Helper()
+	externalServ := &http.Server{
+		Addr:         string(":8009"),
+		WriteTimeout: 60 * time.Second,
+		Handler:      router,
+	}
+	go func() {
+		externalServ.ListenAndServe()
+	}()
+	// wait to listen on the port
+	time.Sleep(500 * time.Millisecond)
+	return func() {
+		externalServ.Shutdown(context.TODO())
+	}
+}
+
+func postRelationships(t *testing.T, expectCode int, accessToken string, req *msc2836.EventRelationshipRequest) *msc2836.EventRelationshipResponse {
+	t.Helper()
+	var r msc2836.EventRelationshipRequest
+	r.Defaults()
+	data, err := json.Marshal(req)
+	if err != nil {
+		t.Fatalf("failed to marshal request: %s", err)
+	}
+	httpReq, err := http.NewRequest(
+		"POST", "http://localhost:8009/_matrix/client/unstable/event_relationships",
+		bytes.NewBuffer(data),
+	)
+	httpReq.Header.Set("Authorization", "Bearer "+accessToken)
+	if err != nil {
+		t.Fatalf("failed to prepare request: %s", err)
+	}
+	res, err := client.Do(httpReq)
+	if err != nil {
+		t.Fatalf("failed to do request: %s", err)
+	}
+	if res.StatusCode != expectCode {
+		body, _ := ioutil.ReadAll(res.Body)
+		t.Fatalf("wrong response code, got %d want %d - body: %s", res.StatusCode, expectCode, string(body))
+	}
+	if res.StatusCode == 200 {
+		var result msc2836.EventRelationshipResponse
+		if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
+			t.Fatalf("response 200 OK but failed to deserialise JSON : %s", err)
+		}
+		return &result
+	}
+	return nil
+}
+
+func assertContains(t *testing.T, result *msc2836.EventRelationshipResponse, wantEventIDs []string) {
+	t.Helper()
+	gotEventIDs := make([]string, len(result.Events))
+	for i, ev := range result.Events {
+		gotEventIDs[i] = ev.EventID
+	}
+	if len(gotEventIDs) != len(wantEventIDs) {
+		t.Fatalf("length mismatch: got %v want %v", gotEventIDs, wantEventIDs)
+	}
+	for i := range gotEventIDs {
+		if gotEventIDs[i] != wantEventIDs[i] {
+			t.Errorf("wrong item in position %d - got %s want %s", i, gotEventIDs[i], wantEventIDs[i])
+		}
+	}
+}
+
+type testUserAPI struct {
+	accessTokens map[string]userapi.Device
+}
+
+func (u *testUserAPI) InputAccountData(ctx context.Context, req *userapi.InputAccountDataRequest, res *userapi.InputAccountDataResponse) error {
+	return nil
+}
+func (u *testUserAPI) PerformAccountCreation(ctx context.Context, req *userapi.PerformAccountCreationRequest, res *userapi.PerformAccountCreationResponse) error {
+	return nil
+}
+func (u *testUserAPI) PerformPasswordUpdate(ctx context.Context, req *userapi.PerformPasswordUpdateRequest, res *userapi.PerformPasswordUpdateResponse) error {
+	return nil
+}
+func (u *testUserAPI) PerformDeviceCreation(ctx context.Context, req *userapi.PerformDeviceCreationRequest, res *userapi.PerformDeviceCreationResponse) error {
+	return nil
+}
+func (u *testUserAPI) PerformDeviceDeletion(ctx context.Context, req *userapi.PerformDeviceDeletionRequest, res *userapi.PerformDeviceDeletionResponse) error {
+	return nil
+}
+func (u *testUserAPI) PerformDeviceUpdate(ctx context.Context, req *userapi.PerformDeviceUpdateRequest, res *userapi.PerformDeviceUpdateResponse) error {
+	return nil
+}
+func (u *testUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.PerformLastSeenUpdateRequest, res *userapi.PerformLastSeenUpdateResponse) error {
+	return nil
+}
+func (u *testUserAPI) PerformAccountDeactivation(ctx context.Context, req *userapi.PerformAccountDeactivationRequest, res *userapi.PerformAccountDeactivationResponse) error {
+	return nil
+}
+func (u *testUserAPI) QueryProfile(ctx context.Context, req *userapi.QueryProfileRequest, res *userapi.QueryProfileResponse) error {
+	return nil
+}
+func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error {
+	dev, ok := u.accessTokens[req.AccessToken]
+	if !ok {
+		res.Err = fmt.Errorf("unknown token")
+		return nil
+	}
+	res.Device = &dev
+	return nil
+}
+func (u *testUserAPI) QueryDevices(ctx context.Context, req *userapi.QueryDevicesRequest, res *userapi.QueryDevicesResponse) error {
+	return nil
+}
+func (u *testUserAPI) QueryAccountData(ctx context.Context, req *userapi.QueryAccountDataRequest, res *userapi.QueryAccountDataResponse) error {
+	return nil
+}
+func (u *testUserAPI) QueryDeviceInfos(ctx context.Context, req *userapi.QueryDeviceInfosRequest, res *userapi.QueryDeviceInfosResponse) error {
+	return nil
+}
+func (u *testUserAPI) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error {
+	return nil
+}
+
+type testRoomserverAPI struct {
+	// use a trace API as it implements method stubs so we don't need to have them here.
+	// We'll override the functions we care about.
+	roomserver.RoomserverInternalAPITrace
+	userToJoinedRooms map[string][]string
+	events            map[string]*gomatrixserverlib.HeaderedEvent
+}
+
+func (r *testRoomserverAPI) QueryEventsByID(ctx context.Context, req *roomserver.QueryEventsByIDRequest, res *roomserver.QueryEventsByIDResponse) error {
+	for _, eventID := range req.EventIDs {
+		ev := r.events[eventID]
+		if ev != nil {
+			res.Events = append(res.Events, ev)
+		}
+	}
+	return nil
+}
+
+func (r *testRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *roomserver.QueryMembershipForUserRequest, res *roomserver.QueryMembershipForUserResponse) error {
+	rooms := r.userToJoinedRooms[req.UserID]
+	for _, roomID := range rooms {
+		if roomID == req.RoomID {
+			res.IsInRoom = true
+			res.HasBeenInRoom = true
+			res.Membership = "join"
+			break
+		}
+	}
+	return nil
+}
+
+func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserver.RoomserverInternalAPI, events []*gomatrixserverlib.HeaderedEvent) *mux.Router {
+	t.Helper()
+	cfg := &config.Dendrite{}
+	cfg.Defaults()
+	cfg.Global.ServerName = "localhost"
+	cfg.MSCs.Database.ConnectionString = "file:msc2836_test.db"
+	cfg.MSCs.MSCs = []string{"msc2836"}
+	base := &setup.BaseDendrite{
+		Cfg:                    cfg,
+		PublicClientAPIMux:     mux.NewRouter().PathPrefix(httputil.PublicClientPathPrefix).Subrouter(),
+		PublicFederationAPIMux: mux.NewRouter().PathPrefix(httputil.PublicFederationPathPrefix).Subrouter(),
+	}
+
+	err := msc2836.Enable(base, rsAPI, nil, userAPI, nil)
+	if err != nil {
+		t.Fatalf("failed to enable MSC2836: %s", err)
+	}
+	for _, ev := range events {
+		hooks.Run(hooks.KindNewEventPersisted, ev)
+	}
+	return base.PublicClientAPIMux
+}
+
+type fledglingEvent struct {
+	Type     string
+	StateKey *string
+	Content  interface{}
+	Sender   string
+	RoomID   string
+}
+
+func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *gomatrixserverlib.HeaderedEvent) {
+	t.Helper()
+	roomVer := gomatrixserverlib.RoomVersionV6
+	seed := make([]byte, ed25519.SeedSize) // zero seed
+	key := ed25519.NewKeyFromSeed(seed)
+	eb := gomatrixserverlib.EventBuilder{
+		Sender:   ev.Sender,
+		Depth:    999,
+		Type:     ev.Type,
+		StateKey: ev.StateKey,
+		RoomID:   ev.RoomID,
+	}
+	err := eb.SetContent(ev.Content)
+	if err != nil {
+		t.Fatalf("mustCreateEvent: failed to marshal event content %+v", ev.Content)
+	}
+	// make sure the origin_server_ts changes so we can test recency
+	time.Sleep(1 * time.Millisecond)
+	signedEvent, err := eb.Build(time.Now(), gomatrixserverlib.ServerName("localhost"), "ed25519:test", key, roomVer)
+	if err != nil {
+		t.Fatalf("mustCreateEvent: failed to sign event: %s", err)
+	}
+	h := signedEvent.Headered(roomVer)
+	return h
+}

internal/mscs/msc2836/storage.go

@@ -0,0 +1,226 @@
+package msc2836
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type eventInfo struct {
+	EventID        string
+	OriginServerTS gomatrixserverlib.Timestamp
+	RoomID         string
+	Servers        []string
+}
+
+type Database interface {
+	// StoreRelation stores the parent->child and child->parent relationship for later querying.
+	// Also stores the event metadata e.g timestamp
+	StoreRelation(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent) error
+	// ChildrenForParent returns the events who have the given `eventID` as an m.relationship with the
+	// provided `relType`. The returned slice is sorted by origin_server_ts according to whether
+	// `recentFirst` is true or false.
+	ChildrenForParent(ctx context.Context, eventID, relType string, recentFirst bool) ([]eventInfo, error)
+}
+
+type DB struct {
+	db                                     *sql.DB
+	writer                                 sqlutil.Writer
+	insertEdgeStmt                         *sql.Stmt
+	insertNodeStmt                         *sql.Stmt
+	selectChildrenForParentOldestFirstStmt *sql.Stmt
+	selectChildrenForParentRecentFirstStmt *sql.Stmt
+}
+
+// NewDatabase loads the database for msc2836
+func NewDatabase(dbOpts *config.DatabaseOptions) (Database, error) {
+	if dbOpts.ConnectionString.IsPostgres() {
+		return newPostgresDatabase(dbOpts)
+	}
+	return newSQLiteDatabase(dbOpts)
+}
+
+func newPostgresDatabase(dbOpts *config.DatabaseOptions) (Database, error) {
+	d := DB{
+		writer: sqlutil.NewDummyWriter(),
+	}
+	var err error
+	if d.db, err = sqlutil.Open(dbOpts); err != nil {
+		return nil, err
+	}
+	_, err = d.db.Exec(`
+	CREATE TABLE IF NOT EXISTS msc2836_edges (
+		parent_event_id TEXT NOT NULL,
+		child_event_id TEXT NOT NULL,
+		rel_type TEXT NOT NULL,
+		parent_room_id TEXT NOT NULL,
+		parent_servers TEXT NOT NULL,
+		CONSTRAINT msc2836_edges_uniq UNIQUE (parent_event_id, child_event_id, rel_type)
+	);
+
+	CREATE TABLE IF NOT EXISTS msc2836_nodes (
+		event_id TEXT PRIMARY KEY NOT NULL,
+		origin_server_ts BIGINT NOT NULL,
+		room_id TEXT NOT NULL
+	);
+	`)
+	if err != nil {
+		return nil, err
+	}
+	if d.insertEdgeStmt, err = d.db.Prepare(`
+		INSERT INTO msc2836_edges(parent_event_id, child_event_id, rel_type, parent_room_id, parent_servers) VALUES($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING
+	`); err != nil {
+		return nil, err
+	}
+	if d.insertNodeStmt, err = d.db.Prepare(`
+		INSERT INTO msc2836_nodes(event_id, origin_server_ts, room_id) VALUES($1, $2, $3) ON CONFLICT DO NOTHING
+	`); err != nil {
+		return nil, err
+	}
+	selectChildrenQuery := `
+	SELECT child_event_id, origin_server_ts, room_id FROM msc2836_edges
+	LEFT JOIN msc2836_nodes ON msc2836_edges.child_event_id = msc2836_nodes.event_id
+	WHERE parent_event_id = $1 AND rel_type = $2
+	ORDER BY origin_server_ts
+	`
+	if d.selectChildrenForParentOldestFirstStmt, err = d.db.Prepare(selectChildrenQuery + "ASC"); err != nil {
+		return nil, err
+	}
+	if d.selectChildrenForParentRecentFirstStmt, err = d.db.Prepare(selectChildrenQuery + "DESC"); err != nil {
+		return nil, err
+	}
+	return &d, err
+}
+
+func newSQLiteDatabase(dbOpts *config.DatabaseOptions) (Database, error) {
+	d := DB{
+		writer: sqlutil.NewExclusiveWriter(),
+	}
+	var err error
+	if d.db, err = sqlutil.Open(dbOpts); err != nil {
+		return nil, err
+	}
+	_, err = d.db.Exec(`
+	CREATE TABLE IF NOT EXISTS msc2836_edges (
+		parent_event_id TEXT NOT NULL,
+		child_event_id TEXT NOT NULL,
+		rel_type TEXT NOT NULL,
+		parent_room_id TEXT NOT NULL,
+		parent_servers TEXT NOT NULL,
+		UNIQUE (parent_event_id, child_event_id, rel_type)
+	);
+
+	CREATE TABLE IF NOT EXISTS msc2836_nodes (
+		event_id TEXT PRIMARY KEY NOT NULL,
+		origin_server_ts BIGINT NOT NULL,
+		room_id TEXT NOT NULL
+	);
+	`)
+	if err != nil {
+		return nil, err
+	}
+	if d.insertEdgeStmt, err = d.db.Prepare(`
+		INSERT INTO msc2836_edges(parent_event_id, child_event_id, rel_type, parent_room_id, parent_servers) VALUES($1, $2, $3, $4, $5) ON CONFLICT (parent_event_id, child_event_id, rel_type) DO NOTHING
+	`); err != nil {
+		return nil, err
+	}
+	if d.insertNodeStmt, err = d.db.Prepare(`
+		INSERT INTO msc2836_nodes(event_id, origin_server_ts, room_id) VALUES($1, $2, $3) ON CONFLICT DO NOTHING
+	`); err != nil {
+		return nil, err
+	}
+	selectChildrenQuery := `
+	SELECT child_event_id, origin_server_ts, room_id FROM msc2836_edges
+	LEFT JOIN msc2836_nodes ON msc2836_edges.child_event_id = msc2836_nodes.event_id
+	WHERE parent_event_id = $1 AND rel_type = $2
+	ORDER BY origin_server_ts
+	`
+	if d.selectChildrenForParentOldestFirstStmt, err = d.db.Prepare(selectChildrenQuery + "ASC"); err != nil {
+		return nil, err
+	}
+	if d.selectChildrenForParentRecentFirstStmt, err = d.db.Prepare(selectChildrenQuery + "DESC"); err != nil {
+		return nil, err
+	}
+	return &d, nil
+}
+
+func (p *DB) StoreRelation(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent) error {
+	parent, child, relType := parentChildEventIDs(ev)
+	if parent == "" || child == "" {
+		return nil
+	}
+	relationRoomID, relationServers := roomIDAndServers(ev)
+	relationServersJSON, err := json.Marshal(relationServers)
+	if err != nil {
+		return err
+	}
+	return p.writer.Do(p.db, nil, func(txn *sql.Tx) error {
+		_, err := txn.Stmt(p.insertEdgeStmt).ExecContext(ctx, parent, child, relType, relationRoomID, string(relationServersJSON))
+		if err != nil {
+			return err
+		}
+		_, err = txn.Stmt(p.insertNodeStmt).ExecContext(ctx, ev.EventID(), ev.OriginServerTS(), ev.RoomID())
+		return err
+	})
+}
+
+func (p *DB) ChildrenForParent(ctx context.Context, eventID, relType string, recentFirst bool) ([]eventInfo, error) {
+	var rows *sql.Rows
+	var err error
+	if recentFirst {
+		rows, err = p.selectChildrenForParentRecentFirstStmt.QueryContext(ctx, eventID, relType)
+	} else {
+		rows, err = p.selectChildrenForParentOldestFirstStmt.QueryContext(ctx, eventID, relType)
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close() // nolint: errcheck
+	var children []eventInfo
+	for rows.Next() {
+		var evInfo eventInfo
+		if err := rows.Scan(&evInfo.EventID, &evInfo.OriginServerTS, &evInfo.RoomID); err != nil {
+			return nil, err
+		}
+		children = append(children, evInfo)
+	}
+	return children, nil
+}
+
+func parentChildEventIDs(ev *gomatrixserverlib.HeaderedEvent) (parent, child, relType string) {
+	if ev == nil {
+		return
+	}
+	body := struct {
+		Relationship struct {
+			RelType string `json:"rel_type"`
+			EventID string `json:"event_id"`
+		} `json:"m.relationship"`
+	}{}
+	if err := json.Unmarshal(ev.Content(), &body); err != nil {
+		return
+	}
+	if body.Relationship.EventID == "" || body.Relationship.RelType == "" {
+		return
+	}
+	return body.Relationship.EventID, ev.EventID(), body.Relationship.RelType
+}
+
+func roomIDAndServers(ev *gomatrixserverlib.HeaderedEvent) (roomID string, servers []string) {
+	servers = []string{}
+	if ev == nil {
+		return
+	}
+	body := struct {
+		RoomID  string   `json:"relationship_room_id"`
+		Servers []string `json:"relationship_servers"`
+	}{}
+	if err := json.Unmarshal(ev.Unsigned(), &body); err != nil {
+		return
+	}
+	return body.RoomID, body.Servers
+}

internal/mscs/mscs.go

@@ -0,0 +1,42 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package mscs implements Matrix Spec Changes from https://github.com/matrix-org/matrix-doc
+package mscs
+
+import (
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/mscs/msc2836"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+// Enable MSCs - returns an error on unknown MSCs
+func Enable(base *setup.BaseDendrite, monolith *setup.Monolith) error {
+	for _, msc := range base.Cfg.MSCs.MSCs {
+		if err := EnableMSC(base, monolith, msc); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func EnableMSC(base *setup.BaseDendrite, monolith *setup.Monolith, msc string) error {
+	switch msc {
+	case "msc2836":
+		return msc2836.Enable(base, monolith.RoomserverAPI, monolith.FederationSenderAPI, monolith.UserAPI, monolith.KeyRing)
+	default:
+		return fmt.Errorf("EnableMSC: unknown msc '%s'", msc)
+	}
+}

internal/setup/flags.go

@@ -16,18 +16,28 @@ package setup
 
 import (
 	"flag"
+	"fmt"
+	"os"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
-
 	"github.com/sirupsen/logrus"
 )
 
-var configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")
+var (
+	configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")
+	version    = flag.Bool("version", false, "Shows the current version and exits immediately.")
+)
 
 // ParseFlags parses the commandline flags and uses them to create a config.
 func ParseFlags(monolith bool) *config.Dendrite {
 	flag.Parse()
 
+	if *version {
+		fmt.Println(internal.VersionString())
+		os.Exit(0)
+	}
+
 	if *configPath == "" {
 		logrus.Fatal("--config must be supplied")
 	}

internal/setup/kafka/kafka.go

@@ -17,12 +17,17 @@ func SetupConsumerProducer(cfg *config.Kafka) (sarama.Consumer, sarama.SyncProdu
 
 // setupKafka creates kafka consumer/producer pair from the config.
 func setupKafka(cfg *config.Kafka) (sarama.Consumer, sarama.SyncProducer) {
-	consumer, err := sarama.NewConsumer(cfg.Addresses, nil)
+	sCfg := sarama.NewConfig()
+	sCfg.Producer.MaxMessageBytes = *cfg.MaxMessageBytes
+	sCfg.Producer.Return.Successes = true
+	sCfg.Consumer.Fetch.Default = int32(*cfg.MaxMessageBytes)
+
+	consumer, err := sarama.NewConsumer(cfg.Addresses, sCfg)
 	if err != nil {
 		logrus.WithError(err).Panic("failed to start kafka consumer")
 	}
 
-	producer, err := sarama.NewSyncProducer(cfg.Addresses, nil)
+	producer, err := sarama.NewSyncProducer(cfg.Addresses, sCfg)
 	if err != nil {
 		logrus.WithError(err).Panic("failed to setup kafka producers")
 	}

internal/transactions/transactions_test.go

@@ -14,6 +14,7 @@ package transactions
 
 import (
 	"net/http"
+	"strconv"
 	"testing"
 
 	"github.com/matrix-org/util"
@@ -44,8 +45,8 @@ func TestCache(t *testing.T) {
 	for i := 1; i <= 100; i++ {
 		fakeTxnCache.AddTransaction(
 			fakeAccessToken,
-			fakeTxnID+string(i),
-			&util.JSONResponse{Code: http.StatusOK, JSON: fakeType{ID: string(i)}},
+			fakeTxnID+strconv.Itoa(i),
+			&util.JSONResponse{Code: http.StatusOK, JSON: fakeType{ID: strconv.Itoa(i)}},
 		)
 	}
 

internal/version.go

@@ -16,8 +16,8 @@ var build string
 
 const (
 	VersionMajor = 0
-	VersionMinor = 2
-	VersionPatch = 0
+	VersionMinor = 3
+	VersionPatch = 1
 	VersionTag   = "" // example: "rc1"
 )
 

roomserver/acls/acls.go

@@ -61,7 +61,7 @@ func NewServerACLs(db ServerACLDatabase) *ServerACLs {
 			continue
 		}
 		if state != nil {
-			acls.OnServerACLUpdate(&state.Event)
+			acls.OnServerACLUpdate(state.Event)
 		}
 	}
 	return acls

roomserver/api/api.go

@@ -153,6 +153,9 @@ type RoomserverInternalAPI interface {
 		response *PerformBackfillResponse,
 	) error
 
+	// PerformForget forgets a rooms history for a specific user
+	PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error
+
 	// Asks for the default room version as preferred by the server.
 	QueryRoomVersionCapabilities(
 		ctx context.Context,

roomserver/api/api_trace.go

@@ -204,6 +204,16 @@ func (t *RoomserverInternalAPITrace) PerformBackfill(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) PerformForget(
+	ctx context.Context,
+	req *PerformForgetRequest,
+	res *PerformForgetResponse,
+) error {
+	err := t.Impl.PerformForget(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformForget req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
 func (t *RoomserverInternalAPITrace) QueryRoomVersionCapabilities(
 	ctx context.Context,
 	req *QueryRoomVersionCapabilitiesRequest,

roomserver/api/input.go

@@ -53,7 +53,7 @@ type InputRoomEvent struct {
 	// This controls how the event is processed.
 	Kind Kind `json:"kind"`
 	// The event JSON for the event to add.
-	Event gomatrixserverlib.HeaderedEvent `json:"event"`
+	Event *gomatrixserverlib.HeaderedEvent `json:"event"`
 	// List of state event IDs that authenticate this event.
 	// These are likely derived from the "auth_events" JSON key of the event.
 	// But can be different because the "auth_events" key can be incomplete or wrong.

roomserver/api/output.go

@@ -99,7 +99,7 @@ const (
 // prev_events.
 type OutputNewRoomEvent struct {
 	// The Event.
-	Event gomatrixserverlib.HeaderedEvent `json:"event"`
+	Event *gomatrixserverlib.HeaderedEvent `json:"event"`
 	// Does the event completely rewrite the room state? If so, then AddsStateEventIDs
 	// will contain the entire room state.
 	RewritesState bool `json:"rewrites_state"`
@@ -116,7 +116,7 @@ type OutputNewRoomEvent struct {
 	// may decide a bunch of state events on one branch are now valid, so they will be
 	// present in this list. This is useful when trying to maintain the current state of a room
 	// as to do so you need to include both these events and `Event`.
-	AddStateEvents []gomatrixserverlib.HeaderedEvent `json:"adds_state_events"`
+	AddStateEvents []*gomatrixserverlib.HeaderedEvent `json:"adds_state_events"`
 
 	// The state event IDs that were removed from the state of the room by this event.
 	RemovesStateEventIDs []string `json:"removes_state_event_ids"`
@@ -173,7 +173,7 @@ type OutputNewRoomEvent struct {
 // the original event to save space, so you cannot use that slice alone.
 // Instead, use this function which will add the original event if it is present
 // in `AddsStateEventIDs`.
-func (ore *OutputNewRoomEvent) AddsState() []gomatrixserverlib.HeaderedEvent {
+func (ore *OutputNewRoomEvent) AddsState() []*gomatrixserverlib.HeaderedEvent {
 	includeOutputEvent := false
 	for _, id := range ore.AddsStateEventIDs {
 		if id == ore.Event.EventID() {
@@ -198,7 +198,7 @@ func (ore *OutputNewRoomEvent) AddsState() []gomatrixserverlib.HeaderedEvent {
 // should build their current room state up from OutputNewRoomEvents only.
 type OutputOldRoomEvent struct {
 	// The Event.
-	Event gomatrixserverlib.HeaderedEvent `json:"event"`
+	Event *gomatrixserverlib.HeaderedEvent `json:"event"`
 }
 
 // An OutputNewInviteEvent is written whenever an invite becomes active.
@@ -208,7 +208,7 @@ type OutputNewInviteEvent struct {
 	// The room version of the invited room.
 	RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
 	// The "m.room.member" invite event.
-	Event gomatrixserverlib.HeaderedEvent `json:"event"`
+	Event *gomatrixserverlib.HeaderedEvent `json:"event"`
 }
 
 // An OutputRetireInviteEvent is written whenever an existing invite is no longer
@@ -235,7 +235,7 @@ type OutputRedactedEvent struct {
 	// The event ID that was redacted
 	RedactedEventID string
 	// The value of `unsigned.redacted_because` - the redaction event itself
-	RedactedBecause gomatrixserverlib.HeaderedEvent
+	RedactedBecause *gomatrixserverlib.HeaderedEvent
 }
 
 // An OutputNewPeek is written whenever a user starts peeking into a room

roomserver/api/perform.go

@@ -83,7 +83,8 @@ type PerformJoinRequest struct {
 
 type PerformJoinResponse struct {
 	// The room ID, populated on success.
-	RoomID string `json:"room_id"`
+	RoomID    string `json:"room_id"`
+	JoinedVia gomatrixserverlib.ServerName
 	// If non-nil, the join request failed. Contains more information why it failed.
 	Error *PerformError
 }
@@ -98,7 +99,7 @@ type PerformLeaveResponse struct {
 
 type PerformInviteRequest struct {
 	RoomVersion     gomatrixserverlib.RoomVersion             `json:"room_version"`
-	Event           gomatrixserverlib.HeaderedEvent           `json:"event"`
+	Event           *gomatrixserverlib.HeaderedEvent          `json:"event"`
 	InviteRoomState []gomatrixserverlib.InviteV2StrippedState `json:"invite_room_state"`
 	SendAsServer    string                                    `json:"send_as_server"`
 	TransactionID   *TransactionID                            `json:"transaction_id"`
@@ -147,7 +148,7 @@ func (r *PerformBackfillRequest) PrevEventIDs() []string {
 // PerformBackfillResponse is a response to PerformBackfill.
 type PerformBackfillResponse struct {
 	// Missing events, arbritrary order.
-	Events []gomatrixserverlib.HeaderedEvent `json:"events"`
+	Events []*gomatrixserverlib.HeaderedEvent `json:"events"`
 }
 
 type PerformPublishRequest struct {
@@ -181,3 +182,12 @@ type PerformInboundPeekResponse struct {
 	// The event at which this state was captured
 	LatestEvent gomatrixserverlib.HeaderedEvent `json:"latest_event"`
 }
+
+// PerformForgetRequest is a request to PerformForget
+type PerformForgetRequest struct {
+	RoomID string `json:"room_id"`
+	UserID string `json:"user_id"`
+}
+
+type PerformForgetResponse struct{}
+

roomserver/api/query.go

@@ -50,7 +50,7 @@ type QueryLatestEventsAndStateResponse struct {
 	// This list will be in an arbitrary order.
 	// These are used to set the auth_events when sending an event.
 	// These are used to check whether the event is allowed.
-	StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"`
+	StateEvents []*gomatrixserverlib.HeaderedEvent `json:"state_events"`
 	// The depth of the latest events.
 	// This is one greater than the maximum depth of the latest events.
 	// This is used to set the depth when sending an event.
@@ -80,7 +80,7 @@ type QueryStateAfterEventsResponse struct {
 	PrevEventsExist bool `json:"prev_events_exist"`
 	// The state events requested.
 	// This list will be in an arbitrary order.
-	StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"`
+	StateEvents []*gomatrixserverlib.HeaderedEvent `json:"state_events"`
 }
 
 type QueryMissingAuthPrevEventsRequest struct {
@@ -119,7 +119,7 @@ type QueryEventsByIDResponse struct {
 	// fails to read it from the database then it will fail
 	// the entire request.
 	// This list will be in an arbitrary order.
-	Events []gomatrixserverlib.HeaderedEvent `json:"events"`
+	Events []*gomatrixserverlib.HeaderedEvent `json:"events"`
 }
 
 // QueryMembershipForUserRequest is a request to QueryMembership
@@ -140,7 +140,9 @@ type QueryMembershipForUserResponse struct {
 	// True if the user is in room.
 	IsInRoom bool `json:"is_in_room"`
 	// The current membership
-	Membership string
+	Membership string `json:"membership"`
+	// True if the user asked to forget this room.
+	IsRoomForgotten bool `json:"is_room_forgotten"`
 }
 
 // QueryMembershipsForRoomRequest is a request to QueryMembershipsForRoom
@@ -160,6 +162,8 @@ type QueryMembershipsForRoomResponse struct {
 	// True if the user has been in room before and has either stayed in it or
 	// left it.
 	HasBeenInRoom bool `json:"has_been_in_room"`
+	// True if the user asked to forget this room.
+	IsRoomForgotten bool `json:"is_room_forgotten"`
 }
 
 // QueryServerJoinedToRoomRequest is a request to QueryServerJoinedToRoom
@@ -209,7 +213,7 @@ type QueryMissingEventsRequest struct {
 // QueryMissingEventsResponse is a response to QueryMissingEvents
 type QueryMissingEventsResponse struct {
 	// Missing events, arbritrary order.
-	Events []gomatrixserverlib.HeaderedEvent `json:"events"`
+	Events []*gomatrixserverlib.HeaderedEvent `json:"events"`
 }
 
 // QueryStateAndAuthChainRequest is a request to QueryStateAndAuthChain
@@ -238,8 +242,8 @@ type QueryStateAndAuthChainResponse struct {
 	PrevEventsExist bool `json:"prev_events_exist"`
 	// The state and auth chain events that were requested.
 	// The lists will be in an arbitrary order.
-	StateEvents     []gomatrixserverlib.HeaderedEvent `json:"state_events"`
-	AuthChainEvents []gomatrixserverlib.HeaderedEvent `json:"auth_chain_events"`
+	StateEvents     []*gomatrixserverlib.HeaderedEvent `json:"state_events"`
+	AuthChainEvents []*gomatrixserverlib.HeaderedEvent `json:"auth_chain_events"`
 }
 
 // QueryRoomVersionCapabilitiesRequest asks for the default room version

roomserver/api/wrapper.go

@@ -25,7 +25,7 @@ import (
 // SendEvents to the roomserver The events are written with KindNew.
 func SendEvents(
 	ctx context.Context, rsAPI RoomserverInternalAPI,
-	kind Kind, events []gomatrixserverlib.HeaderedEvent,
+	kind Kind, events []*gomatrixserverlib.HeaderedEvent,
 	sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID,
 ) error {
 	ires := make([]InputRoomEvent, len(events))
@@ -46,7 +46,7 @@ func SendEvents(
 // marked as `true` in haveEventIDs.
 func SendEventWithState(
 	ctx context.Context, rsAPI RoomserverInternalAPI, kind Kind,
-	state *gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent,
+	state *gomatrixserverlib.RespState, event *gomatrixserverlib.HeaderedEvent,
 	haveEventIDs map[string]bool,
 ) error {
 	outliers, err := state.Events()
@@ -97,7 +97,7 @@ func SendInputRoomEvents(
 // If we are in the room then the event should be sent using the SendEvents method.
 func SendInvite(
 	ctx context.Context,
-	rsAPI RoomserverInternalAPI, inviteEvent gomatrixserverlib.HeaderedEvent,
+	rsAPI RoomserverInternalAPI, inviteEvent *gomatrixserverlib.HeaderedEvent,
 	inviteRoomState []gomatrixserverlib.InviteV2StrippedState,
 	sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID,
 ) error {
@@ -134,7 +134,7 @@ func GetEvent(ctx context.Context, rsAPI RoomserverInternalAPI, eventID string)
 	if len(res.Events) != 1 {
 		return nil
 	}
-	return &res.Events[0]
+	return res.Events[0]
 }
 
 // GetStateEvent returns the current state event in the room or nil.

roomserver/auth/auth.go

@@ -25,7 +25,7 @@ import (
 func IsServerAllowed(
 	serverName gomatrixserverlib.ServerName,
 	serverCurrentlyInRoom bool,
-	authEvents []gomatrixserverlib.Event,
+	authEvents []*gomatrixserverlib.Event,
 ) bool {
 	historyVisibility := HistoryVisibilityForRoom(authEvents)
 
@@ -52,7 +52,7 @@ func IsServerAllowed(
 	return false
 }
 
-func HistoryVisibilityForRoom(authEvents []gomatrixserverlib.Event) string {
+func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) string {
 	// https://matrix.org/docs/spec/client_server/r0.6.0#id87
 	// By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared.
 	visibility := "shared"
@@ -78,7 +78,7 @@ func HistoryVisibilityForRoom(authEvents []gomatrixserverlib.Event) string {
 	return visibility
 }
 
-func IsAnyUserOnServerWithMembership(serverName gomatrixserverlib.ServerName, authEvents []gomatrixserverlib.Event, wantMembership string) bool {
+func IsAnyUserOnServerWithMembership(serverName gomatrixserverlib.ServerName, authEvents []*gomatrixserverlib.Event, wantMembership string) bool {
 	for _, ev := range authEvents {
 		membership, err := ev.Membership()
 		if err != nil || membership != wantMembership {

roomserver/internal/alias.go

@@ -229,7 +229,7 @@ func (r *RoomserverInternalAPI) sendUpdatedAliasesEvent(
 	// Add auth events
 	authEvents := gomatrixserverlib.NewAuthEvents(nil)
 	for i := range res.StateEvents {
-		err = authEvents.AddEvent(&res.StateEvents[i].Event)
+		err = authEvents.AddEvent(res.StateEvents[i].Event)
 		if err != nil {
 			return err
 		}

roomserver/internal/api.go

@@ -27,6 +27,7 @@ type RoomserverInternalAPI struct {
 	*perform.Leaver
 	*perform.Publisher
 	*perform.Backfiller
+	*perform.Forgetter
 	DB                     storage.Database
 	Cfg                    *config.RoomServer
 	Producer               sarama.SyncProducer
@@ -117,6 +118,9 @@ func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSen
 		// than trying random servers
 		PreferServers: r.PerspectiveServerNames,
 	}
+	r.Forgetter = &perform.Forgetter{
+		DB: r.DB,
+	}
 }
 
 func (r *RoomserverInternalAPI) PerformInvite(
@@ -148,3 +152,11 @@ func (r *RoomserverInternalAPI) PerformLeave(
 	}
 	return r.WriteOutputEvents(req.RoomID, outputEvents)
 }
+
+func (r *RoomserverInternalAPI) PerformForget(
+	ctx context.Context,
+	req *api.PerformForgetRequest,
+	resp *api.PerformForgetResponse,
+) error {
+	return r.Forgetter.PerformForget(ctx, req, resp)
+}

roomserver/internal/helpers/auth.go

@@ -31,7 +31,7 @@ import (
 func CheckForSoftFail(
 	ctx context.Context,
 	db storage.Database,
-	event gomatrixserverlib.HeaderedEvent,
+	event *gomatrixserverlib.HeaderedEvent,
 	stateEventIDs []string,
 ) (bool, error) {
 	rewritesState := len(stateEventIDs) > 1
@@ -72,7 +72,7 @@ func CheckForSoftFail(
 	}
 
 	// Work out which of the state events we actually need.
-	stateNeeded := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{event.Unwrap()})
+	stateNeeded := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{event.Unwrap()})
 
 	// Load the actual auth events from the database.
 	authEvents, err := loadAuthEvents(ctx, db, stateNeeded, authStateEntries)
@@ -93,7 +93,7 @@ func CheckForSoftFail(
 func CheckAuthEvents(
 	ctx context.Context,
 	db storage.Database,
-	event gomatrixserverlib.HeaderedEvent,
+	event *gomatrixserverlib.HeaderedEvent,
 	authEventIDs []string,
 ) ([]types.EventNID, error) {
 	// Grab the numeric IDs for the supplied auth state events from the database.
@@ -104,7 +104,7 @@ func CheckAuthEvents(
 	authStateEntries = types.DeduplicateStateEntries(authStateEntries)
 
 	// Work out which of the state events we actually need.
-	stateNeeded := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{event.Unwrap()})
+	stateNeeded := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{event.Unwrap()})
 
 	// Load the actual auth events from the database.
 	authEvents, err := loadAuthEvents(ctx, db, stateNeeded, authStateEntries)
@@ -168,7 +168,7 @@ func (ae *authEvents) lookupEventWithEmptyStateKey(typeNID types.EventTypeNID) *
 	if !ok {
 		return nil
 	}
-	return &event.Event
+	return event.Event
 }
 
 func (ae *authEvents) lookupEvent(typeNID types.EventTypeNID, stateKey string) *gomatrixserverlib.Event {
@@ -187,7 +187,7 @@ func (ae *authEvents) lookupEvent(typeNID types.EventTypeNID, stateKey string) *
 	if !ok {
 		return nil
 	}
-	return &event.Event
+	return event.Event
 }
 
 // loadAuthEvents loads the events needed for authentication from the supplied room state.

roomserver/internal/helpers/helpers.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/auth"
@@ -67,7 +68,7 @@ func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverNam
 	if err != nil {
 		return false, err
 	}
-	gmslEvents := make([]gomatrixserverlib.Event, len(events))
+	gmslEvents := make([]*gomatrixserverlib.Event, len(events))
 	for i := range events {
 		gmslEvents[i] = events[i].Event
 	}
@@ -190,13 +191,13 @@ func StateBeforeEvent(ctx context.Context, db storage.Database, info types.RoomI
 
 func LoadEvents(
 	ctx context.Context, db storage.Database, eventNIDs []types.EventNID,
-) ([]gomatrixserverlib.Event, error) {
+) ([]*gomatrixserverlib.Event, error) {
 	stateEvents, err := db.Events(ctx, eventNIDs)
 	if err != nil {
 		return nil, err
 	}
 
-	result := make([]gomatrixserverlib.Event, len(stateEvents))
+	result := make([]*gomatrixserverlib.Event, len(stateEvents))
 	for i := range stateEvents {
 		result[i] = stateEvents[i].Event
 	}
@@ -205,7 +206,7 @@ func LoadEvents(
 
 func LoadStateEvents(
 	ctx context.Context, db storage.Database, stateEntries []types.StateEntry,
-) ([]gomatrixserverlib.Event, error) {
+) ([]*gomatrixserverlib.Event, error) {
 	eventNIDs := make([]types.EventNID, len(stateEntries))
 	for i := range stateEntries {
 		eventNIDs[i] = stateEntries[i].EventNID
@@ -222,12 +223,45 @@ func CheckServerAllowedToSeeEvent(
 		if errors.Is(err, sql.ErrNoRows) {
 			return false, nil
 		}
-		return false, err
+		return false, fmt.Errorf("roomState.LoadStateAtEvent: %w", err)
 	}
 
-	// TODO: We probably want to make it so that we don't have to pull
-	// out all the state if possible.
-	stateAtEvent, err := LoadStateEvents(ctx, db, stateEntries)
+	// Extract all of the event state key NIDs from the room state.
+	var stateKeyNIDs []types.EventStateKeyNID
+	for _, entry := range stateEntries {
+		stateKeyNIDs = append(stateKeyNIDs, entry.EventStateKeyNID)
+	}
+
+	// Then request those state key NIDs from the database.
+	stateKeys, err := db.EventStateKeys(ctx, stateKeyNIDs)
+	if err != nil {
+		return false, fmt.Errorf("db.EventStateKeys: %w", err)
+	}
+
+	// If the event state key doesn't match the given servername
+	// then we'll filter it out. This does preserve state keys that
+	// are "" since these will contain history visibility etc.
+	for nid, key := range stateKeys {
+		if key != "" && !strings.HasSuffix(key, ":"+string(serverName)) {
+			delete(stateKeys, nid)
+		}
+	}
+
+	// Now filter through all of the state events for the room.
+	// If the state key NID appears in the list of valid state
+	// keys then we'll add it to the list of filtered entries.
+	var filteredEntries []types.StateEntry
+	for _, entry := range stateEntries {
+		if _, ok := stateKeys[entry.EventStateKeyNID]; ok {
+			filteredEntries = append(filteredEntries, entry)
+		}
+	}
+
+	if len(filteredEntries) == 0 {
+		return false, nil
+	}
+
+	stateAtEvent, err := LoadStateEvents(ctx, db, filteredEntries)
 	if err != nil {
 		return false, err
 	}

roomserver/internal/input/input.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/Shopify/sarama"
+	"github.com/matrix-org/dendrite/internal/hooks"
 	"github.com/matrix-org/dendrite/roomserver/acls"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/storage"
@@ -61,7 +62,11 @@ func (w *inputWorker) start() {
 	for {
 		select {
 		case task := <-w.input:
+			hooks.Run(hooks.KindNewEventReceived, &task.event.Event)
 			_, task.err = w.r.processRoomEvent(task.ctx, task.event)
+			if task.err == nil {
+				hooks.Run(hooks.KindNewEventPersisted, &task.event.Event)
+			}
 			task.wg.Done()
 		case <-time.After(time.Second * 5):
 			return
@@ -92,7 +97,7 @@ func (r *Inputer) WriteOutputEvents(roomID string, updates []api.OutputEvent) er
 			})
 			if updates[i].NewRoomEvent.Event.Type() == "m.room.server_acl" && updates[i].NewRoomEvent.Event.StateKeyEquals("") {
 				ev := updates[i].NewRoomEvent.Event.Unwrap()
-				defer r.ACLs.OnServerACLUpdate(&ev)
+				defer r.ACLs.OnServerACLUpdate(ev)
 			}
 		}
 		logger.Infof("Producing to topic '%s'", r.OutputRoomEventTopic)
@@ -102,7 +107,13 @@ func (r *Inputer) WriteOutputEvents(roomID string, updates []api.OutputEvent) er
 			Value: sarama.ByteEncoder(value),
 		}
 	}
-	return r.Producer.SendMessages(messages)
+	errs := r.Producer.SendMessages(messages)
+	if errs != nil {
+		for _, err := range errs.(sarama.ProducerErrors) {
+			log.WithError(err).WithField("message_bytes", err.Msg.Value.Length()).Error("Write to kafka failed")
+		}
+	}
+	return errs
 }
 
 // InputRoomEvents implements api.RoomserverInternalAPI

roomserver/internal/input/input_events.go

@@ -111,11 +111,11 @@ func (r *Inputer) processRoomEvent(
 
 	// if storing this event results in it being redacted then do so.
 	if !isRejected && redactedEventID == event.EventID() {
-		r, rerr := eventutil.RedactEvent(redactionEvent, &event)
+		r, rerr := eventutil.RedactEvent(redactionEvent, event)
 		if rerr != nil {
 			return "", fmt.Errorf("eventutil.RedactEvent: %w", rerr)
 		}
-		event = *r
+		event = r
 	}
 
 	// For outliers we can stop after we've stored the event itself as it
@@ -215,7 +215,7 @@ func (r *Inputer) calculateAndSetState(
 	input *api.InputRoomEvent,
 	roomInfo types.RoomInfo,
 	stateAtEvent *types.StateAtEvent,
-	event gomatrixserverlib.Event,
+	event *gomatrixserverlib.Event,
 	isRejected bool,
 ) error {
 	var err error

roomserver/internal/input/input_latest_events.go

@@ -50,7 +50,7 @@ func (r *Inputer) updateLatestEvents(
 	ctx context.Context,
 	roomInfo *types.RoomInfo,
 	stateAtEvent types.StateAtEvent,
-	event gomatrixserverlib.Event,
+	event *gomatrixserverlib.Event,
 	sendAsServer string,
 	transactionID *api.TransactionID,
 	rewritesState bool,
@@ -92,7 +92,7 @@ type latestEventsUpdater struct {
 	updater       *shared.LatestEventsUpdater
 	roomInfo      *types.RoomInfo
 	stateAtEvent  types.StateAtEvent
-	event         gomatrixserverlib.Event
+	event         *gomatrixserverlib.Event
 	transactionID *api.TransactionID
 	rewritesState bool
 	// Which server to send this event as.
@@ -140,7 +140,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// Work out what the latest events are. This will include the new
 	// event if it is not already referenced.
 	extremitiesChanged, err := u.calculateLatest(
-		oldLatest, &u.event,
+		oldLatest, u.event,
 		types.StateAtEventAndReference{
 			EventReference: u.event.EventReference(),
 			StateAtEvent:   u.stateAtEvent,
@@ -373,7 +373,7 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error)
 
 // extraEventsForIDs returns the full events for the event IDs given, but does not include the current event being
 // updated.
-func (u *latestEventsUpdater) extraEventsForIDs(roomVersion gomatrixserverlib.RoomVersion, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) {
+func (u *latestEventsUpdater) extraEventsForIDs(roomVersion gomatrixserverlib.RoomVersion, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) {
 	var extraEventIDs []string
 	for _, e := range eventIDs {
 		if e == u.event.EventID() {
@@ -388,7 +388,7 @@ func (u *latestEventsUpdater) extraEventsForIDs(roomVersion gomatrixserverlib.Ro
 	if err != nil {
 		return nil, err
 	}
-	var h []gomatrixserverlib.HeaderedEvent
+	var h []*gomatrixserverlib.HeaderedEvent
 	for _, e := range extraEvents {
 		h = append(h, e.Headered(roomVersion))
 	}

roomserver/internal/input/input_membership.go

@@ -62,13 +62,13 @@ func (r *Inputer) updateMemberships(
 		if change.removedEventNID != 0 {
 			ev, _ := helpers.EventMap(events).Lookup(change.removedEventNID)
 			if ev != nil {
-				re = &ev.Event
+				re = ev.Event
 			}
 		}
 		if change.addedEventNID != 0 {
 			ev, _ := helpers.EventMap(events).Lookup(change.addedEventNID)
 			if ev != nil {
-				ae = &ev.Event
+				ae = ev.Event
 			}
 		}
 		if updates, err = r.updateMembership(updater, targetUserNID, re, ae, updates); err != nil {

roomserver/internal/perform/perform_backfill.go

@@ -83,7 +83,7 @@ func (r *Backfiller) PerformBackfill(
 	}
 
 	// Retrieve events from the list that was filled previously.
-	var loadedEvents []gomatrixserverlib.Event
+	var loadedEvents []*gomatrixserverlib.Event
 	loadedEvents, err = helpers.LoadEvents(ctx, r.DB, resultNIDs)
 	if err != nil {
 		return err
@@ -211,10 +211,10 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom
 		}
 	}
 
-	var newEvents []gomatrixserverlib.HeaderedEvent
+	var newEvents []*gomatrixserverlib.HeaderedEvent
 	for _, ev := range missingMap {
 		if ev != nil {
-			newEvents = append(newEvents, *ev)
+			newEvents = append(newEvents, ev)
 		}
 	}
 	util.GetLogger(ctx).Infof("Persisting %d new events", len(newEvents))
@@ -232,7 +232,7 @@ type backfillRequester struct {
 	// per-request state
 	servers                 []gomatrixserverlib.ServerName
 	eventIDToBeforeStateIDs map[string][]string
-	eventIDMap              map[string]gomatrixserverlib.Event
+	eventIDMap              map[string]*gomatrixserverlib.Event
 }
 
 func newBackfillRequester(
@@ -248,13 +248,13 @@ func newBackfillRequester(
 		fsAPI:                   fsAPI,
 		thisServer:              thisServer,
 		eventIDToBeforeStateIDs: make(map[string][]string),
-		eventIDMap:              make(map[string]gomatrixserverlib.Event),
+		eventIDMap:              make(map[string]*gomatrixserverlib.Event),
 		bwExtrems:               bwExtrems,
 		preferServer:            preferServer,
 	}
 }
 
-func (b *backfillRequester) StateIDsBeforeEvent(ctx context.Context, targetEvent gomatrixserverlib.HeaderedEvent) ([]string, error) {
+func (b *backfillRequester) StateIDsBeforeEvent(ctx context.Context, targetEvent *gomatrixserverlib.HeaderedEvent) ([]string, error) {
 	b.eventIDMap[targetEvent.EventID()] = targetEvent.Unwrap()
 	if ids, ok := b.eventIDToBeforeStateIDs[targetEvent.EventID()]; ok {
 		return ids, nil
@@ -305,7 +305,7 @@ FederationHit:
 	return nil, lastErr
 }
 
-func (b *backfillRequester) calculateNewStateIDs(targetEvent, prevEvent gomatrixserverlib.Event, prevEventStateIDs []string) []string {
+func (b *backfillRequester) calculateNewStateIDs(targetEvent, prevEvent *gomatrixserverlib.Event, prevEventStateIDs []string) []string {
 	newStateIDs := prevEventStateIDs[:]
 	if prevEvent.StateKey() == nil {
 		// state is the same as the previous event
@@ -343,7 +343,7 @@ func (b *backfillRequester) calculateNewStateIDs(targetEvent, prevEvent gomatrix
 }
 
 func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion,
-	event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) {
+	event *gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) {
 
 	// try to fetch the events from the database first
 	events, err := b.ProvideEvents(roomVer, eventIDs)
@@ -355,7 +355,7 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr
 		if len(events) == len(eventIDs) {
 			result := make(map[string]*gomatrixserverlib.Event)
 			for i := range events {
-				result[events[i].EventID()] = &events[i]
+				result[events[i].EventID()] = events[i]
 				b.eventIDMap[events[i].EventID()] = events[i]
 			}
 			return result, nil
@@ -372,7 +372,7 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr
 		return nil, err
 	}
 	for eventID, ev := range result {
-		b.eventIDMap[eventID] = *ev
+		b.eventIDMap[eventID] = ev
 	}
 	return result, nil
 }
@@ -426,7 +426,7 @@ FindSuccessor:
 	}
 
 	// possibly return all joined servers depending on history visiblity
-	memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries)
+	memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries, b.thisServer)
 	if err != nil {
 		logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules")
 		return nil
@@ -476,7 +476,7 @@ func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverl
 	return tx, err
 }
 
-func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion, eventIDs []string) ([]gomatrixserverlib.Event, error) {
+func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion, eventIDs []string) ([]*gomatrixserverlib.Event, error) {
 	ctx := context.Background()
 	nidMap, err := b.db.EventNIDs(ctx, eventIDs)
 	if err != nil {
@@ -494,18 +494,19 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion,
 		logrus.WithError(err).WithField("event_nids", eventNIDs).Error("Failed to load events")
 		return nil, err
 	}
-	events := make([]gomatrixserverlib.Event, len(eventsWithNids))
+	events := make([]*gomatrixserverlib.Event, len(eventsWithNids))
 	for i := range eventsWithNids {
 		events[i] = eventsWithNids[i].Event
 	}
 	return events, nil
 }
 
-// joinEventsFromHistoryVisibility returns all CURRENTLY joined members if the provided state indicated a 'shared' history visibility.
+// joinEventsFromHistoryVisibility returns all CURRENTLY joined members if our server can read the room history
 // TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just
 //       pull all events and then filter by that table.
 func joinEventsFromHistoryVisibility(
-	ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry) ([]types.Event, error) {
+	ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry,
+	thisServer gomatrixserverlib.ServerName) ([]types.Event, error) {
 
 	var eventNIDs []types.EventNID
 	for _, entry := range stateEntries {
@@ -521,13 +522,15 @@ func joinEventsFromHistoryVisibility(
 	if err != nil {
 		return nil, err
 	}
-	events := make([]gomatrixserverlib.Event, len(stateEvents))
+	events := make([]*gomatrixserverlib.Event, len(stateEvents))
 	for i := range stateEvents {
 		events[i] = stateEvents[i].Event
 	}
-	visibility := auth.HistoryVisibilityForRoom(events)
-	if visibility != "shared" {
-		logrus.Infof("ServersAtEvent history visibility not shared: %s", visibility)
+
+	// Can we see events in the room?
+	canSeeEvents := auth.IsServerAllowed(thisServer, true, events)
+	if !canSeeEvents {
+		logrus.Infof("ServersAtEvent history not visible to us: %s", auth.HistoryVisibilityForRoom(events))
 		return nil, nil
 	}
 	// get joined members
@@ -542,7 +545,7 @@ func joinEventsFromHistoryVisibility(
 	return db.Events(ctx, joinEventNIDs)
 }
 
-func persistEvents(ctx context.Context, db storage.Database, events []gomatrixserverlib.HeaderedEvent) (types.RoomNID, map[string]types.Event) {
+func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixserverlib.HeaderedEvent) (types.RoomNID, map[string]types.Event) {
 	var roomNID types.RoomNID
 	backfilledEventMap := make(map[string]types.Event)
 	for j, ev := range events {
@@ -570,7 +573,7 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse
 		// redacted, which we don't care about since we aren't returning it in this backfill.
 		if redactedEventID == ev.EventID() {
 			eventToRedact := ev.Unwrap()
-			redactedEvent, err := eventutil.RedactEvent(redactionEvent, &eventToRedact)
+			redactedEvent, err := eventutil.RedactEvent(redactionEvent, eventToRedact)
 			if err != nil {
 				logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to redact event")
 				continue

roomserver/internal/perform/perform_forget.go

@@ -0,0 +1,35 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package perform
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/storage"
+)
+
+type Forgetter struct {
+	DB storage.Database
+}
+
+// PerformForget implements api.RoomServerQueryAPI
+func (f *Forgetter) PerformForget(
+	ctx context.Context,
+	request *api.PerformForgetRequest,
+	response *api.PerformForgetResponse,
+) error {
+	return f.DB.ForgetRoom(ctx, request.UserID, request.RoomID, true)
+}

roomserver/internal/perform/perform_invite.go

@@ -86,7 +86,7 @@ func (r *Inviter) PerformInvite(
 
 	var isAlreadyJoined bool
 	if info != nil {
-		_, isAlreadyJoined, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey())
+		_, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey())
 		if err != nil {
 			return nil, fmt.Errorf("r.DB.GetMembership: %w", err)
 		}
@@ -198,7 +198,7 @@ func (r *Inviter) PerformInvite(
 		}
 
 		unwrapped := event.Unwrap()
-		outputUpdates, err := helpers.UpdateToInviteMembership(updater, &unwrapped, nil, req.Event.RoomVersion)
+		outputUpdates, err := helpers.UpdateToInviteMembership(updater, unwrapped, nil, req.Event.RoomVersion)
 		if err != nil {
 			return nil, fmt.Errorf("updateToInviteMembership: %w", err)
 		}
@@ -248,11 +248,11 @@ func buildInviteStrippedState(
 		return nil, err
 	}
 	inviteState := []gomatrixserverlib.InviteV2StrippedState{
-		gomatrixserverlib.NewInviteV2StrippedState(&input.Event.Event),
+		gomatrixserverlib.NewInviteV2StrippedState(input.Event.Event),
 	}
 	stateEvents = append(stateEvents, types.Event{Event: input.Event.Unwrap()})
 	for _, event := range stateEvents {
-		inviteState = append(inviteState, gomatrixserverlib.NewInviteV2StrippedState(&event.Event))
+		inviteState = append(inviteState, gomatrixserverlib.NewInviteV2StrippedState(event.Event))
 	}
 	return inviteState, nil
 }

roomserver/internal/perform/perform_join.go

@@ -47,7 +47,7 @@ func (r *Joiner) PerformJoin(
 	req *api.PerformJoinRequest,
 	res *api.PerformJoinResponse,
 ) {
-	roomID, err := r.performJoin(ctx, req)
+	roomID, joinedVia, err := r.performJoin(ctx, req)
 	if err != nil {
 		perr, ok := err.(*api.PerformError)
 		if ok {
@@ -59,21 +59,22 @@ func (r *Joiner) PerformJoin(
 		}
 	}
 	res.RoomID = roomID
+	res.JoinedVia = joinedVia
 }
 
 func (r *Joiner) performJoin(
 	ctx context.Context,
 	req *api.PerformJoinRequest,
-) (string, error) {
+) (string, gomatrixserverlib.ServerName, error) {
 	_, domain, err := gomatrixserverlib.SplitID('@', req.UserID)
 	if err != nil {
-		return "", &api.PerformError{
+		return "", "", &api.PerformError{
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("Supplied user ID %q in incorrect format", req.UserID),
 		}
 	}
 	if domain != r.Cfg.Matrix.ServerName {
-		return "", &api.PerformError{
+		return "", "", &api.PerformError{
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("User %q does not belong to this homeserver", req.UserID),
 		}
@@ -84,7 +85,7 @@ func (r *Joiner) performJoin(
 	if strings.HasPrefix(req.RoomIDOrAlias, "#") {
 		return r.performJoinRoomByAlias(ctx, req)
 	}
-	return "", &api.PerformError{
+	return "", "", &api.PerformError{
 		Code: api.PerformErrorBadRequest,
 		Msg:  fmt.Sprintf("Room ID or alias %q is invalid", req.RoomIDOrAlias),
 	}
@@ -93,11 +94,11 @@ func (r *Joiner) performJoin(
 func (r *Joiner) performJoinRoomByAlias(
 	ctx context.Context,
 	req *api.PerformJoinRequest,
-) (string, error) {
+) (string, gomatrixserverlib.ServerName, error) {
 	// Get the domain part of the room alias.
 	_, domain, err := gomatrixserverlib.SplitID('#', req.RoomIDOrAlias)
 	if err != nil {
-		return "", fmt.Errorf("Alias %q is not in the correct format", req.RoomIDOrAlias)
+		return "", "", fmt.Errorf("Alias %q is not in the correct format", req.RoomIDOrAlias)
 	}
 	req.ServerNames = append(req.ServerNames, domain)
 
@@ -115,7 +116,7 @@ func (r *Joiner) performJoinRoomByAlias(
 		err = r.FSAPI.PerformDirectoryLookup(ctx, &dirReq, &dirRes)
 		if err != nil {
 			logrus.WithError(err).Errorf("error looking up alias %q", req.RoomIDOrAlias)
-			return "", fmt.Errorf("Looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err)
+			return "", "", fmt.Errorf("Looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err)
 		}
 		roomID = dirRes.RoomID
 		req.ServerNames = append(req.ServerNames, dirRes.ServerNames...)
@@ -123,13 +124,13 @@ func (r *Joiner) performJoinRoomByAlias(
 		// Otherwise, look up if we know this room alias locally.
 		roomID, err = r.DB.GetRoomIDForAlias(ctx, req.RoomIDOrAlias)
 		if err != nil {
-			return "", fmt.Errorf("Lookup room alias %q failed: %w", req.RoomIDOrAlias, err)
+			return "", "", fmt.Errorf("Lookup room alias %q failed: %w", req.RoomIDOrAlias, err)
 		}
 	}
 
 	// If the room ID is empty then we failed to look up the alias.
 	if roomID == "" {
-		return "", fmt.Errorf("Alias %q not found", req.RoomIDOrAlias)
+		return "", "", fmt.Errorf("Alias %q not found", req.RoomIDOrAlias)
 	}
 
 	// If we do, then pluck out the room ID and continue the join.
@@ -142,11 +143,11 @@ func (r *Joiner) performJoinRoomByAlias(
 func (r *Joiner) performJoinRoomByID(
 	ctx context.Context,
 	req *api.PerformJoinRequest,
-) (string, error) {
+) (string, gomatrixserverlib.ServerName, error) {
 	// Get the domain part of the room ID.
 	_, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias)
 	if err != nil {
-		return "", &api.PerformError{
+		return "", "", &api.PerformError{
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("Room ID %q is invalid: %s", req.RoomIDOrAlias, err),
 		}
@@ -169,7 +170,7 @@ func (r *Joiner) performJoinRoomByID(
 		Redacts:  "",
 	}
 	if err = eb.SetUnsigned(struct{}{}); err != nil {
-		return "", fmt.Errorf("eb.SetUnsigned: %w", err)
+		return "", "", fmt.Errorf("eb.SetUnsigned: %w", err)
 	}
 
 	// It is possible for the request to include some "content" for the
@@ -180,7 +181,7 @@ func (r *Joiner) performJoinRoomByID(
 	}
 	req.Content["membership"] = gomatrixserverlib.Join
 	if err = eb.SetContent(req.Content); err != nil {
-		return "", fmt.Errorf("eb.SetContent: %w", err)
+		return "", "", fmt.Errorf("eb.SetContent: %w", err)
 	}
 
 	// Force a federated join if we aren't in the room and we've been
@@ -194,7 +195,7 @@ func (r *Joiner) performJoinRoomByID(
 	if err == nil && isInvitePending {
 		_, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender)
 		if ierr != nil {
-			return "", fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
+			return "", "", fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
 		}
 
 		// If we were invited by someone from another server then we can
@@ -206,8 +207,10 @@ func (r *Joiner) performJoinRoomByID(
 	}
 
 	// If we should do a forced federated join then do that.
+	var joinedVia gomatrixserverlib.ServerName
 	if forceFederatedJoin {
-		return req.RoomIDOrAlias, r.performFederatedJoinRoomByID(ctx, req)
+		joinedVia, err = r.performFederatedJoinRoomByID(ctx, req)
+		return req.RoomIDOrAlias, joinedVia, err
 	}
 
 	// Try to construct an actual join event from the template.
@@ -249,7 +252,7 @@ func (r *Joiner) performJoinRoomByID(
 			inputRes := api.InputRoomEventsResponse{}
 			r.Inputer.InputRoomEvents(ctx, &inputReq, &inputRes)
 			if err = inputRes.Err(); err != nil {
-				return "", &api.PerformError{
+				return "", "", &api.PerformError{
 					Code: api.PerformErrorNotAllowed,
 					Msg:  fmt.Sprintf("InputRoomEvents auth failed: %s", err),
 				}
@@ -265,7 +268,7 @@ func (r *Joiner) performJoinRoomByID(
 			// Otherwise we'll try a federated join as normal, since it's quite
 			// possible that the room still exists on other servers.
 			if len(req.ServerNames) == 0 {
-				return "", &api.PerformError{
+				return "", "", &api.PerformError{
 					Code: api.PerformErrorNoRoom,
 					Msg:  fmt.Sprintf("Room ID %q does not exist", req.RoomIDOrAlias),
 				}
@@ -273,24 +276,25 @@ func (r *Joiner) performJoinRoomByID(
 		}
 
 		// Perform a federated room join.
-		return req.RoomIDOrAlias, r.performFederatedJoinRoomByID(ctx, req)
+		joinedVia, err = r.performFederatedJoinRoomByID(ctx, req)
+		return req.RoomIDOrAlias, joinedVia, err
 
 	default:
 		// Something else went wrong.
-		return "", fmt.Errorf("Error joining local room: %q", err)
+		return "", "", fmt.Errorf("Error joining local room: %q", err)
 	}
 
 	// By this point, if req.RoomIDOrAlias contained an alias, then
 	// it will have been overwritten with a room ID by performJoinRoomByAlias.
 	// We should now include this in the response so that the CS API can
 	// return the right room ID.
-	return req.RoomIDOrAlias, nil
+	return req.RoomIDOrAlias, r.Cfg.Matrix.ServerName, nil
 }
 
 func (r *Joiner) performFederatedJoinRoomByID(
 	ctx context.Context,
 	req *api.PerformJoinRequest,
-) error {
+) (gomatrixserverlib.ServerName, error) {
 	// Try joining by all of the supplied server names.
 	fedReq := fsAPI.PerformJoinRequest{
 		RoomID:      req.RoomIDOrAlias, // the room ID to try and join
@@ -301,13 +305,13 @@ func (r *Joiner) performFederatedJoinRoomByID(
 	fedRes := fsAPI.PerformJoinResponse{}
 	r.FSAPI.PerformJoin(ctx, &fedReq, &fedRes)
 	if fedRes.LastError != nil {
-		return &api.PerformError{
+		return "", &api.PerformError{
 			Code:       api.PerformErrRemote,
 			Msg:        fedRes.LastError.Message,
 			RemoteCode: fedRes.LastError.Code,
 		}
 	}
-	return nil
+	return fedRes.JoinedVia, nil
 }
 
 func buildEvent(

roomserver/internal/query/query.go

@@ -204,11 +204,13 @@ func (r *Queryer) QueryMembershipForUser(
 		return fmt.Errorf("QueryMembershipForUser: unknown room %s", request.RoomID)
 	}
 
-	membershipEventNID, stillInRoom, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID)
+	membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID)
 	if err != nil {
 		return err
 	}
 
+	response.IsRoomForgotten = isRoomforgotten
+
 	if membershipEventNID == 0 {
 		response.HasBeenInRoom = false
 		return nil
@@ -241,11 +243,13 @@ func (r *Queryer) QueryMembershipsForRoom(
 		return err
 	}
 
-	membershipEventNID, stillInRoom, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender)
+	membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender)
 	if err != nil {
 		return err
 	}
 
+	response.IsRoomForgotten = isRoomforgotten
+
 	if membershipEventNID == 0 {
 		response.HasBeenInRoom = false
 		response.JoinEvents = nil
@@ -412,7 +416,7 @@ func (r *Queryer) QueryMissingEvents(
 		return err
 	}
 
-	response.Events = make([]gomatrixserverlib.HeaderedEvent, 0, len(loadedEvents)-len(eventsToFilter))
+	response.Events = make([]*gomatrixserverlib.HeaderedEvent, 0, len(loadedEvents)-len(eventsToFilter))
 	for _, event := range loadedEvents {
 		if !eventsToFilter[event.EventID()] {
 			roomVersion, verr := r.roomVersion(event.RoomID())
@@ -483,7 +487,7 @@ func (r *Queryer) QueryStateAndAuthChain(
 	return err
 }
 
-func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo types.RoomInfo, eventIDs []string) ([]gomatrixserverlib.Event, error) {
+func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo types.RoomInfo, eventIDs []string) ([]*gomatrixserverlib.Event, error) {
 	roomState := state.NewStateResolution(r.DB, roomInfo)
 	prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs)
 	if err != nil {
@@ -514,13 +518,13 @@ type eventsFromIDs func(context.Context, []string) ([]types.Event, error)
 // given events. Will *not* error if we don't have all auth events.
 func GetAuthChain(
 	ctx context.Context, fn eventsFromIDs, authEventIDs []string,
-) ([]gomatrixserverlib.Event, error) {
+) ([]*gomatrixserverlib.Event, error) {
 	// List of event IDs to fetch. On each pass, these events will be requested
 	// from the database and the `eventsToFetch` will be updated with any new
 	// events that we have learned about and need to find. When `eventsToFetch`
 	// is eventually empty, we should have reached the end of the chain.
 	eventsToFetch := authEventIDs
-	authEventsMap := make(map[string]gomatrixserverlib.Event)
+	authEventsMap := make(map[string]*gomatrixserverlib.Event)
 
 	for len(eventsToFetch) > 0 {
 		// Try to retrieve the events from the database.
@@ -551,7 +555,7 @@ func GetAuthChain(
 
 	// We've now retrieved all of the events we can. Flatten them down into an
 	// array and return them.
-	var authEvents []gomatrixserverlib.Event
+	var authEvents []*gomatrixserverlib.Event
 	for _, event := range authEventsMap {
 		authEvents = append(authEvents, event)
 	}

roomserver/internal/query/query_test.go

@@ -26,12 +26,12 @@ import (
 
 // used to implement RoomserverInternalAPIEventDB to test getAuthChain
 type getEventDB struct {
-	eventMap map[string]gomatrixserverlib.Event
+	eventMap map[string]*gomatrixserverlib.Event
 }
 
 func createEventDB() *getEventDB {
 	return &getEventDB{
-		eventMap: make(map[string]gomatrixserverlib.Event),
+		eventMap: make(map[string]*gomatrixserverlib.Event),
 	}
 }