From 0974f6e2c055d8d06b5ea9c175252b22b2399fe2 Mon Sep 17 00:00:00 2001
From: S7evinK <tfaelligen@gmail.com>
Date: Mon, 19 Oct 2020 10:38:10 +0200
Subject: [PATCH 01/24] Fix internal http api (#1535)

Signed-off-by: Till Faelligen <tfaelligen@gmail.com>
---
 userapi/inthttp/server.go | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go
index e24aad3a9..81e936e58 100644
--- a/userapi/inthttp/server.go
+++ b/userapi/inthttp/server.go
@@ -39,7 +39,7 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	internalAPIMux.Handle(PerformAccountCreationPath,
+	internalAPIMux.Handle(PerformPasswordUpdatePath,
 		httputil.MakeInternalAPI("performPasswordUpdate", func(req *http.Request) util.JSONResponse {
 			request := api.PerformPasswordUpdateRequest{}
 			response := api.PerformPasswordUpdateResponse{}
@@ -169,7 +169,7 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	internalAPIMux.Handle(QueryDeviceInfosPath,
+	internalAPIMux.Handle(QuerySearchProfilesPath,
 		httputil.MakeInternalAPI("querySearchProfiles", func(req *http.Request) util.JSONResponse {
 			request := api.QuerySearchProfilesRequest{}
 			response := api.QuerySearchProfilesResponse{}
@@ -182,4 +182,17 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(InputAccountDataPath,
+		httputil.MakeInternalAPI("inputAccountDataPath", func(req *http.Request) util.JSONResponse {
+			request := api.InputAccountDataRequest{}
+			response := api.InputAccountDataResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := s.InputAccountData(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }

From 6e63df1d9a3eadf924d518a1a02f04dfd03ad6b1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 19 Oct 2020 14:59:13 +0100
Subject: [PATCH 02/24] KindOld (#1531)

* Add KindOld

* Don't process latest events/memberships for old events

* Allow federationsender to ignore duplicate key entries when LatestEventIDs is duplicated by RS output events

* Signal to downstream components if an event has become a forward extremity

* Don't exclude from sync

* Soft-fail checks on KindNew

* Don't run the latest events updater at all for KindOld

* Don't make federation sender change after all

* Kind in federation sender join

* Don't send isForwardExtremity

* Fix syncapi

* Update comments

* Fix SendEventWithState

* Update sytest-whitelist

* Generate old output events

* Sync API consumes old room events

* Update comments
---
 clientapi/routing/createroom.go               |  1 +
 clientapi/routing/membership.go               |  1 +
 clientapi/routing/profile.go                  |  4 +-
 clientapi/routing/redaction.go                |  2 +-
 clientapi/routing/sendevent.go                |  1 +
 clientapi/threepid/invites.go                 |  1 +
 federationapi/routing/join.go                 |  1 +
 federationapi/routing/leave.go                |  1 +
 federationapi/routing/send.go                 |  3 ++
 federationapi/routing/threepid.go             |  3 +-
 federationsender/internal/perform.go          |  1 +
 roomserver/api/input.go                       | 16 ++++++--
 roomserver/api/output.go                      | 18 +++++++++
 roomserver/api/wrapper.go                     | 14 ++++---
 roomserver/internal/input/input_events.go     | 39 +++++++++++++------
 .../internal/input/input_latest_events.go     | 11 +++---
 roomserver/roomserver_test.go                 |  4 +-
 syncapi/consumers/roomserver.go               | 37 +++++++++++++++++-
 sytest-whitelist                              |  1 +
 19 files changed, 125 insertions(+), 34 deletions(-)

diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index 9655339cd..cff3c9813 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -344,6 +344,7 @@ func createRoom(
 		if err = roomserverAPI.SendEventWithState(
 			req.Context(),
 			rsAPI,
+			roomserverAPI.KindNew,
 			&gomatrixserverlib.RespState{
 				StateEvents: accumulated,
 				AuthEvents:  accumulated,
diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go
index 88cb23647..fe0795577 100644
--- a/clientapi/routing/membership.go
+++ b/clientapi/routing/membership.go
@@ -76,6 +76,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us
 
 	if err = roomserverAPI.SendEvents(
 		ctx, rsAPI,
+		api.KindNew,
 		[]gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)},
 		cfg.Matrix.ServerName,
 		nil,
diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go
index 60669a0c8..bbe35facd 100644
--- a/clientapi/routing/profile.go
+++ b/clientapi/routing/profile.go
@@ -170,7 +170,7 @@ func SetAvatarURL(
 		return jsonerror.InternalServerError()
 	}
 
-	if err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil {
+	if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, nil); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
 		return jsonerror.InternalServerError()
 	}
@@ -288,7 +288,7 @@ func SetDisplayName(
 		return jsonerror.InternalServerError()
 	}
 
-	if err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil {
+	if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, nil); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
 		return jsonerror.InternalServerError()
 	}
diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go
index 9701685e0..266c0aff2 100644
--- a/clientapi/routing/redaction.go
+++ b/clientapi/routing/redaction.go
@@ -121,7 +121,7 @@ func SendRedaction(
 			JSON: jsonerror.NotFound("Room does not exist"),
 		}
 	}
-	if err = roomserverAPI.SendEvents(context.Background(), rsAPI, []gomatrixserverlib.HeaderedEvent{*e}, cfg.Matrix.ServerName, nil); err != nil {
+	if err = roomserverAPI.SendEvents(context.Background(), rsAPI, api.KindNew, []gomatrixserverlib.HeaderedEvent{*e}, cfg.Matrix.ServerName, nil); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents")
 		return jsonerror.InternalServerError()
 	}
diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index 9744a5640..1303663ff 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -92,6 +92,7 @@ func SendEvent(
 	// event ID in case of duplicate transaction is discarded
 	if err := api.SendEvents(
 		req.Context(), rsAPI,
+		api.KindNew,
 		[]gomatrixserverlib.HeaderedEvent{
 			e.Headered(verRes.RoomVersion),
 		},
diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go
index b9575a284..272d3407d 100644
--- a/clientapi/threepid/invites.go
+++ b/clientapi/threepid/invites.go
@@ -361,6 +361,7 @@ func emit3PIDInviteEvent(
 
 	return api.SendEvents(
 		ctx, rsAPI,
+		api.KindNew,
 		[]gomatrixserverlib.HeaderedEvent{
 			(*event).Headered(queryRes.RoomVersion),
 		},
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index c637116f7..12f205366 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -290,6 +290,7 @@ func SendJoin(
 	if !alreadyJoined {
 		if err = api.SendEvents(
 			httpReq.Context(), rsAPI,
+			api.KindNew,
 			[]gomatrixserverlib.HeaderedEvent{
 				event.Headered(stateAndAuthChainResponse.RoomVersion),
 			},
diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go
index e16dfcc2e..fb81d9319 100644
--- a/federationapi/routing/leave.go
+++ b/federationapi/routing/leave.go
@@ -256,6 +256,7 @@ func SendLeave(
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
 	if err = api.SendEvents(
 		httpReq.Context(), rsAPI,
+		api.KindNew,
 		[]gomatrixserverlib.HeaderedEvent{
 			event.Headered(verRes.RoomVersion),
 		},
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 783fdc3b8..76dc3a2ee 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -403,6 +403,7 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) er
 	return api.SendEvents(
 		context.Background(),
 		t.rsAPI,
+		api.KindNew,
 		[]gomatrixserverlib.HeaderedEvent{
 			e.Headered(stateResp.RoomVersion),
 		},
@@ -586,6 +587,7 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser
 	err = api.SendEventWithState(
 		context.Background(),
 		t.rsAPI,
+		api.KindOld,
 		resolvedState,
 		backwardsExtremity.Headered(roomVersion),
 		t.haveEventIDs(),
@@ -605,6 +607,7 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser
 	if err = api.SendEvents(
 		context.Background(),
 		t.rsAPI,
+		api.KindOld,
 		append(headeredNewEvents, e.Headered(roomVersion)),
 		api.DoNotSendToOtherServers,
 		nil,
diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go
index ec6cc1488..4db5273af 100644
--- a/federationapi/routing/threepid.go
+++ b/federationapi/routing/threepid.go
@@ -89,7 +89,7 @@ func CreateInvitesFrom3PIDInvites(
 	}
 
 	// Send all the events
-	if err := api.SendEvents(req.Context(), rsAPI, evs, cfg.Matrix.ServerName, nil); err != nil {
+	if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, evs, cfg.Matrix.ServerName, nil); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
 		return jsonerror.InternalServerError()
 	}
@@ -174,6 +174,7 @@ func ExchangeThirdPartyInvite(
 	// Send the event to the roomserver
 	if err = api.SendEvents(
 		httpReq.Context(), rsAPI,
+		api.KindNew,
 		[]gomatrixserverlib.HeaderedEvent{
 			signedEvent.Event.Headered(verRes.RoomVersion),
 		},
diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go
index 254883e63..3904ab856 100644
--- a/federationsender/internal/perform.go
+++ b/federationsender/internal/perform.go
@@ -248,6 +248,7 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer(
 		// returned state to the roomserver to update our local view.
 		if err = roomserverAPI.SendEventWithState(
 			ctx, r.rsAPI,
+			roomserverAPI.KindNew,
 			respState,
 			event.Headered(respMakeJoin.RoomVersion),
 			nil,
diff --git a/roomserver/api/input.go b/roomserver/api/input.go
index dd693203b..e1a8afa00 100644
--- a/roomserver/api/input.go
+++ b/roomserver/api/input.go
@@ -21,17 +21,25 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
+type Kind int
+
 const (
 	// KindOutlier event fall outside the contiguous event graph.
 	// We do not have the state for these events.
 	// These events are state events used to authenticate other events.
 	// They can become part of the contiguous event graph via backfill.
-	KindOutlier = 1
+	KindOutlier Kind = iota + 1
 	// KindNew event extend the contiguous graph going forwards.
 	// They usually don't need state, but may include state if the
 	// there was a new event that references an event that we don't
-	// have a copy of.
-	KindNew = 2
+	// have a copy of. New events will influence the fwd extremities
+	// of the room and output events will be generated as a result.
+	KindNew
+	// KindOld event extend the graph backwards, or fill gaps in
+	// history. They may or may not include state. They will not be
+	// considered for forward extremities, and output events will NOT
+	// be generated for them.
+	KindOld
 )
 
 // DoNotSendToOtherServers tells us not to send the event to other matrix
@@ -43,7 +51,7 @@ const DoNotSendToOtherServers = ""
 type InputRoomEvent struct {
 	// Whether this event is new, backfilled or an outlier.
 	// This controls how the event is processed.
-	Kind int `json:"kind"`
+	Kind Kind `json:"kind"`
 	// The event JSON for the event to add.
 	Event gomatrixserverlib.HeaderedEvent `json:"event"`
 	// List of state event IDs that authenticate this event.
diff --git a/roomserver/api/output.go b/roomserver/api/output.go
index d57f3b04c..9cb814a47 100644
--- a/roomserver/api/output.go
+++ b/roomserver/api/output.go
@@ -24,6 +24,8 @@ type OutputType string
 const (
 	// OutputTypeNewRoomEvent indicates that the event is an OutputNewRoomEvent
 	OutputTypeNewRoomEvent OutputType = "new_room_event"
+	// OutputTypeOldRoomEvent indicates that the event is an OutputOldRoomEvent
+	OutputTypeOldRoomEvent OutputType = "old_room_event"
 	// OutputTypeNewInviteEvent indicates that the event is an OutputNewInviteEvent
 	OutputTypeNewInviteEvent OutputType = "new_invite_event"
 	// OutputTypeRetireInviteEvent indicates that the event is an OutputRetireInviteEvent
@@ -58,6 +60,8 @@ type OutputEvent struct {
 	Type OutputType `json:"type"`
 	// The content of event with type OutputTypeNewRoomEvent
 	NewRoomEvent *OutputNewRoomEvent `json:"new_room_event,omitempty"`
+	// The content of event with type OutputTypeOldRoomEvent
+	OldRoomEvent *OutputOldRoomEvent `json:"old_room_event,omitempty"`
 	// The content of event with type OutputTypeNewInviteEvent
 	NewInviteEvent *OutputNewInviteEvent `json:"new_invite_event,omitempty"`
 	// The content of event with type OutputTypeRetireInviteEvent
@@ -178,6 +182,20 @@ func (ore *OutputNewRoomEvent) AddsState() []gomatrixserverlib.HeaderedEvent {
 	return append(ore.AddStateEvents, ore.Event)
 }
 
+// An OutputOldRoomEvent is written when the roomserver receives an old event.
+// This will typically happen as a result of getting either missing events
+// or backfilling. Downstream components may wish to send these events to
+// clients when it is advantageous to do so, but with the consideration that
+// the event is likely a historic event.
+//
+// Old events do not update forward extremities or the current room state,
+// therefore they must not be treated as if they do. Downstream components
+// should build their current room state up from OutputNewRoomEvents only.
+type OutputOldRoomEvent struct {
+	// The Event.
+	Event gomatrixserverlib.HeaderedEvent `json:"event"`
+}
+
 // An OutputNewInviteEvent is written whenever an invite becomes active.
 // Invite events can be received outside of an existing room so have to be
 // tracked separately from the room events themselves.
diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go
index a38c00df7..9e8219103 100644
--- a/roomserver/api/wrapper.go
+++ b/roomserver/api/wrapper.go
@@ -24,13 +24,14 @@ import (
 
 // SendEvents to the roomserver The events are written with KindNew.
 func SendEvents(
-	ctx context.Context, rsAPI RoomserverInternalAPI, events []gomatrixserverlib.HeaderedEvent,
+	ctx context.Context, rsAPI RoomserverInternalAPI,
+	kind Kind, events []gomatrixserverlib.HeaderedEvent,
 	sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID,
 ) error {
 	ires := make([]InputRoomEvent, len(events))
 	for i, event := range events {
 		ires[i] = InputRoomEvent{
-			Kind:          KindNew,
+			Kind:          kind,
 			Event:         event,
 			AuthEventIDs:  event.AuthEventIDs(),
 			SendAsServer:  string(sendAsServer),
@@ -40,12 +41,13 @@ func SendEvents(
 	return SendInputRoomEvents(ctx, rsAPI, ires)
 }
 
-// SendEventWithState writes an event with KindNew to the roomserver
+// SendEventWithState writes an event with the specified kind to the roomserver
 // with the state at the event as KindOutlier before it. Will not send any event that is
 // marked as `true` in haveEventIDs
 func SendEventWithState(
-	ctx context.Context, rsAPI RoomserverInternalAPI, state *gomatrixserverlib.RespState,
-	event gomatrixserverlib.HeaderedEvent, haveEventIDs map[string]bool,
+	ctx context.Context, rsAPI RoomserverInternalAPI, kind Kind,
+	state *gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent,
+	haveEventIDs map[string]bool,
 ) error {
 	outliers, err := state.Events()
 	if err != nil {
@@ -70,7 +72,7 @@ func SendEventWithState(
 	}
 
 	ires = append(ires, InputRoomEvent{
-		Kind:          KindNew,
+		Kind:          kind,
 		Event:         event,
 		AuthEventIDs:  event.AuthEventIDs(),
 		HasState:      true,
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 113341591..67031609f 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -119,7 +119,7 @@ func (r *Inputer) processRoomEvent(
 		// We haven't calculated a state for this event yet.
 		// Lets calculate one.
 		err = r.calculateAndSetState(ctx, input, *roomInfo, &stateAtEvent, event, isRejected)
-		if err != nil {
+		if err != nil && input.Kind != api.KindOld {
 			return "", fmt.Errorf("r.calculateAndSetState: %w", err)
 		}
 	}
@@ -136,16 +136,31 @@ func (r *Inputer) processRoomEvent(
 		return event.EventID(), rejectionErr
 	}
 
-	if err = r.updateLatestEvents(
-		ctx,                 // context
-		roomInfo,            // room info for the room being updated
-		stateAtEvent,        // state at event (below)
-		event,               // event
-		input.SendAsServer,  // send as server
-		input.TransactionID, // transaction ID
-		input.HasState,      // rewrites state?
-	); err != nil {
-		return "", fmt.Errorf("r.updateLatestEvents: %w", err)
+	switch input.Kind {
+	case api.KindNew:
+		if err = r.updateLatestEvents(
+			ctx,                 // context
+			roomInfo,            // room info for the room being updated
+			stateAtEvent,        // state at event (below)
+			event,               // event
+			input.SendAsServer,  // send as server
+			input.TransactionID, // transaction ID
+			input.HasState,      // rewrites state?
+		); err != nil {
+			return "", fmt.Errorf("r.updateLatestEvents: %w", err)
+		}
+	case api.KindOld:
+		err = r.WriteOutputEvents(event.RoomID(), []api.OutputEvent{
+			{
+				Type: api.OutputTypeOldRoomEvent,
+				OldRoomEvent: &api.OutputOldRoomEvent{
+					Event: headered,
+				},
+			},
+		})
+		if err != nil {
+			return "", fmt.Errorf("r.WriteOutputEvents (old): %w", err)
+		}
 	}
 
 	// processing this event resulted in an event (which may not be the one we're processing)
@@ -163,7 +178,7 @@ func (r *Inputer) processRoomEvent(
 			},
 		})
 		if err != nil {
-			return "", fmt.Errorf("r.WriteOutputEvents: %w", err)
+			return "", fmt.Errorf("r.WriteOutputEvents (redactions): %w", err)
 		}
 	}
 
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index ca5d214d7..5adcd0877 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -164,8 +164,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 		return fmt.Errorf("u.api.updateMemberships: %w", err)
 	}
 
-	var update *api.OutputEvent
-	update, err = u.makeOutputNewRoomEvent()
+	update, err := u.makeOutputNewRoomEvent()
 	if err != nil {
 		return fmt.Errorf("u.makeOutputNewRoomEvent: %w", err)
 	}
@@ -259,6 +258,8 @@ func (u *latestEventsUpdater) latestState() error {
 	return nil
 }
 
+// calculateLatest works out the new set of forward extremities. Returns
+// true if the new event is included in those extremites, false otherwise.
 func (u *latestEventsUpdater) calculateLatest(
 	oldLatest []types.StateAtEventAndReference,
 	newEvent types.StateAtEventAndReference,
@@ -326,7 +327,6 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error)
 	if err != nil {
 		return nil, err
 	}
-
 	for _, entry := range u.added {
 		ore.AddsStateEventIDs = append(ore.AddsStateEventIDs, eventIDMap[entry.EventNID])
 	}
@@ -339,13 +339,14 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error)
 	for _, entry := range u.stateBeforeEventAdds {
 		ore.StateBeforeAddsEventIDs = append(ore.StateBeforeAddsEventIDs, eventIDMap[entry.EventNID])
 	}
+
 	ore.SendAsServer = u.sendAsServer
 
 	// include extra state events if they were added as nearly every downstream component will care about it
 	// and we'd rather not have them all hit QueryEventsByID at the same time!
 	if len(ore.AddsStateEventIDs) > 0 {
-		ore.AddStateEvents, err = u.extraEventsForIDs(u.roomInfo.RoomVersion, ore.AddsStateEventIDs)
-		if err != nil {
+		var err error
+		if ore.AddStateEvents, err = u.extraEventsForIDs(u.roomInfo.RoomVersion, ore.AddsStateEventIDs); err != nil {
 			return nil, fmt.Errorf("failed to load add_state_events from db: %w", err)
 		}
 	}
diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go
index 1b692a098..c8e60efa3 100644
--- a/roomserver/roomserver_test.go
+++ b/roomserver/roomserver_test.go
@@ -191,7 +191,7 @@ func mustSendEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []js
 	t.Helper()
 	rsAPI, dp := mustCreateRoomserverAPI(t)
 	hevents := mustLoadRawEvents(t, ver, events)
-	if err := api.SendEvents(ctx, rsAPI, hevents, testOrigin, nil); err != nil {
+	if err := api.SendEvents(ctx, rsAPI, api.KindNew, hevents, testOrigin, nil); err != nil {
 		t.Errorf("failed to SendEvents: %s", err)
 	}
 	return rsAPI, dp, hevents
@@ -337,7 +337,7 @@ func TestOutputRewritesState(t *testing.T) {
 	deleteDatabase()
 	rsAPI, producer := mustCreateRoomserverAPI(t)
 	defer deleteDatabase()
-	err := api.SendEvents(context.Background(), rsAPI, originalEvents, testOrigin, nil)
+	err := api.SendEvents(context.Background(), rsAPI, api.KindNew, originalEvents, testOrigin, nil)
 	if err != nil {
 		t.Fatalf("failed to send original events: %s", err)
 	}
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index ca48c8300..373baea54 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -97,6 +97,8 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 			}
 		}
 		return s.onNewRoomEvent(context.TODO(), *output.NewRoomEvent)
+	case api.OutputTypeOldRoomEvent:
+		return s.onOldRoomEvent(context.TODO(), *output.OldRoomEvent)
 	case api.OutputTypeNewInviteEvent:
 		return s.onNewInviteEvent(context.TODO(), *output.NewInviteEvent)
 	case api.OutputTypeRetireInviteEvent:
@@ -168,7 +170,40 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 			log.ErrorKey: err,
 			"add":        msg.AddsStateEventIDs,
 			"del":        msg.RemovesStateEventIDs,
-		}).Panicf("roomserver output log: write event failure")
+		}).Panicf("roomserver output log: write new event failure")
+		return nil
+	}
+
+	if pduPos, err = s.notifyJoinedPeeks(ctx, &ev, pduPos); err != nil {
+		logrus.WithError(err).Errorf("Failed to notifyJoinedPeeks for PDU pos %d", pduPos)
+		return err
+	}
+
+	s.notifier.OnNewEvent(&ev, "", nil, types.NewStreamToken(pduPos, 0, nil))
+
+	return nil
+}
+
+func (s *OutputRoomEventConsumer) onOldRoomEvent(
+	ctx context.Context, msg api.OutputOldRoomEvent,
+) error {
+	ev := msg.Event
+
+	pduPos, err := s.db.WriteEvent(
+		ctx,
+		&ev,
+		[]gomatrixserverlib.HeaderedEvent{},
+		[]string{}, // adds no state
+		[]string{}, // removes no state
+		nil,        // no transaction
+		false,      // not excluded from sync
+	)
+	if err != nil {
+		// panic rather than continue with an inconsistent database
+		log.WithFields(log.Fields{
+			"event":      string(ev.JSON()),
+			log.ErrorKey: err,
+		}).Panicf("roomserver output log: write old event failure")
 		return nil
 	}
 
diff --git a/sytest-whitelist b/sytest-whitelist
index 2ba0a88b2..2d0323318 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -484,3 +484,4 @@ Users cannot kick users who have already left a room
 A prev_batch token from incremental sync can be used in the v1 messages API
 Event with an invalid signature in the send_join response should not cause room join to fail
 Inbound federation rejects typing notifications from wrong remote
+Should not be able to take over the room by pretending there is no PL event

From 45abdcaeb967ea54e1edbe3c1f1d73d1a82a2904 Mon Sep 17 00:00:00 2001
From: Devon Johnson <djohnson1865@gmail.com>
Date: Sat, 17 Oct 2020 21:07:34 -0700
Subject: [PATCH 03/24] Send state after event, not current

Signed-off-by: Devon Johnson <djohnson1865@gmail.com>
---
 clientapi/routing/state.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go
index 2a424cbe8..f69b54bbc 100644
--- a/clientapi/routing/state.go
+++ b/clientapi/routing/state.go
@@ -141,7 +141,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 			util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser")
 			return jsonerror.InternalServerError()
 		}
-		for _, ev := range stateRes.StateEvents {
+		for _, ev := range stateAfterRes.StateEvents {
 			stateEvents = append(
 				stateEvents,
 				gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll),

From 5d80ff11a06d404250933d796bd96d05a9828007 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 19 Oct 2020 15:17:28 +0100
Subject: [PATCH 04/24] Update sytest-whitelist

---
 sytest-whitelist | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sytest-whitelist b/sytest-whitelist
index 2d0323318..5c57bb145 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -485,3 +485,4 @@ A prev_batch token from incremental sync can be used in the v1 messages API
 Event with an invalid signature in the send_join response should not cause room join to fail
 Inbound federation rejects typing notifications from wrong remote
 Should not be able to take over the room by pretending there is no PL event
+Can get rooms/{roomId}/state for a departed room (SPEC-216)

From c7bf122a26a53f4366884b1f912fdafd0501b610 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 19 Oct 2020 15:38:42 +0100
Subject: [PATCH 05/24] Update sytest lists

---
 sytest-blacklist | 5 ++++-
 sytest-whitelist | 1 -
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/sytest-blacklist b/sytest-blacklist
index ff7fdf7e0..f493f94fe 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -57,4 +57,7 @@ The only membership state included in a gapped incremental sync is for senders i
 # Blacklisted out of flakiness after #1479
 Invited user can reject local invite after originator leaves
 Invited user can reject invite for empty room
-If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes
\ No newline at end of file
+If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes
+
+# Blacklisted due to flakiness
+A prev_batch token from incremental sync can be used in the v1 messages API
\ No newline at end of file
diff --git a/sytest-whitelist b/sytest-whitelist
index 5c57bb145..cecf24f75 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -481,7 +481,6 @@ m.room.history_visibility == "joined" allows/forbids appropriately for Guest use
 m.room.history_visibility == "joined" allows/forbids appropriately for Real users
 POST rejects invalid utf-8 in JSON
 Users cannot kick users who have already left a room
-A prev_batch token from incremental sync can be used in the v1 messages API
 Event with an invalid signature in the send_join response should not cause room join to fail
 Inbound federation rejects typing notifications from wrong remote
 Should not be able to take over the room by pretending there is no PL event

From a71360d0992b4bc120e9a98af7755e72e2fc3eb5 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Oct 2020 11:22:46 +0100
Subject: [PATCH 06/24] Update INSTALL.md (#1549)

---
 docs/INSTALL.md | 45 ++++++++++++++++++---------------------------
 1 file changed, 18 insertions(+), 27 deletions(-)

diff --git a/docs/INSTALL.md b/docs/INSTALL.md
index be12d7b86..d246e9221 100644
--- a/docs/INSTALL.md
+++ b/docs/INSTALL.md
@@ -93,7 +93,7 @@ brew services start kafka
 ### SQLite database setup
 
 Dendrite can use the built-in SQLite database engine for small setups.
-The SQLite databases do not need to be preconfigured - Dendrite will
+The SQLite databases do not need to be pre-built - Dendrite will
 create them automatically at startup.
 
 ### Postgres database setup
@@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed:
 * Create the component databases:
 
   ```bash
-  for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do
+  for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do
       sudo -u postgres createdb -O dendrite dendrite_$i
   done
   ```
@@ -135,8 +135,8 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th
 
 * The `server_name` entry to reflect the hostname of your Dendrite server
 * The `database` lines with an updated connection string based on your
-  desired setup, e.g. replacing `component` with the name of the component:
-  * For Postgres: `postgres://dendrite:password@localhost/component`
+  desired setup, e.g. replacing `database` with the name of the database:
+  * For Postgres: `postgres://dendrite:password@localhost/database`
   * For SQLite on disk: `file:component.db` or `file:///path/to/component.db`
   * Postgres and SQLite can be mixed and matched.
 * The `use_naffka` option if using Naffka in a monolith deployment
@@ -147,6 +147,10 @@ then configuring `key_perspectives` (like `matrix.org` in the sample) can
 help to improve reliability considerably by allowing your homeserver to fetch
 public keys for dead homeservers from somewhere else.
 
+**WARNING:** Dendrite supports running all components from the same database in
+Postgres mode, but this is **NOT** a supported configuration with SQLite. When
+using SQLite, all components **MUST** use their own database file.
+
 ## Starting a monolith server
 
 It is possible to use Naffka as an in-process replacement to Kafka when using
@@ -167,30 +171,17 @@ as shown below, it will also listen for HTTPS connections on port 8448.
 
 The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite.
 
-### Client proxy
+### nginx (or other reverse proxy)
 
-This is what Matrix clients will talk to. If you use the script below, point
-your client at `http://localhost:8008`.
+This is what your clients and federated hosts will talk to. It must forward
+requests onto the correct API server based on URL:
 
-```bash
-./bin/client-api-proxy \
---bind-address ":8008" \
---client-api-server-url "http://localhost:7771" \
---sync-api-server-url "http://localhost:7773" \
---media-api-server-url "http://localhost:7774" \
-```
+* `/_matrix/client` to the client API server
+* `/_matrix/federation` to the federation API server
+* `/_matrix/key` to the federation API server
+* `/_matrix/media` to the media API server
 
-### Federation proxy
-
-This is what Matrix servers will talk to. This is only required if you want
-to support federation.
-
-```bash
-./bin/federation-api-proxy \
---bind-address ":8448" \
---federation-api-url "http://localhost:7772" \
---media-api-server-url "http://localhost:7774" \
-```
+See `docs/nginx/polylith-sample.conf` for a sample configuration.
 
 ### Client API server
 
@@ -198,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in
 order to send messages, create and join rooms, etc.
 
 ```bash
-./bin/dendrite-client-api-server --config=dendrite.yaml
+./bin/dendrite-client-api-server --config dendrite.yaml
 ```
 
 ### Sync server
@@ -239,7 +230,7 @@ contacted by other components. This includes the following components.
 This is what implements the room DAG. Clients do not talk to this.
 
 ```bash
-./bin/dendrite-room-server --config=dendrite.yaml
+./bin/dendrite-room-server --config dendrite.yaml
 ```
 
 #### Federation sender

From 92982a402ff553fe6d9179859a96ec33e2f09929 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Oct 2020 11:34:22 +0100
Subject: [PATCH 07/24] Update Docker (#1542)

* Separate Docker images, rather than tags

* Allow specifying tag to build/push/pull

* Include goose in Docker builds
---
 build/docker/Dockerfile.component        |  1 +
 build/docker/docker-compose.monolith.yml |  2 +-
 build/docker/docker-compose.polylith.yml | 61 +++++-------------------
 build/docker/images-build.sh             | 32 +++++++------
 build/docker/images-pull.sh              | 30 ++++++------
 build/docker/images-push.sh              | 30 ++++++------
 6 files changed, 62 insertions(+), 94 deletions(-)

diff --git a/build/docker/Dockerfile.component b/build/docker/Dockerfile.component
index 13634391a..1acf510fb 100644
--- a/build/docker/Dockerfile.component
+++ b/build/docker/Dockerfile.component
@@ -6,6 +6,7 @@ ARG component=monolith
 ENV entrypoint=${component}
 
 COPY --from=base /build/bin/${component} /usr/bin
+COPY --from=base /build/bin/goose /usr/bin
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml
index 336a43984..7d63e1c65 100644
--- a/build/docker/docker-compose.monolith.yml
+++ b/build/docker/docker-compose.monolith.yml
@@ -2,7 +2,7 @@ version: "3.4"
 services:
   monolith:
     hostname: monolith
-    image: matrixdotorg/dendrite:monolith
+    image: matrixdotorg/dendrite-monolith:latest
     command: [
       "--config=dendrite.yaml",
       "--tls-cert=server.crt",
diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml
index 8a4c50e06..e8da9c24a 100644
--- a/build/docker/docker-compose.polylith.yml
+++ b/build/docker/docker-compose.polylith.yml
@@ -1,28 +1,8 @@
 version: "3.4"
 services:
-  client_api_proxy:
-    hostname: client_api_proxy
-    image: matrixdotorg/dendrite:clientproxy
-    command: [
-      "--bind-address=:8008",
-      "--client-api-server-url=http://client_api:8071",
-      "--sync-api-server-url=http://sync_api:8073",
-      "--media-api-server-url=http://media_api:8074"
-    ]
-    volumes:
-      - ./config:/etc/dendrite
-    networks:
-      - internal
-    depends_on:
-      - sync_api
-      - client_api
-      - media_api
-    ports:
-      - "8008:8008"
-
   client_api:
     hostname: client_api
-    image: matrixdotorg/dendrite:clientapi
+    image: matrixdotorg/dendrite-clientapi:latest
     command: [
       "--config=dendrite.yaml"
     ]
@@ -34,7 +14,7 @@ services:
 
   media_api:
     hostname: media_api
-    image: matrixdotorg/dendrite:mediaapi
+    image: matrixdotorg/dendrite-mediaapi:latest
     command: [
       "--config=dendrite.yaml"
     ]
@@ -45,7 +25,7 @@ services:
 
   sync_api:
     hostname: sync_api
-    image: matrixdotorg/dendrite:syncapi
+    image: matrixdotorg/dendrite-syncapi:latest
     command: [
       "--config=dendrite.yaml"
     ]
@@ -56,7 +36,7 @@ services:
 
   room_server:
     hostname: room_server
-    image: matrixdotorg/dendrite:roomserver
+    image: matrixdotorg/dendrite-roomserver:latest
     command: [
       "--config=dendrite.yaml"
     ]
@@ -67,7 +47,7 @@ services:
 
   edu_server:
     hostname: edu_server
-    image: matrixdotorg/dendrite:eduserver
+    image: matrixdotorg/dendrite-eduserver:latest
     command: [
       "--config=dendrite.yaml"
     ]
@@ -76,28 +56,9 @@ services:
     networks:
       - internal
 
-  federation_api_proxy:
-    hostname: federation_api_proxy
-    image: matrixdotorg/dendrite:federationproxy
-    command: [
-      "--bind-address=:8448",
-      "--federation-api-url=http://federation_api:8072",
-      "--media-api-server-url=http://media_api:8074"
-    ]
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - federation_api
-      - federation_sender
-      - media_api
-    networks:
-      - internal
-    ports:
-      - "8448:8448"
-
   federation_api:
     hostname: federation_api
-    image: matrixdotorg/dendrite:federationapi
+    image: matrixdotorg/dendrite-federationapi:latest
     command: [
       "--config=dendrite.yaml"
     ]
@@ -108,7 +69,7 @@ services:
 
   federation_sender:
     hostname: federation_sender
-    image: matrixdotorg/dendrite:federationsender
+    image: matrixdotorg/dendrite-federationsender:latest
     command: [
       "--config=dendrite.yaml"
     ]
@@ -119,7 +80,7 @@ services:
       
   key_server:
     hostname: key_server
-    image: matrixdotorg/dendrite:keyserver
+    image: matrixdotorg/dendrite-keyserver:latest
     command: [
       "--config=dendrite.yaml"
     ]
@@ -130,7 +91,7 @@ services:
 
   signing_key_server:
     hostname: signing_key_server
-    image: matrixdotorg/dendrite:signingkeyserver
+    image: matrixdotorg/dendrite-signingkeyserver:latest
     command: [
       "--config=dendrite.yaml"
     ]
@@ -141,7 +102,7 @@ services:
 
   user_api:
     hostname: user_api
-    image: matrixdotorg/dendrite:userapi
+    image: matrixdotorg/dendrite-userapi:latest
     command: [
         "--config=dendrite.yaml"
     ]
@@ -152,7 +113,7 @@ services:
 
   appservice_api:
     hostname: appservice_api
-    image: matrixdotorg/dendrite:appservice
+    image: matrixdotorg/dendrite-appservice:latest
     command: [
       "--config=dendrite.yaml"
     ]
diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh
index d72bac214..daad63be0 100755
--- a/build/docker/images-build.sh
+++ b/build/docker/images-build.sh
@@ -2,20 +2,22 @@
 
 cd $(git rev-parse --show-toplevel)
 
-docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest .
+TAG=${1:-latest}
 
-docker build -t matrixdotorg/dendrite:monolith          --build-arg component=dendrite-monolith-server          -f build/docker/Dockerfile.component .
+echo "Building tag '${TAG}'"
 
-docker build -t matrixdotorg/dendrite:appservice        --build-arg component=dendrite-appservice-server        -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:clientapi         --build-arg component=dendrite-client-api-server        -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:clientproxy       --build-arg component=client-api-proxy                  -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:eduserver         --build-arg component=dendrite-edu-server               -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationapi     --build-arg component=dendrite-federation-api-server    -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationsender  --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationproxy   --build-arg component=federation-api-proxy              -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:keyserver         --build-arg component=dendrite-key-server               -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:mediaapi          --build-arg component=dendrite-media-api-server         -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:roomserver        --build-arg component=dendrite-room-server              -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:syncapi           --build-arg component=dendrite-sync-api-server          -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:signingkeyserver      --build-arg component=dendrite-signing-key-server    -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:userapi           --build-arg component=dendrite-user-api-server          -f build/docker/Dockerfile.component .
+docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
+
+docker build -t matrixdotorg/dendrite-monolith:${TAG}          --build-arg component=dendrite-monolith-server          -f build/docker/Dockerfile.component .
+
+docker build -t matrixdotorg/dendrite-appservice:${TAG}        --build-arg component=dendrite-appservice-server        -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-clientapi:${TAG}         --build-arg component=dendrite-client-api-server        -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-eduserver:${TAG}         --build-arg component=dendrite-edu-server               -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-federationapi:${TAG}     --build-arg component=dendrite-federation-api-server    -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-federationsender:${TAG}  --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-keyserver:${TAG}         --build-arg component=dendrite-key-server               -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-mediaapi:${TAG}          --build-arg component=dendrite-media-api-server         -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-roomserver:${TAG}        --build-arg component=dendrite-room-server              -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-syncapi:${TAG}           --build-arg component=dendrite-sync-api-server          -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-signingkeyserver:${TAG}  --build-arg component=dendrite-signing-key-server       -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-userapi:${TAG}           --build-arg component=dendrite-user-api-server          -f build/docker/Dockerfile.component .
diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh
index be9185464..e3284a2a6 100755
--- a/build/docker/images-pull.sh
+++ b/build/docker/images-pull.sh
@@ -1,17 +1,19 @@
 #!/bin/bash
 
-docker pull matrixdotorg/dendrite:monolith
+TAG=${1:-latest}
 
-docker pull matrixdotorg/dendrite:appservice
-docker pull matrixdotorg/dendrite:clientapi
-docker pull matrixdotorg/dendrite:clientproxy
-docker pull matrixdotorg/dendrite:eduserver
-docker pull matrixdotorg/dendrite:federationapi
-docker pull matrixdotorg/dendrite:federationsender
-docker pull matrixdotorg/dendrite:federationproxy
-docker pull matrixdotorg/dendrite:keyserver
-docker pull matrixdotorg/dendrite:mediaapi
-docker pull matrixdotorg/dendrite:roomserver
-docker pull matrixdotorg/dendrite:syncapi
-docker pull matrixdotorg/dendrite:signingkeyserver
-docker pull matrixdotorg/dendrite:userapi
+echo "Pulling tag '${TAG}'"
+
+docker pull matrixdotorg/dendrite-monolith:${TAG}
+
+docker pull matrixdotorg/dendrite-appservice:${TAG}
+docker pull matrixdotorg/dendrite-clientapi:${TAG}
+docker pull matrixdotorg/dendrite-eduserver:${TAG}
+docker pull matrixdotorg/dendrite-federationapi:${TAG}
+docker pull matrixdotorg/dendrite-federationsender:${TAG}
+docker pull matrixdotorg/dendrite-keyserver:${TAG}
+docker pull matrixdotorg/dendrite-mediaapi:${TAG}
+docker pull matrixdotorg/dendrite-roomserver:${TAG}
+docker pull matrixdotorg/dendrite-syncapi:${TAG}
+docker pull matrixdotorg/dendrite-signingkeyserver:${TAG}
+docker pull matrixdotorg/dendrite-userapi:${TAG}
diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh
index 64920171b..e4eb773ac 100755
--- a/build/docker/images-push.sh
+++ b/build/docker/images-push.sh
@@ -1,17 +1,19 @@
 #!/bin/bash
 
-docker push matrixdotorg/dendrite:monolith
+TAG=${1:-latest}
 
-docker push matrixdotorg/dendrite:appservice
-docker push matrixdotorg/dendrite:clientapi
-docker push matrixdotorg/dendrite:clientproxy
-docker push matrixdotorg/dendrite:eduserver
-docker push matrixdotorg/dendrite:federationapi
-docker push matrixdotorg/dendrite:federationsender
-docker push matrixdotorg/dendrite:federationproxy
-docker push matrixdotorg/dendrite:keyserver
-docker push matrixdotorg/dendrite:mediaapi
-docker push matrixdotorg/dendrite:roomserver
-docker push matrixdotorg/dendrite:syncapi
-docker push matrixdotorg/dendrite:signingkeyserver
-docker push matrixdotorg/dendrite:userapi
+echo "Pushing tag '${TAG}'"
+
+docker push matrixdotorg/dendrite-monolith:${TAG}
+
+docker push matrixdotorg/dendrite-appservice:${TAG}
+docker push matrixdotorg/dendrite-clientapi:${TAG}
+docker push matrixdotorg/dendrite-eduserver:${TAG}
+docker push matrixdotorg/dendrite-federationapi:${TAG}
+docker push matrixdotorg/dendrite-federationsender:${TAG}
+docker push matrixdotorg/dendrite-keyserver:${TAG}
+docker push matrixdotorg/dendrite-mediaapi:${TAG}
+docker push matrixdotorg/dendrite-roomserver:${TAG}
+docker push matrixdotorg/dendrite-syncapi:${TAG}
+docker push matrixdotorg/dendrite-signingkeyserver:${TAG}
+docker push matrixdotorg/dendrite-userapi:${TAG}

From eb86e2b336e8feaf54eb7fe688c896aa11bd5329 Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Tue, 20 Oct 2020 11:42:54 +0100
Subject: [PATCH 08/24] Fix sqlite locking bugs present on sytest (#1543)

* Fix sqite locking bugs present on sytest

Comments do the explaining.

* Fix deadlock in sqlite mode

Caused by starting a writer whilst within a writer

* Only complain about invalid state deltas for non-overwrite events

* Do not re-process outlier unnecessarily
---
 roomserver/internal/input/input_events.go     | 24 ++++++++++++++++++
 .../internal/input/input_latest_events.go     |  2 +-
 .../storage/shared/latest_events_updater.go   | 14 +++++------
 roomserver/storage/shared/storage.go          | 25 ++++++++++++++++---
 4 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 67031609f..6a5d9d264 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -17,6 +17,7 @@
 package input
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 
@@ -26,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )
 
@@ -44,6 +46,28 @@ func (r *Inputer) processRoomEvent(
 	headered := input.Event
 	event := headered.Unwrap()
 
+	// if we have already got this event then do not process it again, if the input kind is an outlier.
+	// Outliers contain no extra information which may warrant a re-processing.
+	if input.Kind == api.KindOutlier {
+		evs, err := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
+		if err == nil && len(evs) == 1 {
+			// check hash matches if we're on early room versions where the event ID was a random string
+			idFormat, err := headered.RoomVersion.EventIDFormat()
+			if err == nil {
+				switch idFormat {
+				case gomatrixserverlib.EventIDFormatV1:
+					if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {
+						util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
+						return event.EventID(), nil
+					}
+				default:
+					util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
+					return event.EventID(), nil
+				}
+			}
+		}
+	}
+
 	// Check that the event passes authentication checks and work out
 	// the numeric IDs for the auth events.
 	isRejected := false
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 5adcd0877..f76b0a0b4 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -233,7 +233,7 @@ func (u *latestEventsUpdater) latestState() error {
 	if err != nil {
 		return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
 	}
-	if len(u.removed) > len(u.added) {
+	if !u.stateAtEvent.Overwrite && len(u.removed) > len(u.added) {
 		// This really shouldn't happen.
 		// TODO: What is ultimately the best way to handle this situation?
 		logrus.Errorf(
diff --git a/roomserver/storage/shared/latest_events_updater.go b/roomserver/storage/shared/latest_events_updater.go
index b316f639d..8825dc464 100644
--- a/roomserver/storage/shared/latest_events_updater.go
+++ b/roomserver/storage/shared/latest_events_updater.go
@@ -70,16 +70,14 @@ func (u *LatestEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID {
 	return u.currentStateSnapshotNID
 }
 
-// StorePreviousEvents implements types.RoomRecentEventsUpdater
+// StorePreviousEvents implements types.RoomRecentEventsUpdater - This must be called from a Writer
 func (u *LatestEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error {
-	return u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
-		for _, ref := range previousEventReferences {
-			if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
-				return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
-			}
+	for _, ref := range previousEventReferences {
+		if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
+			return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
 		}
-		return nil
-	})
+	}
+	return nil
 }
 
 // IsReferenced implements types.RoomRecentEventsUpdater
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index f2be8b3cf..51dcb8887 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -492,15 +492,32 @@ func (d *Database) StoreEvent(
 		if roomInfo == nil && len(prevEvents) > 0 {
 			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID())
 		}
+		// Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of
+		// GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This
+		// function only does SELECTs though so the created txn (at this point) is just a read txn like
+		// any other so this is fine. If we ever update GetLatestEventsForUpdate or NewLatestEventsUpdater
+		// to do writes however then this will need to go inside `Writer.Do`.
 		updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo)
 		if err != nil {
 			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err)
 		}
-		if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
-			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err)
+		// Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents
+		// and EndTransaction in a writer then it's possible for a new write txn to be made between the two
+		// function calls which will then fail with 'database is locked'. This new write txn would HAVE to be
+		// something like SetRoomAlias/RemoveRoomAlias as normal input events are already done sequentially due to
+		// SupportsConcurrentRoomInputs() == false on sqlite, though this does not apply to setting room aliases
+		// as they don't go via InputRoomEvents
+		err = d.Writer.Do(d.DB, updater.txn, func(txn *sql.Tx) error {
+			if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
+				return fmt.Errorf("updater.StorePreviousEvents: %w", err)
+			}
+			succeeded := true
+			err = sqlutil.EndTransaction(updater, &succeeded)
+			return err
+		})
+		if err != nil {
+			return 0, types.StateAtEvent{}, nil, "", err
 		}
-		succeeded := true
-		err = sqlutil.EndTransaction(updater, &succeeded)
 	}
 
 	return roomNID, types.StateAtEvent{

From 53a745f3332c4bb35dddd80ff8f83e7eb11456ca Mon Sep 17 00:00:00 2001
From: Pika <15848969+ThatNerdyPikachu@users.noreply.github.com>
Date: Tue, 20 Oct 2020 06:47:37 -0400
Subject: [PATCH 09/24] fix create-account (#1546)

---
 cmd/create-account/main.go | 28 +---------------------------
 1 file changed, 1 insertion(+), 27 deletions(-)

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index a9bd92794..f6de2d0d4 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -22,7 +22,6 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
-	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -39,7 +38,6 @@ var (
 	username      = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.")
 	password      = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.")
 	serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.")
-	accessToken   = flag.String("token", "", "Optional. The desired access_token to have. If not specified, a random access_token will be made.")
 )
 
 func main() {
@@ -78,29 +76,5 @@ func main() {
 		os.Exit(1)
 	}
 
-	deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{
-		ConnectionString: config.DataSource(*database),
-	}, serverName)
-	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(1)
-	}
-
-	if *accessToken == "" {
-		t := "token_" + *username
-		accessToken = &t
-	}
-
-	device, err := deviceDB.CreateDevice(
-		context.Background(), *username, nil, *accessToken, nil, "127.0.0.1", "",
-	)
-	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(1)
-	}
-
-	fmt.Println("Created account:")
-	fmt.Printf("user_id      = %s\n", device.UserID)
-	fmt.Printf("device_id    = %s\n", device.ID)
-	fmt.Printf("access_token = %s\n", device.AccessToken)
+	fmt.Println("Created account")
 }

From 837c295c26aa42faf1087368d05ceca2e42d95a0 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 20 Oct 2020 12:29:53 +0100
Subject: [PATCH 10/24] Linting

---
 roomserver/internal/input/input_events.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 6a5d9d264..c055289c9 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -49,11 +49,11 @@ func (r *Inputer) processRoomEvent(
 	// if we have already got this event then do not process it again, if the input kind is an outlier.
 	// Outliers contain no extra information which may warrant a re-processing.
 	if input.Kind == api.KindOutlier {
-		evs, err := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
-		if err == nil && len(evs) == 1 {
+		evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
+		if err2 == nil && len(evs) == 1 {
 			// check hash matches if we're on early room versions where the event ID was a random string
-			idFormat, err := headered.RoomVersion.EventIDFormat()
-			if err == nil {
+			idFormat, err2 := headered.RoomVersion.EventIDFormat()
+			if err2 == nil {
 				switch idFormat {
 				case gomatrixserverlib.EventIDFormatV1:
 					if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {

From 6c3c621de09a62c7ec8d89b2d8e6a9d73833fd0b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Oct 2020 12:36:16 +0100
Subject: [PATCH 11/24] Remove invalid state delta check (#1550)

---
 roomserver/internal/input/input_latest_events.go | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index f76b0a0b4..5631959b7 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -233,18 +233,6 @@ func (u *latestEventsUpdater) latestState() error {
 	if err != nil {
 		return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
 	}
-	if !u.stateAtEvent.Overwrite && len(u.removed) > len(u.added) {
-		// This really shouldn't happen.
-		// TODO: What is ultimately the best way to handle this situation?
-		logrus.Errorf(
-			"Invalid state delta on event %q wants to remove %d state but only add %d state (between state snapshots %d and %d)",
-			u.event.EventID(), len(u.removed), len(u.added), u.oldStateNID, u.newStateNID,
-		)
-		u.added = u.added[:0]
-		u.removed = u.removed[:0]
-		u.newStateNID = u.oldStateNID
-		return nil
-	}
 
 	// Also work out the state before the event removes and the event
 	// adds.

From 39c7a8915ccba3ac4d281ae7938f2e912ad966b4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Oct 2020 16:11:24 +0100
Subject: [PATCH 12/24] Multi-personality polylith binary (#1552)

* Initial work oon multipersonality binary

* Remove old binaries

* Monolith and polylith binaries

* Better logging

* dendrite-poly-multi

* Fix path

* Copyright notices etc

* Tweaks

* Update Docker, INSTALL.md

* Take first argument if flags package doesn't find any args

* Postgres 9.6 or later, fix some more Docker stuff

* Don't create unnecessary e2ekey DB

* Run go mod tidy
---
 build/docker/Dockerfile                       |  2 +-
 build/docker/Dockerfile.component             | 14 ----
 build/docker/Dockerfile.monolith              | 13 ++++
 build/docker/Dockerfile.polylith              | 13 ++++
 build/docker/docker-compose.deps.yml          |  2 +-
 build/docker/docker-compose.monolith.yml      |  1 -
 build/docker/docker-compose.polylith.yml      | 67 ++++++----------
 build/docker/images-build.sh                  | 15 +---
 build/docker/images-pull.sh                   | 13 +---
 build/docker/images-push.sh                   | 13 +---
 build/docker/postgres/create_db.sh            |  2 +-
 cmd/dendrite-polylith-multi/main.go           | 78 +++++++++++++++++++
 .../personalities/appservice.go}              | 11 +--
 .../personalities/clientapi.go}               | 12 +--
 .../personalities/eduserver.go}               | 18 ++---
 .../personalities/federationapi.go}           | 11 +--
 .../personalities/federationsender.go}        | 11 +--
 .../personalities/keyserver.go}               |  9 +--
 .../personalities/mediaapi.go}                | 11 +--
 .../personalities/roomserver.go}              | 11 +--
 .../personalities/signingkeyserver.go}        |  9 +--
 .../personalities/syncapi.go}                 | 12 +--
 .../personalities/userapi.go}                 | 11 +--
 docs/INSTALL.md                               | 26 +++----
 24 files changed, 189 insertions(+), 196 deletions(-)
 delete mode 100644 build/docker/Dockerfile.component
 create mode 100644 build/docker/Dockerfile.monolith
 create mode 100644 build/docker/Dockerfile.polylith
 create mode 100644 cmd/dendrite-polylith-multi/main.go
 rename cmd/{dendrite-appservice-server/main.go => dendrite-polylith-multi/personalities/appservice.go} (83%)
 rename cmd/{dendrite-client-api-server/main.go => dendrite-polylith-multi/personalities/clientapi.go} (87%)
 rename cmd/{dendrite-edu-server/main.go => dendrite-polylith-multi/personalities/eduserver.go} (77%)
 rename cmd/{dendrite-federation-api-server/main.go => dendrite-polylith-multi/personalities/federationapi.go} (86%)
 rename cmd/{dendrite-federation-sender-server/main.go => dendrite-polylith-multi/personalities/federationsender.go} (85%)
 rename cmd/{dendrite-key-server/main.go => dendrite-polylith-multi/personalities/keyserver.go} (87%)
 rename cmd/{dendrite-media-api-server/main.go => dendrite-polylith-multi/personalities/mediaapi.go} (82%)
 rename cmd/{dendrite-room-server/main.go => dendrite-polylith-multi/personalities/roomserver.go} (84%)
 rename cmd/{dendrite-signing-key-server/main.go => dendrite-polylith-multi/personalities/signingkeyserver.go} (86%)
 rename cmd/{dendrite-sync-api-server/main.go => dendrite-polylith-multi/personalities/syncapi.go} (84%)
 rename cmd/{dendrite-user-api-server/main.go => dendrite-polylith-multi/personalities/userapi.go} (84%)

diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
index d8e07681f..5cab0530f 100644
--- a/build/docker/Dockerfile
+++ b/build/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.13.7-alpine3.11 AS builder
+FROM docker.io/golang:1.15-alpine AS builder
 
 RUN apk --update --no-cache add bash build-base
 
diff --git a/build/docker/Dockerfile.component b/build/docker/Dockerfile.component
deleted file mode 100644
index 1acf510fb..000000000
--- a/build/docker/Dockerfile.component
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM matrixdotorg/dendrite:latest AS base
-
-FROM alpine:latest
-
-ARG component=monolith
-ENV entrypoint=${component}
-
-COPY --from=base /build/bin/${component} /usr/bin
-COPY --from=base /build/bin/goose /usr/bin
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT /usr/bin/${entrypoint} $@
\ No newline at end of file
diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith
new file mode 100644
index 000000000..3e9d0cba4
--- /dev/null
+++ b/build/docker/Dockerfile.monolith
@@ -0,0 +1,13 @@
+FROM matrixdotorg/dendrite:latest AS base
+
+FROM alpine:latest
+
+COPY --from=base /build/bin/dendrite-monolith-server /usr/bin
+COPY --from=base /build/bin/goose /usr/bin
+COPY --from=base /build/bin/create-account /usr/bin
+COPY --from=base /build/bin/generate-keys /usr/bin
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
\ No newline at end of file
diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith
new file mode 100644
index 000000000..dd4cbd38f
--- /dev/null
+++ b/build/docker/Dockerfile.polylith
@@ -0,0 +1,13 @@
+FROM matrixdotorg/dendrite:latest AS base
+
+FROM alpine:latest
+
+COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin
+COPY --from=base /build/bin/goose /usr/bin
+COPY --from=base /build/bin/create-account /usr/bin
+COPY --from=base /build/bin/generate-keys /usr/bin
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
\ No newline at end of file
diff --git a/build/docker/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml
index 74e478a8d..1a27ffac0 100644
--- a/build/docker/docker-compose.deps.yml
+++ b/build/docker/docker-compose.deps.yml
@@ -2,7 +2,7 @@ version: "3.4"
 services:
   postgres:
     hostname: postgres
-    image: postgres:9.5
+    image: postgres:9.6
     restart: always
     volumes:
       - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml
index 7d63e1c65..8fb798343 100644
--- a/build/docker/docker-compose.monolith.yml
+++ b/build/docker/docker-compose.monolith.yml
@@ -4,7 +4,6 @@ services:
     hostname: monolith
     image: matrixdotorg/dendrite-monolith:latest
     command: [
-      "--config=dendrite.yaml",
       "--tls-cert=server.crt",
       "--tls-key=server.key"
     ]
diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml
index e8da9c24a..f377e36fc 100644
--- a/build/docker/docker-compose.polylith.yml
+++ b/build/docker/docker-compose.polylith.yml
@@ -2,22 +2,17 @@ version: "3.4"
 services:
   client_api:
     hostname: client_api
-    image: matrixdotorg/dendrite-clientapi:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: clientapi
     volumes:
       - ./config:/etc/dendrite
-      - room_server
     networks:
       - internal
 
   media_api:
     hostname: media_api
-    image: matrixdotorg/dendrite-mediaapi:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: mediaapi
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -25,10 +20,8 @@ services:
 
   sync_api:
     hostname: sync_api
-    image: matrixdotorg/dendrite-syncapi:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: syncapi
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -36,10 +29,8 @@ services:
 
   room_server:
     hostname: room_server
-    image: matrixdotorg/dendrite-roomserver:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: roomserver
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -47,10 +38,8 @@ services:
 
   edu_server:
     hostname: edu_server
-    image: matrixdotorg/dendrite-eduserver:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: eduserver
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -58,10 +47,8 @@ services:
 
   federation_api:
     hostname: federation_api
-    image: matrixdotorg/dendrite-federationapi:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: federationapi
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -69,10 +56,8 @@ services:
 
   federation_sender:
     hostname: federation_sender
-    image: matrixdotorg/dendrite-federationsender:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: federationsender
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -80,10 +65,8 @@ services:
       
   key_server:
     hostname: key_server
-    image: matrixdotorg/dendrite-keyserver:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: keyserver
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -91,10 +74,8 @@ services:
 
   signing_key_server:
     hostname: signing_key_server
-    image: matrixdotorg/dendrite-signingkeyserver:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: signingkeyserver
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -102,10 +83,8 @@ services:
 
   user_api:
     hostname: user_api
-    image: matrixdotorg/dendrite-userapi:latest
-    command: [
-        "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: userapi
     volumes:
         - ./config:/etc/dendrite
     networks:
@@ -113,10 +92,8 @@ services:
 
   appservice_api:
     hostname: appservice_api
-    image: matrixdotorg/dendrite-appservice:latest
-    command: [
-      "--config=dendrite.yaml"
-    ]
+    image: matrixdotorg/dendrite-polylith:latest
+    command: appservice
     volumes:
       - ./config:/etc/dendrite
     networks:
diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh
index daad63be0..f80f6bed2 100755
--- a/build/docker/images-build.sh
+++ b/build/docker/images-build.sh
@@ -8,16 +8,5 @@ echo "Building tag '${TAG}'"
 
 docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
 
-docker build -t matrixdotorg/dendrite-monolith:${TAG}          --build-arg component=dendrite-monolith-server          -f build/docker/Dockerfile.component .
-
-docker build -t matrixdotorg/dendrite-appservice:${TAG}        --build-arg component=dendrite-appservice-server        -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-clientapi:${TAG}         --build-arg component=dendrite-client-api-server        -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-eduserver:${TAG}         --build-arg component=dendrite-edu-server               -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-federationapi:${TAG}     --build-arg component=dendrite-federation-api-server    -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-federationsender:${TAG}  --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-keyserver:${TAG}         --build-arg component=dendrite-key-server               -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-mediaapi:${TAG}          --build-arg component=dendrite-media-api-server         -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-roomserver:${TAG}        --build-arg component=dendrite-room-server              -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-syncapi:${TAG}           --build-arg component=dendrite-sync-api-server          -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-signingkeyserver:${TAG}  --build-arg component=dendrite-signing-key-server       -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite-userapi:${TAG}           --build-arg component=dendrite-user-api-server          -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-monolith:${TAG}  -f build/docker/Dockerfile.monolith .
+docker build -t matrixdotorg/dendrite-polylith:${TAG}  -f build/docker/Dockerfile.polylith .
\ No newline at end of file
diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh
index e3284a2a6..496e80067 100755
--- a/build/docker/images-pull.sh
+++ b/build/docker/images-pull.sh
@@ -5,15 +5,4 @@ TAG=${1:-latest}
 echo "Pulling tag '${TAG}'"
 
 docker pull matrixdotorg/dendrite-monolith:${TAG}
-
-docker pull matrixdotorg/dendrite-appservice:${TAG}
-docker pull matrixdotorg/dendrite-clientapi:${TAG}
-docker pull matrixdotorg/dendrite-eduserver:${TAG}
-docker pull matrixdotorg/dendrite-federationapi:${TAG}
-docker pull matrixdotorg/dendrite-federationsender:${TAG}
-docker pull matrixdotorg/dendrite-keyserver:${TAG}
-docker pull matrixdotorg/dendrite-mediaapi:${TAG}
-docker pull matrixdotorg/dendrite-roomserver:${TAG}
-docker pull matrixdotorg/dendrite-syncapi:${TAG}
-docker pull matrixdotorg/dendrite-signingkeyserver:${TAG}
-docker pull matrixdotorg/dendrite-userapi:${TAG}
+docker pull matrixdotorg/dendrite-polylith:${TAG}
\ No newline at end of file
diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh
index e4eb773ac..fd9b999ea 100755
--- a/build/docker/images-push.sh
+++ b/build/docker/images-push.sh
@@ -5,15 +5,4 @@ TAG=${1:-latest}
 echo "Pushing tag '${TAG}'"
 
 docker push matrixdotorg/dendrite-monolith:${TAG}
-
-docker push matrixdotorg/dendrite-appservice:${TAG}
-docker push matrixdotorg/dendrite-clientapi:${TAG}
-docker push matrixdotorg/dendrite-eduserver:${TAG}
-docker push matrixdotorg/dendrite-federationapi:${TAG}
-docker push matrixdotorg/dendrite-federationsender:${TAG}
-docker push matrixdotorg/dendrite-keyserver:${TAG}
-docker push matrixdotorg/dendrite-mediaapi:${TAG}
-docker push matrixdotorg/dendrite-roomserver:${TAG}
-docker push matrixdotorg/dendrite-syncapi:${TAG}
-docker push matrixdotorg/dendrite-signingkeyserver:${TAG}
-docker push matrixdotorg/dendrite-userapi:${TAG}
+docker push matrixdotorg/dendrite-polylith:${TAG}
\ No newline at end of file
diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh
index 97514467b..7495a3978 100755
--- a/build/docker/postgres/create_db.sh
+++ b/build/docker/postgres/create_db.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
 
-for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice e2ekey naffka; do
+for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice naffka; do
     createdb -U dendrite -O dendrite dendrite_$db
 done
diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go
new file mode 100644
index 000000000..0d6406c01
--- /dev/null
+++ b/cmd/dendrite-polylith-multi/main.go
@@ -0,0 +1,78 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"strings"
+
+	"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/sirupsen/logrus"
+)
+
+type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite)
+
+func main() {
+	cfg := setup.ParseFlags(true)
+
+	component := ""
+	if flag.NFlag() > 0 {
+		component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi
+	} else if len(os.Args) > 1 {
+		component = os.Args[1] // ./dendrite-polylith-multi clientapi
+	}
+
+	components := map[string]entrypoint{
+		"appservice":       personalities.Appservice,
+		"clientapi":        personalities.ClientAPI,
+		"eduserver":        personalities.EDUServer,
+		"federationapi":    personalities.FederationAPI,
+		"federationsender": personalities.FederationSender,
+		"keyserver":        personalities.KeyServer,
+		"mediaapi":         personalities.MediaAPI,
+		"roomserver":       personalities.RoomServer,
+		"signingkeyserver": personalities.SigningKeyServer,
+		"syncapi":          personalities.SyncAPI,
+		"userapi":          personalities.UserAPI,
+	}
+
+	start, ok := components[component]
+	if !ok {
+		if component == "" {
+			logrus.Errorf("No component specified")
+			logrus.Info("The first argument on the command line must be the name of the component to run")
+		} else {
+			logrus.Errorf("Unknown component %q specified", component)
+		}
+
+		var list []string
+		for c := range components {
+			list = append(list, c)
+		}
+		logrus.Infof("Valid components: %s", strings.Join(list, ", "))
+
+		os.Exit(1)
+	}
+
+	logrus.Infof("Starting %q component", component)
+
+	base := setup.NewBaseDendrite(cfg, component, false) // TODO
+	defer base.Close()                                   // nolint: errcheck
+
+	start(base, cfg)
+}
diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-polylith-multi/personalities/appservice.go
similarity index 83%
rename from cmd/dendrite-appservice-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/appservice.go
index 6adbdb17c..7fa87b115 100644
--- a/cmd/dendrite-appservice-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/appservice.go
@@ -1,4 +1,4 @@
-// Copyright 2018 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true)
-
-	defer base.Close() // nolint: errcheck
+func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	rsAPI := base.RoomserverHTTPClient()
 
diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go
similarity index 87%
rename from cmd/dendrite-client-api-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/clientapi.go
index 0061de74f..09fc63ab3 100644
--- a/cmd/dendrite-client-api-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,20 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/internal/transactions"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-
-	base := setup.NewBaseDendrite(cfg, "ClientAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	accountDB := base.CreateAccountsDB()
 	federation := base.CreateFederationClient()
 
diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-polylith-multi/personalities/eduserver.go
similarity index 77%
rename from cmd/dendrite-edu-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/eduserver.go
index 3a34b9a68..a5d2926f1 100644
--- a/cmd/dendrite-edu-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/eduserver.go
@@ -1,3 +1,5 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -10,26 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
-	_ "net/http/pprof"
-
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/sirupsen/logrus"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true)
-	defer func() {
-		if err := base.Close(); err != nil {
-			logrus.WithError(err).Warn("BaseDendrite close failed")
-		}
-	}()
-
+func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
 	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go
similarity index 86%
rename from cmd/dendrite-federation-api-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/federationapi.go
index 3ebb16f4b..a1bbeafad 100644
--- a/cmd/dendrite-federation-api-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "FederationAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	federation := base.CreateFederationClient()
 	serverKeyAPI := base.SigningKeyServerHTTPClient()
diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-polylith-multi/personalities/federationsender.go
similarity index 85%
rename from cmd/dendrite-federation-sender-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/federationsender.go
index 99b416c45..052523789 100644
--- a/cmd/dendrite-federation-sender-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/federationsender.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "FederationSender", true)
-	defer base.Close() // nolint: errcheck
-
+func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	federation := base.CreateFederationClient()
 
 	serverKeyAPI := base.SigningKeyServerHTTPClient()
diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-polylith-multi/personalities/keyserver.go
similarity index 87%
rename from cmd/dendrite-key-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/keyserver.go
index ff5b22236..8c159ad06 100644
--- a/cmd/dendrite-key-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/keyserver.go
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "KeyServer", true)
-	defer base.Close() // nolint: errcheck
-
+func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
 	intAPI.SetUserAPI(base.UserAPIClient())
 
diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/mediaapi.go
similarity index 82%
rename from cmd/dendrite-media-api-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/mediaapi.go
index 2c2fe3b36..64e5bc312 100644
--- a/cmd/dendrite-media-api-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/mediaapi.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/mediaapi"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "MediaAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	client := base.CreateClient()
 
diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-polylith-multi/personalities/roomserver.go
similarity index 84%
rename from cmd/dendrite-room-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/roomserver.go
index d3f145745..91027506d 100644
--- a/cmd/dendrite-room-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/roomserver.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/roomserver"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	serverKeyAPI := base.SigningKeyServerHTTPClient()
 	keyRing := serverKeyAPI.KeyRing()
 
diff --git a/cmd/dendrite-signing-key-server/main.go b/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go
similarity index 86%
rename from cmd/dendrite-signing-key-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/signingkeyserver.go
index a4d48d361..a7bfff10b 100644
--- a/cmd/dendrite-signing-key-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/signingkeyserver"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "SigningKeyServer", true)
-	defer base.Close() // nolint: errcheck
-
+func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	federation := base.CreateFederationClient()
 
 	intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches)
diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go
similarity index 84%
rename from cmd/dendrite-sync-api-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/syncapi.go
index 351dbc5f4..2d5c0b525 100644
--- a/cmd/dendrite-sync-api-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/syncapi.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,19 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/syncapi"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-
-	base := setup.NewBaseDendrite(cfg, "SyncAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	federation := base.CreateFederationClient()
 
diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/userapi.go
similarity index 84%
rename from cmd/dendrite-user-api-server/main.go
rename to cmd/dendrite-polylith-multi/personalities/userapi.go
index fb65fefbc..fe5e4fbd0 100644
--- a/cmd/dendrite-user-api-server/main.go
+++ b/cmd/dendrite-polylith-multi/personalities/userapi.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/userapi"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "UserAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	accountDB := base.CreateAccountsDB()
 
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())
diff --git a/docs/INSTALL.md b/docs/INSTALL.md
index d246e9221..f804193cd 100644
--- a/docs/INSTALL.md
+++ b/docs/INSTALL.md
@@ -24,7 +24,7 @@ use in production environments just yet!
 Dendrite requires:
 
 * Go 1.13 or higher
-* Postgres 9.5 or higher (if using Postgres databases, not needed for SQLite)
+* Postgres 9.6 or higher (if using Postgres databases, not needed for SQLite)
 
 If you want to run a polylith deployment, you also need:
 
@@ -98,7 +98,7 @@ create them automatically at startup.
 
 ### Postgres database setup
 
-Assuming that Postgres 9.5 (or later) is installed:
+Assuming that Postgres 9.6 (or later) is installed:
 
 * Create role, choosing a new password when prompted:
 
@@ -189,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in
 order to send messages, create and join rooms, etc.
 
 ```bash
-./bin/dendrite-client-api-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml clientapi
 ```
 
 ### Sync server
@@ -198,7 +198,7 @@ This is what implements `/sync` requests. Clients talk to this via the proxy
 in order to receive messages.
 
 ```bash
-./bin/dendrite-sync-api-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml syncapi
 ```
 
 ### Media server
@@ -207,7 +207,7 @@ This implements `/media` requests. Clients talk to this via the proxy in
 order to upload and retrieve media.
 
 ```bash
-./bin/dendrite-media-api-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml mediaapi
 ```
 
 ### Federation API server
@@ -217,7 +217,7 @@ order to send transactions.  This is only required if you want to support
 federation.
 
 ```bash
-./bin/dendrite-federation-api-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml federationapi
 ```
 
 ### Internal components
@@ -230,7 +230,7 @@ contacted by other components. This includes the following components.
 This is what implements the room DAG. Clients do not talk to this.
 
 ```bash
-./bin/dendrite-room-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml roomserver
 ```
 
 #### Federation sender
@@ -239,7 +239,7 @@ This sends events from our users to other servers.  This is only required if
 you want to support federation.
 
 ```bash
-./bin/dendrite-federation-sender-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml federationsender
 ```
 
 #### Appservice server
@@ -250,7 +250,7 @@ running locally.  This is only required if you want to support running
 application services on your homeserver.
 
 ```bash
-./bin/dendrite-appservice-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml appservice
 ```
 
 #### Key server
@@ -258,7 +258,7 @@ application services on your homeserver.
 This manages end-to-end encryption keys for users.
 
 ```bash
-./bin/dendrite-key-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver
 ```
 
 #### Signing key server
@@ -266,7 +266,7 @@ This manages end-to-end encryption keys for users.
 This manages signing keys for servers.
 
 ```bash
-./bin/dendrite-signing-key-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml signingkeyserver
 ```
 
 #### EDU server
@@ -274,7 +274,7 @@ This manages signing keys for servers.
 This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to
 
 ```bash
-./bin/dendrite-edu-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver
 ```
 
 #### User server
@@ -283,6 +283,6 @@ This manages user accounts, device access tokens and user account data,
 amongst other things.
 
 ```bash
-./bin/dendrite-user-api-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml userapi
 ```
 

From 6a16d46fba9edf82203b2b3606d9a16ed6ef4ae0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Oct 2020 16:46:53 +0100
Subject: [PATCH 13/24] Version 0.2.0 (#1551)

* v0.2.0-rc2

* Update CHANGES.md (also with some markdown lint suggestions)

* Update version number

* Update CHANGES.md

* Update CHANGES.md

* Update CHANGES.md

* Add known issue

* Update CHANGES.md
---
 CHANGES.md          | 194 +++++++++++++++++++++++++++++---------------
 internal/version.go |   2 +-
 2 files changed, 128 insertions(+), 68 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 17fb75bed..d05b871ac 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,89 +1,149 @@
-# Dendrite 0.1.0 (2020-10-08)
+# Changelog
+
+## Dendrite 0.2.0 (2020-10-20)
+
+### Important
+
+* This release makes breaking changes for polylith deployments, since they now use the multi-personality binary rather than separate binary files
+  * Users of polylith deployments should revise their setups to use the new binary - see the Features section below
+* This release also makes breaking changes for Docker deployments, as are now publishing images to Docker Hub in separate repositories for monolith and polylith
+  * New repositories are as follows: [matrixdotorg/dendrite-monolith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-monolith) and [matrixdotorg/dendrite-polylith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-polylith)
+  * The new `latest` tag will be updated with the latest release, and new versioned tags, e.g. `v0.2.0`, will preserve specific release versions
+  * [Sample Compose configs](https://github.com/matrix-org/dendrite/tree/master/build/docker) have been updated - if you are running a Docker deployment, please review the changes
+  * Images for the client API proxy and federation API proxy are no longer provided as they are unsupported - please use [nginx](docs/nginx/) (or another reverse proxy) instead
+
+### Features
+
+* Dendrite polylith deployments now use a special multi-personality binary, rather than separate binaries
+  * This is cleaner, builds faster and simplifies deployment
+  * The first command line argument states the component to run, e.g. `./dendrite-polylith-multi roomserver`
+* Database migrations are now run at startup
+* Invalid UTF-8 in requests is now rejected (contributed by [Pestdoktor](https://github.com/Pestdoktor))
+* Fully read markers are now implemented in the client API (contributed by [Lesterpig](https://github.com/Lesterpig))
+* Missing auth events are now retrieved from other servers in the room, rather than just the event origin
+* `m.room.create` events are now validated properly when processing a `/send_join` response
+* The roomserver now implements `KindOld` for handling historic events without them becoming forward extremity candidates, i.e. for backfilled or missing events
+
+### Fixes
+
+* State resolution v2 performance has been improved dramatically when dealing with large state sets
+* The roomserver no longer processes outlier events if they are already known
+* A SQLite locking issue in the previous events updater has been fixed
+* The client API `/state` endpoint now correctly returns state after the leave event, if the user has left the room
+* The client API `/createRoom` endpoint now sends cumulative state to the roomserver for the initial room events
+* The federation API `/send` endpoint now correctly requests the entire room state from the roomserver when needed
+* Some internal HTTP API paths have been fixed in the user API (contributed by [S7evinK](https://github.com/S7evinK))
+* A race condition in the rate limiting code resulting in concurrent map writes has been fixed
+* Each component now correctly starts a consumer/producer connection in monolith mode (when using Kafka)
+* State resolution is no longer run for single trusted state snapshots that have been verified before
+* A crash when rolling back the transaction in the latest events updater has been fixed
+* Typing events are now ignored when the sender domain does not match the origin server
+* Duplicate redaction entries no longer result in database errors
+* Recursion has been removed from the code path for retrieving missing events
+* `QueryMissingAuthPrevEvents` now returns events that have no associated state as if they are missing
+* Signing key fetchers no longer ignore keys for the local domain, if retrieving a key that is not known in the local config
+* Federation timeouts have been adjusted so we don't give up on remote requests so quickly
+* `create-account` no longer relies on the device database (contributed by [ThatNerdyPikachu](https://github.com/ThatNerdyPikachu))
+
+### Known issues
+
+* Old events can incorrectly appear in `/sync` as if they are new when retrieving missing events from federated servers, causing them to appear at the bottom of the timeline in clients
+
+## Dendrite 0.1.0 (2020-10-08)
 
 First versioned release of Dendrite.
 
 ## Client-Server API Features
 
 ### Account registration and management
-- Registration: By password only.
-- Login: By password only. No fallback.
-- Logout: Yes.
-- Change password: Yes.
-- Link email/msisdn to account: No.
-- Deactivate account: Yes.
-- Check if username is available: Yes.
-- Account data: Yes.
-- OpenID: No.
+
+* Registration: By password only.
+* Login: By password only. No fallback.
+* Logout: Yes.
+* Change password: Yes.
+* Link email/msisdn to account: No.
+* Deactivate account: Yes.
+* Check if username is available: Yes.
+* Account data: Yes.
+* OpenID: No.
 
 ### Rooms
-- Room creation: Yes, including presets.
-- Joining rooms: Yes, including by alias or `?server_name=`.
-- Event sending: Yes, including transaction IDs.
-- Aliases: Yes.
-- Published room directory: Yes.
-- Kicking users: Yes.
-- Banning users: Yes.
-- Inviting users: Yes, but not third-party invites.
-- Forgetting rooms: No.
-- Room versions: All (v1 - v6)
-- Tagging: Yes.
+
+* Room creation: Yes, including presets.
+* Joining rooms: Yes, including by alias or `?server_name=`.
+* Event sending: Yes, including transaction IDs.
+* Aliases: Yes.
+* Published room directory: Yes.
+* Kicking users: Yes.
+* Banning users: Yes.
+* Inviting users: Yes, but not third-party invites.
+* Forgetting rooms: No.
+* Room versions: All (v1 * v6)
+* Tagging: Yes.
 
 ### User management
-- User directory: Basic support.
-- Ignoring users: No.
-- Groups/Communities: No.
+
+* User directory: Basic support.
+* Ignoring users: No.
+* Groups/Communities: No.
 
 ### Device management
-- Creating devices: Yes.
-- Deleting devices: Yes.
-- Send-to-device messaging: Yes.
+
+* Creating devices: Yes.
+* Deleting devices: Yes.
+* Send-to-device messaging: Yes.
 
 ### Sync
-- Filters: Timeline limit only. Rest unimplemented.
-- Deprecated `/events` and `/initialSync`: No.
+
+* Filters: Timeline limit only. Rest unimplemented.
+* Deprecated `/events` and `/initialSync`: No.
 
 ### Room events
-- Typing: Yes.
-- Receipts: No.
-- Read Markers: No.
-- Presence: No.
-- Content repository (attachments): Yes.
-- History visibility: No, defaults to `joined`.
-- Push notifications: No.
-- Event context: No.
-- Reporting content: No.
+
+* Typing: Yes.
+* Receipts: No.
+* Read Markers: No.
+* Presence: No.
+* Content repository (attachments): Yes.
+* History visibility: No, defaults to `joined`.
+* Push notifications: No.
+* Event context: No.
+* Reporting content: No.
 
 ### End-to-End Encryption
-- Uploading device keys: Yes.
-- Downloading device keys: Yes.
-- Claiming one-time keys: Yes.
-- Querying key changes: Yes.
-- Cross-Signing: No.
+
+* Uploading device keys: Yes.
+* Downloading device keys: Yes.
+* Claiming one-time keys: Yes.
+* Querying key changes: Yes.
+* Cross-Signing: No.
 
 ### Misc
-- Server-side search: No.
-- Guest access: Partial.
-- Room previews: No, partial support for Peeking via MSC2753.
-- Third-Party networks: No.
-- Server notices: No.
-- Policy lists: No.
+
+* Server-side search: No.
+* Guest access: Partial.
+* Room previews: No, partial support for Peeking via MSC2753.
+* Third-Party networks: No.
+* Server notices: No.
+* Policy lists: No.
 
 ## Federation Features
-- Querying keys (incl. notary): Yes.
-- Server ACLs: Yes.
-- Sending transactions: Yes.
-- Joining rooms: Yes.
-- Inviting to rooms: Yes, but not third-party invites.
-- Leaving rooms: Yes.
-- Content repository: Yes.
-- Backfilling / get_missing_events: Yes.
-- Retrieving state of the room (`/state` and `/state_ids`): Yes.
-- Public rooms: Yes.
-- Querying profile data: Yes.
-- Device management: Yes.
-- Send-to-Device messaging: Yes.
-- Querying/Claiming E2E Keys: Yes.
-- Typing: Yes.
-- Presence: No.
-- Receipts: No.
-- OpenID: No.
\ No newline at end of file
+
+* Querying keys (incl. notary): Yes.
+* Server ACLs: Yes.
+* Sending transactions: Yes.
+* Joining rooms: Yes.
+* Inviting to rooms: Yes, but not third-party invites.
+* Leaving rooms: Yes.
+* Content repository: Yes.
+* Backfilling / get_missing_events: Yes.
+* Retrieving state of the room (`/state` and `/state_ids`): Yes.
+* Public rooms: Yes.
+* Querying profile data: Yes.
+* Device management: Yes.
+* Send-to-Device messaging: Yes.
+* Querying/Claiming E2E Keys: Yes.
+* Typing: Yes.
+* Presence: No.
+* Receipts: No.
+* OpenID: No.
\ No newline at end of file
diff --git a/internal/version.go b/internal/version.go
index a9e245d44..040ffa32a 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -16,7 +16,7 @@ var build string
 
 const (
 	VersionMajor = 0
-	VersionMinor = 1
+	VersionMinor = 2
 	VersionPatch = 0
 	VersionTag   = "" // example: "rc1"
 )

From 7ca89ef5116bd918b50c275f6a52d171ea1ccd96 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Oct 2020 17:10:37 +0100
Subject: [PATCH 14/24] Update gomatrixserverlib

---
 go.mod           | 2 +-
 go.sum           | 4 ++--
 sytest-whitelist | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index d3060fa0f..afbe15f14 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351
 	github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.2
diff --git a/go.sum b/go.sum
index 377a2e093..eaec548ba 100644
--- a/go.sum
+++ b/go.sum
@@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 h1:GF1PxbvImWDoz1DQZNMoaYtIqQXtyLAtmQOzwwmw1OI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351 h1:Yg+fvSEvDLjMFkkE4W0e1yBTtCMVmH3tztw/RhTLPcA=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
diff --git a/sytest-whitelist b/sytest-whitelist
index cecf24f75..1a12b591b 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -485,3 +485,4 @@ Event with an invalid signature in the send_join response should not cause room
 Inbound federation rejects typing notifications from wrong remote
 Should not be able to take over the room by pretending there is no PL event
 Can get rooms/{roomId}/state for a departed room (SPEC-216)
+Users cannot set notifications powerlevel higher than their own

From 24e38c413567355a88c0b92f5a325d7b77325ad1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Oct 2020 17:13:12 +0100
Subject: [PATCH 15/24] Internal HTTP APIs over H2C (#1541)

* H2C on internal HTTP because SCIENCE

* Update comments
---
 go.mod                 |  1 +
 internal/setup/base.go | 30 ++++++++++++++++++++++++++++--
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index afbe15f14..d12efd4e0 100644
--- a/go.mod
+++ b/go.mod
@@ -40,6 +40,7 @@ require (
 	github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee
 	go.uber.org/atomic v1.6.0
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
+	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
 	gopkg.in/h2non/bimg.v1 v1.1.4
 	gopkg.in/yaml.v2 v2.3.0
 )
diff --git a/internal/setup/base.go b/internal/setup/base.go
index 8bc4ae17a..4e1cee479 100644
--- a/internal/setup/base.go
+++ b/internal/setup/base.go
@@ -15,8 +15,10 @@
 package setup
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"time"
@@ -25,6 +27,8 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
@@ -107,7 +111,22 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 		logrus.WithError(err).Warnf("Failed to create cache")
 	}
 
-	apiClient := http.Client{Timeout: time.Minute * 10}
+	apiClient := http.Client{
+		Timeout: time.Minute * 10,
+		Transport: &http2.Transport{
+			AllowHTTP: true,
+			DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) {
+				// Ordinarily HTTP/2 would expect TLS, but the remote listener is
+				// H2C-enabled (HTTP/2 without encryption). Overriding the DialTLS
+				// function with a plain Dial allows us to trick the HTTP client
+				// into establishing a HTTP/2 connection without TLS.
+				// TODO: Eventually we will want to look at authenticating and
+				// encrypting these internal HTTP APIs, at which point we will have
+				// to reconsider H2C and change all this anyway.
+				return net.Dial(network, addr)
+			},
+		},
+	}
 	client := http.Client{Timeout: HTTPClientTimeout}
 	if cfg.FederationSender.Proxy.Enabled {
 		client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{
@@ -269,10 +288,17 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 	internalServ := externalServ
 
 	if internalAddr != NoListener && externalAddr != internalAddr {
+		// H2C allows us to accept HTTP/2 connections without TLS
+		// encryption. Since we don't currently require any form of
+		// authentication or encryption on these internal HTTP APIs,
+		// H2C gives us all of the advantages of HTTP/2 (such as
+		// stream multiplexing and avoiding head-of-line blocking)
+		// without enabling TLS.
+		internalH2S := &http2.Server{}
 		internalRouter = mux.NewRouter().SkipClean(true).UseEncodedPath()
 		internalServ = &http.Server{
 			Addr:    string(internalAddr),
-			Handler: internalRouter,
+			Handler: h2c.NewHandler(internalRouter, internalH2S),
 		}
 	}
 

From 7612f64e3c177b388420699f3fec4ecb563b66ee Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Oct 2020 17:23:04 +0100
Subject: [PATCH 16/24] Update gomatrixserverlib to
 matrix-org/gomatrixserverlib#220

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index d12efd4e0..f785dd391 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7
 	github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.2
diff --git a/go.sum b/go.sum
index eaec548ba..7c24516d2 100644
--- a/go.sum
+++ b/go.sum
@@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351 h1:Yg+fvSEvDLjMFkkE4W0e1yBTtCMVmH3tztw/RhTLPcA=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7 h1:YPuewGCKaJh08NslYAhyGiLw2tg6ew9LtkW7Xr+4uTU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=

From 6b8791b8688991ad04e8231af3838aa090c6bd52 Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Tue, 20 Oct 2020 19:32:33 +0100
Subject: [PATCH 17/24] Always call overridden form of GetLatestEventsForUpdate
 (#1554)

This ensures we don't make txns on sqlite still, which can cause
'database is locked' errors.
---
 roomserver/storage/shared/storage.go  | 38 +++++++++++++++------------
 roomserver/storage/sqlite3/storage.go | 35 ++++++++++++------------
 2 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 51dcb8887..aec15ab22 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -27,23 +27,24 @@ import (
 const redactionsArePermanent = true
 
 type Database struct {
-	DB                  *sql.DB
-	Cache               caching.RoomServerCaches
-	Writer              sqlutil.Writer
-	EventsTable         tables.Events
-	EventJSONTable      tables.EventJSON
-	EventTypesTable     tables.EventTypes
-	EventStateKeysTable tables.EventStateKeys
-	RoomsTable          tables.Rooms
-	TransactionsTable   tables.Transactions
-	StateSnapshotTable  tables.StateSnapshot
-	StateBlockTable     tables.StateBlock
-	RoomAliasesTable    tables.RoomAliases
-	PrevEventsTable     tables.PreviousEvents
-	InvitesTable        tables.Invites
-	MembershipTable     tables.Membership
-	PublishedTable      tables.Published
-	RedactionsTable     tables.Redactions
+	DB                         *sql.DB
+	Cache                      caching.RoomServerCaches
+	Writer                     sqlutil.Writer
+	EventsTable                tables.Events
+	EventJSONTable             tables.EventJSON
+	EventTypesTable            tables.EventTypes
+	EventStateKeysTable        tables.EventStateKeys
+	RoomsTable                 tables.Rooms
+	TransactionsTable          tables.Transactions
+	StateSnapshotTable         tables.StateSnapshot
+	StateBlockTable            tables.StateBlock
+	RoomAliasesTable           tables.RoomAliases
+	PrevEventsTable            tables.PreviousEvents
+	InvitesTable               tables.Invites
+	MembershipTable            tables.Membership
+	PublishedTable             tables.Published
+	RedactionsTable            tables.Redactions
+	GetLatestEventsForUpdateFn func(ctx context.Context, roomInfo types.RoomInfo) (*LatestEventsUpdater, error)
 }
 
 func (d *Database) SupportsConcurrentRoomInputs() bool {
@@ -372,6 +373,9 @@ func (d *Database) MembershipUpdater(
 func (d *Database) GetLatestEventsForUpdate(
 	ctx context.Context, roomInfo types.RoomInfo,
 ) (*LatestEventsUpdater, error) {
+	if d.GetLatestEventsForUpdateFn != nil {
+		return d.GetLatestEventsForUpdateFn(ctx, roomInfo)
+	}
 	txn, err := d.DB.Begin()
 	if err != nil {
 		return nil, err
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index 4a74bf736..6d9b860f5 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -120,23 +120,24 @@ func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches)
 		return nil, err
 	}
 	d.Database = shared.Database{
-		DB:                  d.db,
-		Cache:               cache,
-		Writer:              d.writer,
-		EventsTable:         d.events,
-		EventTypesTable:     d.eventTypes,
-		EventStateKeysTable: d.eventStateKeys,
-		EventJSONTable:      d.eventJSON,
-		RoomsTable:          d.rooms,
-		TransactionsTable:   d.transactions,
-		StateBlockTable:     stateBlock,
-		StateSnapshotTable:  stateSnapshot,
-		PrevEventsTable:     d.prevEvents,
-		RoomAliasesTable:    roomAliases,
-		InvitesTable:        d.invites,
-		MembershipTable:     d.membership,
-		PublishedTable:      published,
-		RedactionsTable:     redactions,
+		DB:                         d.db,
+		Cache:                      cache,
+		Writer:                     d.writer,
+		EventsTable:                d.events,
+		EventTypesTable:            d.eventTypes,
+		EventStateKeysTable:        d.eventStateKeys,
+		EventJSONTable:             d.eventJSON,
+		RoomsTable:                 d.rooms,
+		TransactionsTable:          d.transactions,
+		StateBlockTable:            stateBlock,
+		StateSnapshotTable:         stateSnapshot,
+		PrevEventsTable:            d.prevEvents,
+		RoomAliasesTable:           roomAliases,
+		InvitesTable:               d.invites,
+		MembershipTable:            d.membership,
+		PublishedTable:             published,
+		RedactionsTable:            redactions,
+		GetLatestEventsForUpdateFn: d.GetLatestEventsForUpdate,
 	}
 	return &d, nil
 }

From e4f3f38f35ad14a581bed481352a6c1efeb56115 Mon Sep 17 00:00:00 2001
From: Vladislav Povalyaev <33958163+povidev@users.noreply.github.com>
Date: Tue, 20 Oct 2020 22:49:03 +0300
Subject: [PATCH 18/24] Update development percentages (#1553)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f27cb4029..ea61dac13 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,7 @@ Then point your favourite Matrix client at `http://localhost:8008`.
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of October 2020 we're at around 56% CS API coverage and 77% Federation coverage, though check
+updates with CI. As of October 2020 we're at around 57% CS API coverage and 81% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably:
  - Receipts

From 534f9a9eb69ad35c7492c18c65cdbd55892bf930 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 21 Oct 2020 15:37:07 +0100
Subject: [PATCH 19/24] Refactor forward extremities (#1556)

* Add resolve-state helper

* Tweaks

* Refactor forward extremities, again

* Tweaks

* Minor optimisation

* Make path a bit clearer

* Only process state/membership if forward extremities have changed

* Usage comments in resolve-state
---
 cmd/resolve-state/main.go                     | 132 ++++++++++++++++++
 .../internal/input/input_latest_events.go     | 127 ++++++++++-------
 roomserver/state/state.go                     |   8 +-
 3 files changed, 209 insertions(+), 58 deletions(-)
 create mode 100644 cmd/resolve-state/main.go

diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
new file mode 100644
index 000000000..9fb14f056
--- /dev/null
+++ b/cmd/resolve-state/main.go
@@ -0,0 +1,132 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/roomserver/state"
+	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// This is a utility for inspecting state snapshots and running state resolution
+// against real snapshots in an actual database.
+// It takes one or more state snapshot NIDs as arguments, along with a room version
+// to use for unmarshalling events, and will produce resolved output.
+//
+// Usage: ./resolve-state --roomversion=version snapshot [snapshot ...]
+//   e.g. ./resolve-state --roomversion=5 1254 1235 1282
+
+var roomVersion = flag.String("roomversion", "5", "the room version to parse events as")
+
+// nolint:gocyclo
+func main() {
+	ctx := context.Background()
+	cfg := setup.ParseFlags(true)
+	args := os.Args[1:]
+
+	fmt.Println("Room version", *roomVersion)
+
+	snapshotNIDs := []types.StateSnapshotNID{}
+	for _, arg := range args {
+		if i, err := strconv.Atoi(arg); err == nil {
+			snapshotNIDs = append(snapshotNIDs, types.StateSnapshotNID(i))
+		}
+	}
+
+	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
+
+	cache, err := caching.NewInMemoryLRUCache(true)
+	if err != nil {
+		panic(err)
+	}
+
+	roomserverDB, err := storage.Open(&cfg.RoomServer.Database, cache)
+	if err != nil {
+		panic(err)
+	}
+
+	blockNIDs, err := roomserverDB.StateBlockNIDs(ctx, snapshotNIDs)
+	if err != nil {
+		panic(err)
+	}
+
+	var stateEntries []types.StateEntryList
+	for _, list := range blockNIDs {
+		entries, err2 := roomserverDB.StateEntries(ctx, list.StateBlockNIDs)
+		if err2 != nil {
+			panic(err2)
+		}
+		stateEntries = append(stateEntries, entries...)
+	}
+
+	var eventNIDs []types.EventNID
+	for _, entry := range stateEntries {
+		for _, e := range entry.StateEntries {
+			eventNIDs = append(eventNIDs, e.EventNID)
+		}
+	}
+
+	fmt.Println("Fetching", len(eventNIDs), "state events")
+	eventEntries, err := roomserverDB.Events(ctx, eventNIDs)
+	if err != nil {
+		panic(err)
+	}
+
+	authEventIDMap := make(map[string]struct{})
+	eventPtrs := make([]*gomatrixserverlib.Event, len(eventEntries))
+	for i := range eventEntries {
+		eventPtrs[i] = &eventEntries[i].Event
+		for _, authEventID := range eventEntries[i].AuthEventIDs() {
+			authEventIDMap[authEventID] = struct{}{}
+		}
+	}
+
+	authEventIDs := make([]string, 0, len(authEventIDMap))
+	for authEventID := range authEventIDMap {
+		authEventIDs = append(authEventIDs, authEventID)
+	}
+
+	fmt.Println("Fetching", len(authEventIDs), "auth events")
+	authEventEntries, err := roomserverDB.EventsFromIDs(ctx, authEventIDs)
+	if err != nil {
+		panic(err)
+	}
+
+	authEventPtrs := make([]*gomatrixserverlib.Event, len(authEventEntries))
+	for i := range authEventEntries {
+		authEventPtrs[i] = &authEventEntries[i].Event
+	}
+
+	events := make([]gomatrixserverlib.Event, len(eventEntries))
+	authEvents := make([]gomatrixserverlib.Event, len(authEventEntries))
+	for i, ptr := range eventPtrs {
+		events[i] = *ptr
+	}
+	for i, ptr := range authEventPtrs {
+		authEvents[i] = *ptr
+	}
+
+	fmt.Println("Resolving state")
+	resolved, err := state.ResolveConflictsAdhoc(
+		gomatrixserverlib.RoomVersion(*roomVersion),
+		events,
+		authEvents,
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println("Resolved state contains", len(resolved), "events")
+	for _, event := range resolved {
+		fmt.Println()
+		fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey())
+		fmt.Printf("  %s\n", string(event.Content()))
+	}
+}
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 5631959b7..a86e901d5 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -17,7 +17,6 @@
 package input
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 
@@ -28,7 +27,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 // updateLatestEvents updates the list of latest events for this room in the database and writes the
@@ -141,27 +139,30 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 
 	// Work out what the latest events are. This will include the new
 	// event if it is not already referenced.
-	if err := u.calculateLatest(
-		oldLatest,
+	extremitiesChanged, err := u.calculateLatest(
+		oldLatest, &u.event,
 		types.StateAtEventAndReference{
 			EventReference: u.event.EventReference(),
 			StateAtEvent:   u.stateAtEvent,
 		},
-	); err != nil {
+	)
+	if err != nil {
 		return fmt.Errorf("u.calculateLatest: %w", err)
 	}
 
 	// Now that we know what the latest events are, it's time to get the
 	// latest state.
-	if err := u.latestState(); err != nil {
-		return fmt.Errorf("u.latestState: %w", err)
-	}
+	var updates []api.OutputEvent
+	if extremitiesChanged {
+		if err = u.latestState(); err != nil {
+			return fmt.Errorf("u.latestState: %w", err)
+		}
 
-	// If we need to generate any output events then here's where we do it.
-	// TODO: Move this!
-	updates, err := u.api.updateMemberships(u.ctx, u.updater, u.removed, u.added)
-	if err != nil {
-		return fmt.Errorf("u.api.updateMemberships: %w", err)
+		// If we need to generate any output events then here's where we do it.
+		// TODO: Move this!
+		if updates, err = u.api.updateMemberships(u.ctx, u.updater, u.removed, u.added); err != nil {
+			return fmt.Errorf("u.api.updateMemberships: %w", err)
+		}
 	}
 
 	update, err := u.makeOutputNewRoomEvent()
@@ -250,50 +251,74 @@ func (u *latestEventsUpdater) latestState() error {
 // true if the new event is included in those extremites, false otherwise.
 func (u *latestEventsUpdater) calculateLatest(
 	oldLatest []types.StateAtEventAndReference,
-	newEvent types.StateAtEventAndReference,
-) error {
-	var newLatest []types.StateAtEventAndReference
-
-	// First of all, let's see if any of the existing forward extremities
-	// now have entries in the previous events table. If they do then we
-	// will no longer include them as forward extremities.
-	for _, l := range oldLatest {
-		referenced, err := u.updater.IsReferenced(l.EventReference)
-		if err != nil {
-			logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", l.EventID)
-			return fmt.Errorf("u.updater.IsReferenced (old): %w", err)
-		} else if !referenced {
-			newLatest = append(newLatest, l)
-		}
+	newEvent *gomatrixserverlib.Event,
+	newStateAndRef types.StateAtEventAndReference,
+) (bool, error) {
+	// First of all, get a list of all of the events in our current
+	// set of forward extremities.
+	existingRefs := make(map[string]*types.StateAtEventAndReference)
+	existingNIDs := make([]types.EventNID, len(oldLatest))
+	for i, old := range oldLatest {
+		existingRefs[old.EventID] = &oldLatest[i]
+		existingNIDs[i] = old.EventNID
 	}
 
-	// Then check and see if our new event is already included in that set.
-	// This ordinarily won't happen but it covers the edge-case that we've
-	// already seen this event before and it's a forward extremity, so rather
-	// than adding a duplicate, we'll just return the set as complete.
-	for _, l := range newLatest {
-		if l.EventReference.EventID == newEvent.EventReference.EventID && bytes.Equal(l.EventReference.EventSHA256, newEvent.EventReference.EventSHA256) {
-			// We've already referenced this new event so we can just return
-			// the newly completed extremities at this point.
-			u.latest = newLatest
-			return nil
-		}
-	}
-
-	// At this point we've processed the old extremities, and we've checked
-	// that our new event isn't already in that set. Therefore now we can
-	// check if our *new* event is a forward extremity, and if it is, add
-	// it in.
-	referenced, err := u.updater.IsReferenced(newEvent.EventReference)
+	// Look up the old extremity events. This allows us to find their
+	// prev events.
+	events, err := u.api.DB.Events(u.ctx, existingNIDs)
 	if err != nil {
-		logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", newEvent.EventReference.EventID)
-		return fmt.Errorf("u.updater.IsReferenced (new): %w", err)
-	} else if !referenced || len(newLatest) == 0 {
-		newLatest = append(newLatest, newEvent)
+		return false, fmt.Errorf("u.api.DB.Events: %w", err)
+	}
+
+	// Make a list of all of the prev events as referenced by all of
+	// the current forward extremities.
+	existingPrevs := make(map[string]struct{})
+	for _, old := range events {
+		for _, prevEventID := range old.PrevEventIDs() {
+			existingPrevs[prevEventID] = struct{}{}
+		}
+	}
+
+	// If the "new" event is already referenced by a forward extremity
+	// then do nothing - it's not a candidate to be a new extremity if
+	// it has been referenced.
+	if _, ok := existingPrevs[newEvent.EventID()]; ok {
+		return false, nil
+	}
+
+	// If the "new" event is already a forward extremity then stop, as
+	// nothing changes.
+	for _, event := range events {
+		if event.EventID() == newEvent.EventID() {
+			return false, nil
+		}
+	}
+
+	// Include our new event in the extremities.
+	newLatest := []types.StateAtEventAndReference{newStateAndRef}
+
+	// Then run through and see if the other extremities are still valid.
+	// If our new event references them then they are no longer good
+	// candidates.
+	for _, prevEventID := range newEvent.PrevEventIDs() {
+		delete(existingRefs, prevEventID)
+	}
+
+	// Ensure that we don't add any candidate forward extremities from
+	// the old set that are, themselves, referenced by the old set of
+	// forward extremities. This shouldn't happen but guards against
+	// the possibility anyway.
+	for prevEventID := range existingPrevs {
+		delete(existingRefs, prevEventID)
+	}
+
+	// Then re-add any old extremities that are still valid after all.
+	for _, old := range existingRefs {
+		newLatest = append(newLatest, *old)
 	}
 
 	u.latest = newLatest
-	return nil
+	return true, nil
 }
 
 func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) {
diff --git a/roomserver/state/state.go b/roomserver/state/state.go
index 2944f71c1..d23f14c84 100644
--- a/roomserver/state/state.go
+++ b/roomserver/state/state.go
@@ -526,13 +526,7 @@ func (v StateResolution) CalculateAndStoreStateBeforeEvent(
 	isRejected bool,
 ) (types.StateSnapshotNID, error) {
 	// Load the state at the prev events.
-	prevEventRefs := event.PrevEvents()
-	prevEventIDs := make([]string, len(prevEventRefs))
-	for i := range prevEventRefs {
-		prevEventIDs[i] = prevEventRefs[i].EventID
-	}
-
-	prevStates, err := v.db.StateAtEventIDs(ctx, prevEventIDs)
+	prevStates, err := v.db.StateAtEventIDs(ctx, event.PrevEventIDs())
 	if err != nil {
 		return 0, err
 	}

From 04dc019e5e2f60cd387699bb7f4f7eead15ba714 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 21 Oct 2020 16:21:36 +0100
Subject: [PATCH 20/24] Don't set empty state snapshots

---
 roomserver/internal/input/input_latest_events.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index a86e901d5..dc758e9b0 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -163,6 +163,8 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 		if updates, err = u.api.updateMemberships(u.ctx, u.updater, u.removed, u.added); err != nil {
 			return fmt.Errorf("u.api.updateMemberships: %w", err)
 		}
+	} else {
+		u.newStateNID = u.oldStateNID
 	}
 
 	update, err := u.makeOutputNewRoomEvent()

From 3afc623098dc2cc24093466f69e1d9c4bac9d35b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 22 Oct 2020 10:39:16 +0100
Subject: [PATCH 21/24] Fix RewritesState bug (#1557)

* Set RewritesState once

* Check if any new state provided

* Obey rewritesState

* Don't nuke everything the sync API knows when purging state

* Fix panic from duplicate insert

* Consistency

* Use HasState

* Remove nolint

* Clean up joined rooms on state rewrite
---
 federationsender/consumers/roomserver.go      |  6 +++++
 federationsender/storage/interface.go         |  1 +
 .../storage/postgres/joined_hosts_table.go    | 15 +++++++++++
 federationsender/storage/shared/storage.go    | 14 +++++++++++
 .../storage/sqlite3/joined_hosts_table.go     | 25 +++++++++++++++----
 federationsender/storage/tables/interface.go  |  1 +
 .../internal/input/input_latest_events.go     | 12 +++------
 roomserver/roomserver_test.go                 |  2 +-
 syncapi/consumers/roomserver.go               |  2 +-
 syncapi/storage/interface.go                  |  4 +--
 syncapi/storage/shared/syncserver.go          | 11 +-------
 11 files changed, 65 insertions(+), 28 deletions(-)

diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go
index efeb53fa6..ef945694c 100644
--- a/federationsender/consumers/roomserver.go
+++ b/federationsender/consumers/roomserver.go
@@ -87,6 +87,12 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 	case api.OutputTypeNewRoomEvent:
 		ev := &output.NewRoomEvent.Event
 
+		if output.NewRoomEvent.RewritesState {
+			if err := s.db.PurgeRoomState(context.TODO(), ev.RoomID()); err != nil {
+				return fmt.Errorf("s.db.PurgeRoom: %w", err)
+			}
+		}
+
 		if err := s.processMessage(*output.NewRoomEvent); err != nil {
 			// panic rather than continue with an inconsistent database
 			log.WithFields(log.Fields{
diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go
index 734b368fe..a3f5073f9 100644
--- a/federationsender/storage/interface.go
+++ b/federationsender/storage/interface.go
@@ -32,6 +32,7 @@ type Database interface {
 	GetAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error)
 	// GetJoinedHostsForRooms returns the complete set of servers in the rooms given.
 	GetJoinedHostsForRooms(ctx context.Context, roomIDs []string) ([]gomatrixserverlib.ServerName, error)
+	PurgeRoomState(ctx context.Context, roomID string) error
 
 	StoreJSON(ctx context.Context, js string) (*shared.Receipt, error)
 
diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationsender/storage/postgres/joined_hosts_table.go
index cc112b7f3..0bc9335dd 100644
--- a/federationsender/storage/postgres/joined_hosts_table.go
+++ b/federationsender/storage/postgres/joined_hosts_table.go
@@ -53,6 +53,9 @@ const insertJoinedHostsSQL = "" +
 const deleteJoinedHostsSQL = "" +
 	"DELETE FROM federationsender_joined_hosts WHERE event_id = ANY($1)"
 
+const deleteJoinedHostsForRoomSQL = "" +
+	"DELETE FROM federationsender_joined_hosts WHERE room_id = $1"
+
 const selectJoinedHostsSQL = "" +
 	"SELECT event_id, server_name FROM federationsender_joined_hosts" +
 	" WHERE room_id = $1"
@@ -67,6 +70,7 @@ type joinedHostsStatements struct {
 	db                            *sql.DB
 	insertJoinedHostsStmt         *sql.Stmt
 	deleteJoinedHostsStmt         *sql.Stmt
+	deleteJoinedHostsForRoomStmt  *sql.Stmt
 	selectJoinedHostsStmt         *sql.Stmt
 	selectAllJoinedHostsStmt      *sql.Stmt
 	selectJoinedHostsForRoomsStmt *sql.Stmt
@@ -86,6 +90,9 @@ func NewPostgresJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err erro
 	if s.deleteJoinedHostsStmt, err = s.db.Prepare(deleteJoinedHostsSQL); err != nil {
 		return
 	}
+	if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil {
+		return
+	}
 	if s.selectJoinedHostsStmt, err = s.db.Prepare(selectJoinedHostsSQL); err != nil {
 		return
 	}
@@ -117,6 +124,14 @@ func (s *joinedHostsStatements) DeleteJoinedHosts(
 	return err
 }
 
+func (s *joinedHostsStatements) DeleteJoinedHostsForRoom(
+	ctx context.Context, txn *sql.Tx, roomID string,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsForRoomStmt)
+	_, err := stmt.ExecContext(ctx, roomID)
+	return err
+}
+
 func (s *joinedHostsStatements) SelectJoinedHostsWithTx(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) ([]types.JoinedHost, error) {
diff --git a/federationsender/storage/shared/storage.go b/federationsender/storage/shared/storage.go
index 4c80c0792..d5731f31c 100644
--- a/federationsender/storage/shared/storage.go
+++ b/federationsender/storage/shared/storage.go
@@ -148,6 +148,20 @@ func (d *Database) StoreJSON(
 	}, nil
 }
 
+func (d *Database) PurgeRoomState(
+	ctx context.Context, roomID string,
+) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		// If the event is a create event then we'll delete all of the existing
+		// data for the room. The only reason that a create event would be replayed
+		// to us in this way is if we're about to receive the entire room state.
+		if err := d.FederationSenderJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
+			return fmt.Errorf("d.FederationSenderJoinedHosts.DeleteJoinedHosts: %w", err)
+		}
+		return nil
+	})
+}
+
 func (d *Database) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.FederationSenderBlacklist.InsertBlacklist(context.TODO(), txn, serverName)
diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationsender/storage/sqlite3/joined_hosts_table.go
index 3bc45e7d5..1f906808d 100644
--- a/federationsender/storage/sqlite3/joined_hosts_table.go
+++ b/federationsender/storage/sqlite3/joined_hosts_table.go
@@ -53,6 +53,9 @@ const insertJoinedHostsSQL = "" +
 const deleteJoinedHostsSQL = "" +
 	"DELETE FROM federationsender_joined_hosts WHERE event_id = $1"
 
+const deleteJoinedHostsForRoomSQL = "" +
+	"DELETE FROM federationsender_joined_hosts WHERE room_id = $1"
+
 const selectJoinedHostsSQL = "" +
 	"SELECT event_id, server_name FROM federationsender_joined_hosts" +
 	" WHERE room_id = $1"
@@ -64,11 +67,12 @@ const selectJoinedHostsForRoomsSQL = "" +
 	"SELECT DISTINCT server_name FROM federationsender_joined_hosts WHERE room_id IN ($1)"
 
 type joinedHostsStatements struct {
-	db                       *sql.DB
-	insertJoinedHostsStmt    *sql.Stmt
-	deleteJoinedHostsStmt    *sql.Stmt
-	selectJoinedHostsStmt    *sql.Stmt
-	selectAllJoinedHostsStmt *sql.Stmt
+	db                           *sql.DB
+	insertJoinedHostsStmt        *sql.Stmt
+	deleteJoinedHostsStmt        *sql.Stmt
+	deleteJoinedHostsForRoomStmt *sql.Stmt
+	selectJoinedHostsStmt        *sql.Stmt
+	selectAllJoinedHostsStmt     *sql.Stmt
 	// selectJoinedHostsForRoomsStmt *sql.Stmt - prepared at runtime due to variadic
 }
 
@@ -86,6 +90,9 @@ func NewSQLiteJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err error)
 	if s.deleteJoinedHostsStmt, err = db.Prepare(deleteJoinedHostsSQL); err != nil {
 		return
 	}
+	if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil {
+		return
+	}
 	if s.selectJoinedHostsStmt, err = db.Prepare(selectJoinedHostsSQL); err != nil {
 		return
 	}
@@ -118,6 +125,14 @@ func (s *joinedHostsStatements) DeleteJoinedHosts(
 	return nil
 }
 
+func (s *joinedHostsStatements) DeleteJoinedHostsForRoom(
+	ctx context.Context, txn *sql.Tx, roomID string,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsForRoomStmt)
+	_, err := stmt.ExecContext(ctx, roomID)
+	return err
+}
+
 func (s *joinedHostsStatements) SelectJoinedHostsWithTx(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) ([]types.JoinedHost, error) {
diff --git a/federationsender/storage/tables/interface.go b/federationsender/storage/tables/interface.go
index c6f8a2d52..1167a212a 100644
--- a/federationsender/storage/tables/interface.go
+++ b/federationsender/storage/tables/interface.go
@@ -50,6 +50,7 @@ type FederationSenderQueueJSON interface {
 type FederationSenderJoinedHosts interface {
 	InsertJoinedHosts(ctx context.Context, txn *sql.Tx, roomID, eventID string, serverName gomatrixserverlib.ServerName) error
 	DeleteJoinedHosts(ctx context.Context, txn *sql.Tx, eventIDs []string) error
+	DeleteJoinedHostsForRoom(ctx context.Context, txn *sql.Tx, roomID string) error
 	SelectJoinedHostsWithTx(ctx context.Context, txn *sql.Tx, roomID string) ([]types.JoinedHost, error)
 	SelectJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error)
 	SelectAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error)
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index dc758e9b0..2bf6b9f8a 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -116,7 +116,6 @@ type latestEventsUpdater struct {
 
 func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	u.lastEventIDSent = u.updater.LastEventIDSent()
-	u.oldStateNID = u.updater.CurrentStateSnapshotNID()
 
 	// If we are doing a regular event update then we will get the
 	// previous latest events to use as a part of the calculation. If
@@ -125,7 +124,8 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// then start with an empty set - none of the forward extremities
 	// that we knew about before matter anymore.
 	oldLatest := []types.StateAtEventAndReference{}
-	if !u.stateAtEvent.Overwrite {
+	if !u.rewritesState {
+		u.oldStateNID = u.updater.CurrentStateSnapshotNID()
 		oldLatest = u.updater.LatestEvents()
 	}
 
@@ -153,7 +153,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// Now that we know what the latest events are, it's time to get the
 	// latest state.
 	var updates []api.OutputEvent
-	if extremitiesChanged {
+	if extremitiesChanged || u.rewritesState {
 		if err = u.latestState(); err != nil {
 			return fmt.Errorf("u.latestState: %w", err)
 		}
@@ -324,7 +324,6 @@ func (u *latestEventsUpdater) calculateLatest(
 }
 
 func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) {
-
 	latestEventIDs := make([]string, len(u.latest))
 	for i := range u.latest {
 		latestEventIDs[i] = u.latest[i].EventID
@@ -365,11 +364,6 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error)
 			return nil, fmt.Errorf("failed to load add_state_events from db: %w", err)
 		}
 	}
-	// State is rewritten if the input room event HasState and we actually produced a delta on state events.
-	// Without this check, /get_missing_events which produce events with associated (but not complete) state
-	// will incorrectly purge the room and set it to no state. TODO: This is likely flakey, as if /gme produced
-	// a state conflict res which just so happens to include 2+ events we might purge the room state downstream.
-	ore.RewritesState = len(ore.AddsStateEventIDs) > 1
 
 	return &api.OutputEvent{
 		Type:         api.OutputTypeNewRoomEvent,
diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go
index c8e60efa3..41cbd2637 100644
--- a/roomserver/roomserver_test.go
+++ b/roomserver/roomserver_test.go
@@ -379,7 +379,7 @@ func TestOutputRewritesState(t *testing.T) {
 	if len(producer.producedMessages) != 1 {
 		t.Fatalf("Rewritten events got output, want only 1 got %d", len(producer.producedMessages))
 	}
-	outputEvent := producer.producedMessages[0]
+	outputEvent := producer.producedMessages[len(producer.producedMessages)-1]
 	if !outputEvent.NewRoomEvent.RewritesState {
 		t.Errorf("RewritesState flag not set on output event")
 	}
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 373baea54..593bfc5cc 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -149,7 +149,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 	}
 
 	if msg.RewritesState {
-		if err = s.db.PurgeRoom(ctx, ev.RoomID()); err != nil {
+		if err = s.db.PurgeRoomState(ctx, ev.RoomID()); err != nil {
 			return fmt.Errorf("s.db.PurgeRoom: %w", err)
 		}
 	}
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index ce7f1c152..e12a1166e 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -43,9 +43,9 @@ type Database interface {
 	// Returns an error if there was a problem inserting this event.
 	WriteEvent(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, addStateEvents []gomatrixserverlib.HeaderedEvent,
 		addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool) (types.StreamPosition, error)
-	// PurgeRoom completely purges room state from the sync API. This is done when
+	// PurgeRoomState completely purges room state from the sync API. This is done when
 	// receiving an output event that completely resets the state.
-	PurgeRoom(ctx context.Context, roomID string) error
+	PurgeRoomState(ctx context.Context, roomID string) error
 	// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key
 	// If no event could be found, returns nil
 	// If there was an issue during the retrieval, returns an error
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index b9f21913e..a7c07f943 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -276,7 +276,7 @@ func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, e
 	return nil
 }
 
-func (d *Database) PurgeRoom(
+func (d *Database) PurgeRoomState(
 	ctx context.Context, roomID string,
 ) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
@@ -286,15 +286,6 @@ func (d *Database) PurgeRoom(
 		if err := d.CurrentRoomState.DeleteRoomStateForRoom(ctx, txn, roomID); err != nil {
 			return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateForRoom: %w", err)
 		}
-		if err := d.OutputEvents.DeleteEventsForRoom(ctx, txn, roomID); err != nil {
-			return fmt.Errorf("d.Events.DeleteEventsForRoom: %w", err)
-		}
-		if err := d.Topology.DeleteTopologyForRoom(ctx, txn, roomID); err != nil {
-			return fmt.Errorf("d.Topology.DeleteTopologyForRoom: %w", err)
-		}
-		if err := d.BackwardExtremities.DeleteBackwardExtremitiesForRoom(ctx, txn, roomID); err != nil {
-			return fmt.Errorf("d.BackwardExtremities.DeleteBackwardExtremitiesForRoom: %w", err)
-		}
 		return nil
 	})
 }

From f32320a2355b960e45d42c5cbe7cfea820873767 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 22 Oct 2020 11:50:48 +0100
Subject: [PATCH 22/24] Exclude old state events from sync when consumed from
 roomserver (#1548)

* Exclude old room events from sync when consumed from roomserver

* Fix comment

* Experimental hack
---
 syncapi/consumers/roomserver.go | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 593bfc5cc..ac1128c11 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -189,14 +189,20 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent(
 ) error {
 	ev := msg.Event
 
+	// TODO: The state key check when excluding from sync is designed
+	// to stop us from lying to clients with old state, whilst still
+	// allowing normal timeline events through. This is an absolute
+	// hack but until we have some better strategy for dealing with
+	// old events in the sync API, this should at least prevent us
+	// from confusing clients into thinking they've joined/left rooms.
 	pduPos, err := s.db.WriteEvent(
 		ctx,
 		&ev,
 		[]gomatrixserverlib.HeaderedEvent{},
-		[]string{}, // adds no state
-		[]string{}, // removes no state
-		nil,        // no transaction
-		false,      // not excluded from sync
+		[]string{},           // adds no state
+		[]string{},           // removes no state
+		nil,                  // no transaction
+		ev.StateKey() != nil, // exclude from sync?
 	)
 	if err != nil {
 		// panic rather than continue with an inconsistent database

From b66b2eab32b1d31c3cbdc5fb733022d69b66da80 Mon Sep 17 00:00:00 2001
From: Adit Sachde <23707194+aditsachde@users.noreply.github.com>
Date: Thu, 22 Oct 2020 07:21:31 -0400
Subject: [PATCH 23/24] Document dendrite server key format (#1520)

Co-authored-by: Kegsay <kegan@matrix.org>
---
 docs/INSTALL.md         |  4 ++++
 docs/serverkeyformat.md | 29 +++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 docs/serverkeyformat.md

diff --git a/docs/INSTALL.md b/docs/INSTALL.md
index f804193cd..1cecd047c 100644
--- a/docs/INSTALL.md
+++ b/docs/INSTALL.md
@@ -129,6 +129,10 @@ for federation and the server signing key:
 ./bin/generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
 ```
 
+If you have server keys from an older synapse instance, 
+[convert them](serverkeyformat.md#converting-synapse-keys) to Dendrite's PEM 
+format and configure them as `old_private_keys` in your config.
+
 ### Configuration file
 
 Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Things that will need editing include *at least*:
diff --git a/docs/serverkeyformat.md b/docs/serverkeyformat.md
new file mode 100644
index 000000000..feda93454
--- /dev/null
+++ b/docs/serverkeyformat.md
@@ -0,0 +1,29 @@
+# Server Key Format
+
+Dendrite stores the server signing key in the PEM format with the following structure.
+
+```
+-----BEGIN MATRIX PRIVATE KEY-----
+Key-ID: ed25519:<Key Handle>
+
+<Base64 Encoded Key Data>
+-----END MATRIX PRIVATE KEY-----
+```
+
+## Converting Synapse Keys
+
+If you have signing keys from a previous synapse server, you should ideally configure them as `old_private_keys` in your Dendrite config file. Synapse stores signing keys in the following format.
+
+```
+ed25519 <Key Handle> <Base64 Encoded Key Data>
+```
+
+To convert this key to Dendrite's PEM format, use the following template. **It is important to include the equals sign, as the key data needs to be padded to 32 bytes.**
+
+```
+-----BEGIN MATRIX PRIVATE KEY-----
+Key-ID: ed25519:<Key Handle>
+
+<Base64 Encoded Key Data>=
+-----END MATRIX PRIVATE KEY-----
+```
\ No newline at end of file

From 59428cdde3ed3f30f01b886c641e6dd1f313c294 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 22 Oct 2020 12:38:12 +0100
Subject: [PATCH 24/24] Version 0.2.1 (#1558)

* Version 0.2.1

* Credit where credit due

* Update CHANGES.md

* Include PRs
---
 CHANGES.md          | 12 ++++++++++++
 internal/version.go |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index d05b871ac..095ab9c5b 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,17 @@
 # Changelog
 
+## Dendrite 0.2.1 (2020-10-22)
+
+### Fixes
+
+* Forward extremities are now calculated using only references from other extremities, rather than including outliers, which should fix cases where state can become corrupted ([#1556](https://github.com/matrix-org/dendrite/pull/1556))
+* Old state events will no longer be processed by the sync API as new, which should fix some cases where clients incorrectly believe they have joined or left rooms ([#1548](https://github.com/matrix-org/dendrite/pull/1548))
+* More SQLite database locking issues have been resolved in the latest events updater ([#1554](https://github.com/matrix-org/dendrite/pull/1554))
+* Internal HTTP API calls are now made using H2C (HTTP/2) in polylith mode, mitigating some potential head-of-line blocking issues ([#1541](https://github.com/matrix-org/dendrite/pull/1541))
+* Roomserver output events no longer incorrectly flag state rewrites ([#1557](https://github.com/matrix-org/dendrite/pull/1557))
+* Notification levels are now parsed correctly in power level events ([gomatrixserverlib#228](https://github.com/matrix-org/gomatrixserverlib/pull/228), contributed by [Pestdoktor](https://github.com/Pestdoktor))
+* Invalid UTF-8 is now correctly rejected when making federation requests ([gomatrixserverlib#229](https://github.com/matrix-org/gomatrixserverlib/pull/229), contributed by [Pestdoktor](https://github.com/Pestdoktor))
+
 ## Dendrite 0.2.0 (2020-10-20)
 
 ### Important
diff --git a/internal/version.go b/internal/version.go
index 040ffa32a..21f697086 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 2
-	VersionPatch = 0
+	VersionPatch = 1
 	VersionTag   = "" // example: "rc1"
 )