From a06d0921c9a8551a7f488ad8ae972f1b982a49c1 Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Wed, 24 Jun 2020 18:19:14 +0100
Subject: [PATCH 01/23] Make same membership transitions 403, add sytests
 (#1161)

* Make same membership transitions 403, add sytests

* Update blacklist
---
 roomserver/api/perform.go | 5 +++++
 sytest-blacklist          | 3 +++
 sytest-whitelist          | 2 ++
 3 files changed, 10 insertions(+)

diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index 0b8e6df25..12ba15167 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -38,6 +38,11 @@ func (p *PerformError) JSONResponse() util.JSONResponse {
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden(p.Msg),
 		}
+	case PerformErrorNoOperation:
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden(p.Msg),
+		}
 	default:
 		return util.ErrorResponse(p)
 	}
diff --git a/sytest-blacklist b/sytest-blacklist
index 9f140ed1c..65e6c1b16 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -45,6 +45,9 @@ Can recv device messages over federation
 Device messages over federation wake up /sync
 Wildcard device messages over federation wake up /sync
 
+# See https://github.com/matrix-org/sytest/pull/901
+Remote invited user can see room metadata
+
 # We don't implement soft-failed events yet, but because the /send response is vague,
 # this test thinks it's all fine...
 Inbound federation accepts a second soft-failed event
diff --git a/sytest-whitelist b/sytest-whitelist
index 0036d60ea..18bb7ca43 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -357,6 +357,8 @@ Getting state checks the events requested belong to the room
 Getting state IDs checks the events requested belong to the room
 Can invite users to invite-only rooms
 Uninvited users cannot join the room
+Users cannot invite themselves to a room
+Users cannot invite a user that is already in the room
 Invited user can reject invite
 Invited user can reject invite for empty room
 Invited user can reject local invite after originator leaves

From e560619f76d3c54c018ed8117c20346ab79007b0 Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Wed, 24 Jun 2020 18:19:54 +0100
Subject: [PATCH 02/23] Refactor SendMembership - make ban test pass (#1160)

* Refactor SendMembership - make ban test pass

* Only check invite auth events for local invites
---
 clientapi/routing/createroom.go       |   7 +-
 clientapi/routing/membership.go       | 255 ++++++++++++++++----------
 clientapi/routing/routing.go          |  42 ++++-
 clientapi/threepid/invites.go         |   4 +-
 roomserver/api/query.go               |   2 +
 roomserver/internal/perform_invite.go |  20 ++
 roomserver/internal/query.go          |  11 +-
 sytest-whitelist                      |   1 +
 8 files changed, 237 insertions(+), 105 deletions(-)

diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index 8682b03a4..42e1895ce 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -28,7 +28,6 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
@@ -373,13 +372,9 @@ func createRoom(
 
 	// If this is a direct message then we should invite the participants.
 	for _, invitee := range r.Invite {
-		// Build the membership request.
-		body := threepid.MembershipRequest{
-			UserID: invitee,
-		}
 		// Build the invite event.
 		inviteEvent, err := buildMembershipEvent(
-			req.Context(), body, accountDB, device, gomatrixserverlib.Invite,
+			req.Context(), invitee, "", accountDB, device, gomatrixserverlib.Invite,
 			roomID, true, cfg, evTime, rsAPI, asAPI,
 		)
 		if err != nil {
diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go
index aff1730c5..1f316384b 100644
--- a/clientapi/routing/membership.go
+++ b/clientapi/routing/membership.go
@@ -38,40 +38,141 @@ import (
 
 var errMissingUserID = errors.New("'user_id' must be supplied")
 
-// SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
-// by building a m.room.member event then sending it to the room server
-// TODO: Can we improve the cyclo count here? Separate code paths for invites?
-// nolint:gocyclo
-func SendMembership(
+func SendBan(
 	req *http.Request, accountDB accounts.Database, device *userapi.Device,
-	roomID string, membership string, cfg *config.Dendrite,
+	roomID string, cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
-	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
-	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	if reqErr != nil {
+		return *reqErr
+	}
+	return sendMembership(req.Context(), accountDB, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+}
+
+func sendMembership(ctx context.Context, accountDB accounts.Database, device *userapi.Device,
+	roomID, membership, reason string, cfg *config.Dendrite, targetUserID string, evTime time.Time,
+	roomVer gomatrixserverlib.RoomVersion,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
+
+	event, err := buildMembershipEvent(
+		ctx, targetUserID, reason, accountDB, device, membership,
+		roomID, false, cfg, evTime, rsAPI, asAPI,
+	)
+	if err == errMissingUserID {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	} else if err == eventutil.ErrRoomNoExists {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound(err.Error()),
+		}
+	} else if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
+		return jsonerror.InternalServerError()
+	}
+
+	_, err = roomserverAPI.SendEvents(
+		ctx, rsAPI,
+		[]gomatrixserverlib.HeaderedEvent{event.Headered(roomVer)},
+		cfg.Matrix.ServerName,
+		nil,
+	)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
+		return jsonerror.InternalServerError()
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func SendKick(
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	roomID string, cfg *config.Dendrite,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	if reqErr != nil {
+		return *reqErr
+	}
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("missing user_id"),
 		}
 	}
 
-	var body threepid.MembershipRequest
-	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
+	var queryRes roomserverAPI.QueryMembershipForUserResponse
+	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+		RoomID: roomID,
+		UserID: body.UserID,
+	}, &queryRes)
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	// kick is only valid if the user is not currently banned
+	if queryRes.Membership == "ban" {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Unknown("cannot /kick banned users"),
+		}
+	}
+	// TODO: should we be using SendLeave instead?
+	return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+}
+
+func SendUnban(
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	roomID string, cfg *config.Dendrite,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	if reqErr != nil {
+		return *reqErr
+	}
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("missing user_id"),
+		}
+	}
+
+	var queryRes roomserverAPI.QueryMembershipForUserResponse
+	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+		RoomID: roomID,
+		UserID: body.UserID,
+	}, &queryRes)
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	// unban is only valid if the user is currently banned
+	if queryRes.Membership != "ban" {
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.Unknown("can only /unban users that are banned"),
+		}
+	}
+	// TODO: should we be using SendLeave instead?
+	return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+}
+
+func SendInvite(
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	roomID string, cfg *config.Dendrite,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	if reqErr != nil {
 		return *reqErr
 	}
 
-	evTime, err := httputil.ParseTSParam(req)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
-		}
-	}
-
 	inviteStored, jsonErrResp := checkAndProcessThreepid(
-		req, device, &body, cfg, rsAPI, accountDB,
-		membership, roomID, evTime,
+		req, device, body, cfg, rsAPI, accountDB, roomID, evTime,
 	)
 	if jsonErrResp != nil {
 		return *jsonErrResp
@@ -88,7 +189,7 @@ func SendMembership(
 	}
 
 	event, err := buildMembershipEvent(
-		req.Context(), body, accountDB, device, membership,
+		req.Context(), body.UserID, body.Reason, accountDB, device, "invite",
 		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
 	if err == errMissingUserID {
@@ -106,61 +207,32 @@ func SendMembership(
 		return jsonerror.InternalServerError()
 	}
 
-	var returnData interface{} = struct{}{}
-
-	switch membership {
-	case gomatrixserverlib.Invite:
-		// Invites need to be handled specially
-		perr := roomserverAPI.SendInvite(
-			req.Context(), rsAPI,
-			event.Headered(verRes.RoomVersion),
-			nil, // ask the roomserver to draw up invite room state for us
-			cfg.Matrix.ServerName,
-			nil,
-		)
-		if perr != nil {
-			util.GetLogger(req.Context()).WithError(perr).Error("producer.SendInvite failed")
-			return perr.JSONResponse()
-		}
-	case gomatrixserverlib.Join:
-		// The join membership requires the room id to be sent in the response
-		returnData = struct {
-			RoomID string `json:"room_id"`
-		}{roomID}
-		fallthrough
-	default:
-		_, err = roomserverAPI.SendEvents(
-			req.Context(), rsAPI,
-			[]gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)},
-			cfg.Matrix.ServerName,
-			nil,
-		)
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
-			return jsonerror.InternalServerError()
-		}
+	perr := roomserverAPI.SendInvite(
+		req.Context(), rsAPI,
+		event.Headered(roomVer),
+		nil, // ask the roomserver to draw up invite room state for us
+		cfg.Matrix.ServerName,
+		nil,
+	)
+	if perr != nil {
+		util.GetLogger(req.Context()).WithError(perr).Error("producer.SendInvite failed")
+		return perr.JSONResponse()
 	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: returnData,
+		JSON: struct{}{},
 	}
 }
 
 func buildMembershipEvent(
 	ctx context.Context,
-	body threepid.MembershipRequest, accountDB accounts.Database,
+	targetUserID, reason string, accountDB accounts.Database,
 	device *userapi.Device,
 	membership, roomID string, isDirect bool,
 	cfg *config.Dendrite, evTime time.Time,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*gomatrixserverlib.Event, error) {
-	stateKey, reason, err := getMembershipStateKey(body, device, membership)
-	if err != nil {
-		return nil, err
-	}
-
-	profile, err := loadProfile(ctx, stateKey, cfg, accountDB, asAPI)
+	profile, err := loadProfile(ctx, targetUserID, cfg, accountDB, asAPI)
 	if err != nil {
 		return nil, err
 	}
@@ -169,12 +241,7 @@ func buildMembershipEvent(
 		Sender:   device.UserID,
 		RoomID:   roomID,
 		Type:     "m.room.member",
-		StateKey: &stateKey,
-	}
-
-	// "unban" or "kick" isn't a valid membership value, change it to "leave"
-	if membership == "unban" || membership == "kick" {
-		membership = gomatrixserverlib.Leave
+		StateKey: &targetUserID,
 	}
 
 	content := gomatrixserverlib.MemberContent{
@@ -218,29 +285,33 @@ func loadProfile(
 	return profile, err
 }
 
-// getMembershipStateKey extracts the target user ID of a membership change.
-// For "join" and "leave" this will be the ID of the user making the change.
-// For "ban", "unban", "kick" and "invite" the target user ID will be in the JSON request body.
-// In the latter case, if there was an issue retrieving the user ID from the request body,
-// returns a JSONResponse with a corresponding error code and message.
-func getMembershipStateKey(
-	body threepid.MembershipRequest, device *userapi.Device, membership string,
-) (stateKey string, reason string, err error) {
-	if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite {
-		// If we're in this case, the state key is contained in the request body,
-		// possibly along with a reason (for "kick" and "ban") so we need to parse
-		// it
-		if body.UserID == "" {
-			err = errMissingUserID
-			return
+func extractRequestData(req *http.Request, roomID string, rsAPI api.RoomserverInternalAPI) (
+	body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse,
+) {
+	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
+	verRes := api.QueryRoomVersionForRoomResponse{}
+	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+		resErr = &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
 		}
+		return
+	}
+	roomVer = verRes.RoomVersion
 
-		stateKey = body.UserID
-		reason = body.Reason
-	} else {
-		stateKey = device.UserID
+	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
+		resErr = reqErr
+		return
 	}
 
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		resErr = &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+		return
+	}
 	return
 }
 
@@ -251,13 +322,13 @@ func checkAndProcessThreepid(
 	cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	accountDB accounts.Database,
-	membership, roomID string,
+	roomID string,
 	evTime time.Time,
 ) (inviteStored bool, errRes *util.JSONResponse) {
 
 	inviteStored, err := threepid.CheckAndProcessInvite(
 		req.Context(), device, body, cfg, rsAPI, accountDB,
-		membership, roomID, evTime,
+		roomID, evTime,
 	)
 	if err == threepid.ErrMissingParameter {
 		return inviteStored, &util.JSONResponse{
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 825ac50f2..eadcfd1ab 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -101,6 +101,17 @@ func Setup(
 			return GetJoinedRooms(req, device, accountDB)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/join",
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return JoinRoomByIDOrAlias(
+				req, device, rsAPI, accountDB, vars["roomID"],
+			)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/leave",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -112,13 +123,40 @@ func Setup(
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}",
+	r0mux.Handle("/rooms/{roomID}/ban",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI)
+			return SendBan(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/invite",
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendInvite(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/kick",
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendKick(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/unban",
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendUnban(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go
index c308cb1f4..89bc86064 100644
--- a/clientapi/threepid/invites.go
+++ b/clientapi/threepid/invites.go
@@ -88,10 +88,10 @@ func CheckAndProcessInvite(
 	ctx context.Context,
 	device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI, db accounts.Database,
-	membership string, roomID string,
+	roomID string,
 	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
-	if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") {
+	if body.Address == "" && body.IDServer == "" && body.Medium == "" {
 		// If none of the 3PID-specific fields are supplied, it's a standard invite
 		// so return nil for it to be processed as such
 		return
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index 6586b1af3..f0cb9374b 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -112,6 +112,8 @@ type QueryMembershipForUserResponse struct {
 	HasBeenInRoom bool `json:"has_been_in_room"`
 	// True if the user is in room.
 	IsInRoom bool `json:"is_in_room"`
+	// The current membership
+	Membership string
 }
 
 // QueryMembershipsForRoomRequest is a request to QueryMembershipsForRoom
diff --git a/roomserver/internal/perform_invite.go b/roomserver/internal/perform_invite.go
index c65c87f91..4600bec0b 100644
--- a/roomserver/internal/perform_invite.go
+++ b/roomserver/internal/perform_invite.go
@@ -55,6 +55,7 @@ func (r *RoomserverInternalAPI) performInvite(ctx context.Context,
 	return nil
 }
 
+// nolint:gocyclo
 func (r *RoomserverInternalAPI) processInviteEvent(
 	ctx context.Context,
 	ow *RoomserverInternalAPI,
@@ -135,6 +136,25 @@ func (r *RoomserverInternalAPI) processInviteEvent(
 	}
 
 	event := input.Event.Unwrap()
+
+	// check that the user is allowed to do this. We can only do this check if it is
+	// a local invite as we have the auth events, else we have to take it on trust.
+	if loopback != nil {
+		_, err = checkAuthEvents(ctx, r.DB, input.Event, input.Event.AuthEventIDs())
+		if err != nil {
+			log.WithError(err).WithField("event_id", event.EventID()).WithField("auth_event_ids", event.AuthEventIDs()).Error(
+				"processInviteEvent.checkAuthEvents failed for event",
+			)
+			if _, ok := err.(*gomatrixserverlib.NotAllowed); ok {
+				return nil, &api.PerformError{
+					Msg:  err.Error(),
+					Code: api.PerformErrorNotAllowed,
+				}
+			}
+			return nil, err
+		}
+	}
+
 	if len(input.InviteRoomState) > 0 {
 		// If we were supplied with some invite room state already (which is
 		// most likely to be if the event came in over federation) then use
diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go
index 4fc8e4c25..19236bfbd 100644
--- a/roomserver/internal/query.go
+++ b/roomserver/internal/query.go
@@ -225,13 +225,18 @@ func (r *RoomserverInternalAPI) QueryMembershipForUser(
 	}
 
 	response.IsInRoom = stillInRoom
-	eventIDMap, err := r.DB.EventIDs(ctx, []types.EventNID{membershipEventNID})
+
+	evs, err := r.DB.Events(ctx, []types.EventNID{membershipEventNID})
 	if err != nil {
 		return err
 	}
+	if len(evs) != 1 {
+		return fmt.Errorf("failed to load membership event for event NID %d", membershipEventNID)
+	}
 
-	response.EventID = eventIDMap[membershipEventNID]
-	return nil
+	response.EventID = evs[0].EventID()
+	response.Membership, err = evs[0].Membership()
+	return err
 }
 
 // QueryMembershipsForRoom implements api.RoomserverInternalAPI
diff --git a/sytest-whitelist b/sytest-whitelist
index 18bb7ca43..ce97e8af7 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -365,3 +365,4 @@ Invited user can reject local invite after originator leaves
 Typing notification sent to local room members
 Typing notifications also sent to remote room members
 Typing can be explicitly stopped
+Banned user is kicked and may not rejoin until unbanned

From 46de400aa02b537a3851fdd6a2a2a31267a0b5c1 Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Wed, 24 Jun 2020 20:46:28 +0100
Subject: [PATCH 03/23] Hopefully fix databased is locked errors on sqlite
 account creation (#1162)

---
 userapi/storage/accounts/sqlite3/storage.go | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go
index dbf6606c3..d84f25b1f 100644
--- a/userapi/storage/accounts/sqlite3/storage.go
+++ b/userapi/storage/accounts/sqlite3/storage.go
@@ -42,7 +42,7 @@ type Database struct {
 	filter       filterStatements
 	serverName   gomatrixserverlib.ServerName
 
-	createGuestAccountMu sync.Mutex
+	createAccountMu sync.Mutex
 }
 
 // NewDatabase creates a new accounts and profiles database
@@ -129,14 +129,14 @@ func (d *Database) SetDisplayName(
 // CreateGuestAccount makes a new guest account and creates an empty profile
 // for this account.
 func (d *Database) CreateGuestAccount(ctx context.Context) (acc *api.Account, err error) {
+	// We need to lock so we sequentially create numeric localparts. If we don't, two calls to
+	// this function will cause the same number to be selected and one will fail with 'database is locked'
+	// when the first txn upgrades to a write txn. We also need to lock the account creation else we can
+	// race with CreateAccount
+	// We know we'll be the only process since this is sqlite ;) so a lock here will be all that is needed.
+	d.createAccountMu.Lock()
+	defer d.createAccountMu.Unlock()
 	err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
-		// We need to lock so we sequentially create numeric localparts. If we don't, two calls to
-		// this function will cause the same number to be selected and one will fail with 'database is locked'
-		// when the first txn upgrades to a write txn.
-		// We know we'll be the only process since this is sqlite ;) so a lock here will be all that is needed.
-		d.createGuestAccountMu.Lock()
-		defer d.createGuestAccountMu.Unlock()
-
 		var numLocalpart int64
 		numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn)
 		if err != nil {
@@ -155,6 +155,9 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *api.Account, er
 func (d *Database) CreateAccount(
 	ctx context.Context, localpart, plaintextPassword, appserviceID string,
 ) (acc *api.Account, err error) {
+	// Create one account at a time else we can get 'database is locked'.
+	d.createAccountMu.Lock()
+	defer d.createAccountMu.Unlock()
 	err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
 		acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID)
 		return err

From c2d34422d65e81eee6e9d0c31a4c5a446fa9678a Mon Sep 17 00:00:00 2001
From: Ashley Nelson <amn@fantashley.com>
Date: Thu, 25 Jun 2020 06:27:09 -0500
Subject: [PATCH 04/23] Remove trailing slash in client api proxy (#1163)

Signed-off-by: Ashley Nelson <amn@fantashley.com>
---
 cmd/client-api-proxy/main.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cmd/client-api-proxy/main.go b/cmd/client-api-proxy/main.go
index 979b0b042..ebc0629f8 100644
--- a/cmd/client-api-proxy/main.go
+++ b/cmd/client-api-proxy/main.go
@@ -58,9 +58,8 @@ var (
 )
 
 func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
-	if !strings.HasSuffix(targetURL, "/") {
-		targetURL += "/"
-	}
+	targetURL = strings.TrimSuffix(targetURL, "/")
+
 	// Check that we can parse the URL.
 	_, err := url.Parse(targetURL)
 	if err != nil {

From 43cddfe00f53e3a2df4769be31c66cd818a2966e Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Thu, 25 Jun 2020 15:04:48 +0100
Subject: [PATCH 05/23] Return remote errors from FS.PerformJoin (#1164)

* Return remote errors from FS.PerformJoin

Follows the same pattern as PerformJoin on roomserver (no error return).

Also return the right format for incompatible room version errors.

Makes a bunch of tests pass!

* Handle network errors better when returning remote HTTP errors

* Linting

* Fix tests

* Update whitelist, pass network errors through in API=1 mode
---
 clientapi/jsonerror/jsonerror.go     | 14 ++++++++++++--
 federationsender/api/api.go          |  4 +++-
 federationsender/internal/perform.go | 28 +++++++++++++++++++++++-----
 federationsender/inthttp/client.go   | 12 ++++++++++--
 federationsender/inthttp/server.go   |  4 +---
 go.mod                               |  2 +-
 go.sum                               |  4 ++--
 roomserver/api/perform.go            | 19 +++++++++++++++++--
 roomserver/internal/perform_join.go  | 10 +++++++---
 sytest-whitelist                     |  7 +++++++
 10 files changed, 83 insertions(+), 21 deletions(-)

diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go
index 85e887aec..7f8f264b7 100644
--- a/clientapi/jsonerror/jsonerror.go
+++ b/clientapi/jsonerror/jsonerror.go
@@ -125,10 +125,20 @@ func GuestAccessForbidden(msg string) *MatrixError {
 	return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg}
 }
 
+type IncompatibleRoomVersionError struct {
+	RoomVersion string `json:"room_version"`
+	Error       string `json:"error"`
+	Code        string `json:"errcode"`
+}
+
 // IncompatibleRoomVersion is an error which is returned when the client
 // requests a room with a version that is unsupported.
-func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *MatrixError {
-	return &MatrixError{"M_INCOMPATIBLE_ROOM_VERSION", string(roomVersion)}
+func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *IncompatibleRoomVersionError {
+	return &IncompatibleRoomVersionError{
+		Code:        "M_INCOMPATIBLE_ROOM_VERSION",
+		RoomVersion: string(roomVersion),
+		Error:       "Your homeserver does not support the features required to join this room",
+	}
 }
 
 // UnsupportedRoomVersion is an error which is returned when the client
diff --git a/federationsender/api/api.go b/federationsender/api/api.go
index 02c762582..d90ffd290 100644
--- a/federationsender/api/api.go
+++ b/federationsender/api/api.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/matrix-org/dendrite/federationsender/types"
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -28,7 +29,7 @@ type FederationSenderInternalAPI interface {
 		ctx context.Context,
 		request *PerformJoinRequest,
 		response *PerformJoinResponse,
-	) error
+	)
 	// Handle an instruction to make_leave & send_leave with a remote server.
 	PerformLeave(
 		ctx context.Context,
@@ -62,6 +63,7 @@ type PerformJoinRequest struct {
 }
 
 type PerformJoinResponse struct {
+	LastError *gomatrix.HTTPError
 }
 
 type PerformLeaveRequest struct {
diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go
index 7ced4af86..96b1149d9 100644
--- a/federationsender/internal/perform.go
+++ b/federationsender/internal/perform.go
@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -9,6 +10,7 @@ import (
 	"github.com/matrix-org/dendrite/federationsender/internal/perform"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
@@ -40,7 +42,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 	ctx context.Context,
 	request *api.PerformJoinRequest,
 	response *api.PerformJoinResponse,
-) (err error) {
+) {
 	// Look up the supported room versions.
 	var supportedVersions []gomatrixserverlib.RoomVersion
 	for version := range version.SupportedRoomVersions() {
@@ -63,6 +65,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 
 	// Try each server that we were provided until we land on one that
 	// successfully completes the make-join send-join dance.
+	var lastErr error
 	for _, serverName := range request.ServerNames {
 		if err := r.performJoinUsingServer(
 			ctx,
@@ -76,17 +79,32 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 				"server_name": serverName,
 				"room_id":     request.RoomID,
 			}).Warnf("Failed to join room through server")
+			lastErr = err
 			continue
 		}
 
 		// We're all good.
-		return nil
+		return
 	}
 
 	// If we reach here then we didn't complete a join for some reason.
-	return fmt.Errorf(
-		"failed to join user %q to room %q through %d server(s)",
-		request.UserID, request.RoomID, len(request.ServerNames),
+	var httpErr gomatrix.HTTPError
+	if ok := errors.As(lastErr, &httpErr); ok {
+		httpErr.Message = string(httpErr.Contents)
+		// Clear the wrapped error, else serialising to JSON (in polylith mode) will fail
+		httpErr.WrappedError = nil
+		response.LastError = &httpErr
+	} else {
+		response.LastError = &gomatrix.HTTPError{
+			Code:         0,
+			WrappedError: nil,
+			Message:      lastErr.Error(),
+		}
+	}
+
+	logrus.Errorf(
+		"failed to join user %q to room %q through %d server(s): last error %s",
+		request.UserID, request.RoomID, len(request.ServerNames), lastErr,
 	)
 }
 
diff --git a/federationsender/inthttp/client.go b/federationsender/inthttp/client.go
index 5da4b35f9..25de99cce 100644
--- a/federationsender/inthttp/client.go
+++ b/federationsender/inthttp/client.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/gomatrix"
 	"github.com/opentracing/opentracing-go"
 )
 
@@ -77,12 +78,19 @@ func (h *httpFederationSenderInternalAPI) PerformJoin(
 	ctx context.Context,
 	request *api.PerformJoinRequest,
 	response *api.PerformJoinResponse,
-) error {
+) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest")
 	defer span.Finish()
 
 	apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	if err != nil {
+		response.LastError = &gomatrix.HTTPError{
+			Message:      err.Error(),
+			Code:         0,
+			WrappedError: err,
+		}
+	}
 }
 
 // Handle an instruction to make_join & send_join with a remote server.
diff --git a/federationsender/inthttp/server.go b/federationsender/inthttp/server.go
index babd3ae13..a4f3d63d0 100644
--- a/federationsender/inthttp/server.go
+++ b/federationsender/inthttp/server.go
@@ -33,9 +33,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
 				return util.MessageResponse(http.StatusBadRequest, err.Error())
 			}
-			if err := intAPI.PerformJoin(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
+			intAPI.PerformJoin(req.Context(), &request, &response)
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
diff --git a/go.mod b/go.mod
index 6bfce8441..57cdb9095 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible
diff --git a/go.sum b/go.sum
index 6178f152b..973b003c6 100644
--- a/go.sum
+++ b/go.sum
@@ -371,8 +371,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137 h1:+eBh4L04+08IslvFM071TNrQTggU317GsQKzZ1SGEVo=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1 h1:3yS6hw01X72jpJuAPGVOY+QFD9cpAETR/6Hq2WYKbpU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index 12ba15167..5d8d88a5a 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -1,6 +1,7 @@
 package api
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 
@@ -12,8 +13,9 @@ import (
 type PerformErrorCode int
 
 type PerformError struct {
-	Msg  string
-	Code PerformErrorCode
+	Msg        string
+	RemoteCode int // remote HTTP status code, for PerformErrRemote
+	Code       PerformErrorCode
 }
 
 func (p *PerformError) Error() string {
@@ -43,6 +45,17 @@ func (p *PerformError) JSONResponse() util.JSONResponse {
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden(p.Msg),
 		}
+	case PerformErrRemote:
+		// if the code is 0 then something bad happened and it isn't
+		// a remote HTTP error being encapsulated, e.g network error to remote.
+		if p.RemoteCode == 0 {
+			return util.ErrorResponse(fmt.Errorf("%s", p.Msg))
+		}
+		return util.JSONResponse{
+			Code: p.RemoteCode,
+			// TODO: Should we assert this is in fact JSON? E.g gjson parse?
+			JSON: json.RawMessage(p.Msg),
+		}
 	default:
 		return util.ErrorResponse(p)
 	}
@@ -57,6 +70,8 @@ const (
 	PerformErrorNoRoom PerformErrorCode = 3
 	// PerformErrorNoOperation means that the request resulted in nothing happening e.g invite->invite or leave->leave.
 	PerformErrorNoOperation PerformErrorCode = 4
+	// PerformErrRemote means that the request failed and the PerformError.Msg is the raw remote JSON error response
+	PerformErrRemote PerformErrorCode = 5
 )
 
 type PerformJoinRequest struct {
diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go
index d409b6849..b594c2d87 100644
--- a/roomserver/internal/perform_join.go
+++ b/roomserver/internal/perform_join.go
@@ -270,9 +270,13 @@ func (r *RoomserverInternalAPI) performFederatedJoinRoomByID(
 		Content:     req.Content,       // the membership event content
 	}
 	fedRes := fsAPI.PerformJoinResponse{}
-	if err := r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes); err != nil {
-		return fmt.Errorf("Error joining federated room: %q", err)
+	r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes)
+	if fedRes.LastError != nil {
+		return &api.PerformError{
+			Code:       api.PerformErrRemote,
+			Msg:        fedRes.LastError.Message,
+			RemoteCode: fedRes.LastError.Code,
+		}
 	}
-
 	return nil
 }
diff --git a/sytest-whitelist b/sytest-whitelist
index ce97e8af7..eb28898ab 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -366,3 +366,10 @@ Typing notification sent to local room members
 Typing notifications also sent to remote room members
 Typing can be explicitly stopped
 Banned user is kicked and may not rejoin until unbanned
+Inbound federation rejects attempts to join v1 rooms from servers without v1 support
+Inbound federation rejects attempts to join v2 rooms from servers lacking version support
+Inbound federation rejects attempts to join v2 rooms from servers only supporting v1
+Outbound federation passes make_join failures through to the client
+Outbound federation correctly handles unsupported room versions
+Remote users may not join unfederated rooms
+Guest users denied access over federation if guest access prohibited

From 67f7a53f12b67f69ca90dd251918463d03e3c271 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 25 Jun 2020 15:06:46 +0100
Subject: [PATCH 06/23] Add missing typing test

---
 sytest-whitelist | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sytest-whitelist b/sytest-whitelist
index eb28898ab..4bc7baefe 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -362,6 +362,7 @@ Users cannot invite a user that is already in the room
 Invited user can reject invite
 Invited user can reject invite for empty room
 Invited user can reject local invite after originator leaves
+PUT /rooms/:room_id/typing/:user_id sets typing notification
 Typing notification sent to local room members
 Typing notifications also sent to remote room members
 Typing can be explicitly stopped

From 7a8282fccfc435e20f3fd2763745dbd3b0de2bed Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Thu, 25 Jun 2020 17:07:40 +0100
Subject: [PATCH 07/23] Handle invite v1 (#1165)

* Implement invite v1 for sytest mainly

* Bump gmsl version which falls back to invite v1 if v2 404s

* Update whitelist
---
 federationapi/routing/invite.go  | 67 +++++++++++++++++++++++++-------
 federationapi/routing/routing.go | 18 ++++++++-
 go.mod                           |  2 +-
 go.sum                           |  2 +
 sytest-whitelist                 |  3 ++
 5 files changed, 77 insertions(+), 15 deletions(-)

diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go
index b1d84f254..4a49463a2 100644
--- a/federationapi/routing/invite.go
+++ b/federationapi/routing/invite.go
@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -27,8 +28,8 @@ import (
 	"github.com/matrix-org/util"
 )
 
-// Invite implements /_matrix/federation/v2/invite/{roomID}/{eventID}
-func Invite(
+// InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID}
+func InviteV2(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
@@ -44,14 +45,58 @@ func Invite(
 			JSON: jsonerror.NotJSON("The request body could not be decoded into an invite request. " + err.Error()),
 		}
 	}
-	event := inviteReq.Event()
+	return processInvite(
+		httpReq.Context(), inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys,
+	)
+}
+
+// InviteV1 implements /_matrix/federation/v1/invite/{roomID}/{eventID}
+func InviteV1(
+	httpReq *http.Request,
+	request *gomatrixserverlib.FederationRequest,
+	roomID string,
+	eventID string,
+	cfg *config.Dendrite,
+	rsAPI api.RoomserverInternalAPI,
+	keys gomatrixserverlib.JSONVerifier,
+) util.JSONResponse {
+	roomVer := gomatrixserverlib.RoomVersionV1
+	body := request.Content()
+	event, err := gomatrixserverlib.NewEventFromTrustedJSON(body, false, roomVer)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotJSON("The request body could not be decoded into an invite v1 request: " + err.Error()),
+		}
+	}
+	var strippedState []gomatrixserverlib.InviteV2StrippedState
+	if err := json.Unmarshal(event.Unsigned(), &strippedState); err != nil {
+		// just warn, they may not have added any.
+		util.GetLogger(httpReq.Context()).Warnf("failed to extract stripped state from invite event")
+	}
+	return processInvite(
+		httpReq.Context(), event, roomVer, strippedState, roomID, eventID, cfg, rsAPI, keys,
+	)
+}
+
+func processInvite(
+	ctx context.Context,
+	event gomatrixserverlib.Event,
+	roomVer gomatrixserverlib.RoomVersion,
+	strippedState []gomatrixserverlib.InviteV2StrippedState,
+	roomID string,
+	eventID string,
+	cfg *config.Dendrite,
+	rsAPI api.RoomserverInternalAPI,
+	keys gomatrixserverlib.JSONVerifier,
+) util.JSONResponse {
 
 	// Check that we can accept invites for this room version.
-	if _, err := roomserverVersion.SupportedRoomVersion(inviteReq.RoomVersion()); err != nil {
+	if _, err := roomserverVersion.SupportedRoomVersion(roomVer); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(
-				fmt.Sprintf("Room version %q is not supported by this server.", inviteReq.RoomVersion()),
+				fmt.Sprintf("Room version %q is not supported by this server.", roomVer),
 			),
 		}
 	}
@@ -80,9 +125,9 @@ func Invite(
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}
-	verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests)
+	verifyResults, err := keys.VerifyJSONs(ctx, verifyRequests)
 	if err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("keys.VerifyJSONs failed")
+		util.GetLogger(ctx).WithError(err).Error("keys.VerifyJSONs failed")
 		return jsonerror.InternalServerError()
 	}
 	if verifyResults[0].Error != nil {
@@ -99,13 +144,9 @@ func Invite(
 
 	// Add the invite event to the roomserver.
 	if perr := api.SendInvite(
-		httpReq.Context(), rsAPI,
-		signedEvent.Headered(inviteReq.RoomVersion()),
-		inviteReq.InviteRoomState(),
-		event.Origin(),
-		nil,
+		ctx, rsAPI, signedEvent.Headered(roomVer), strippedState, event.Origin(), nil,
 	); perr != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed")
+		util.GetLogger(ctx).WithError(err).Error("producer.SendInvite failed")
 		return perr.JSONResponse()
 	}
 
diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go
index 645f397de..0afea7d04 100644
--- a/federationapi/routing/routing.go
+++ b/federationapi/routing/routing.go
@@ -83,10 +83,26 @@ func Setup(
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
 
+	v1fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
+		"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
+		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
+			res := InviteV1(
+				httpReq, request, vars["roomID"], vars["eventID"],
+				cfg, rsAPI, keys,
+			)
+			return util.JSONResponse{
+				Code: res.Code,
+				JSON: []interface{}{
+					res.Code, res.JSON,
+				},
+			}
+		},
+	)).Methods(http.MethodPut, http.MethodOptions)
+
 	v2fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
 		"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
-			return Invite(
+			return InviteV2(
 				httpReq, request, vars["roomID"], vars["eventID"],
 				cfg, rsAPI, keys,
 			)
diff --git a/go.mod b/go.mod
index 57cdb9095..7d968b23e 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible
diff --git a/go.sum b/go.sum
index 973b003c6..5a9547ef3 100644
--- a/go.sum
+++ b/go.sum
@@ -373,6 +373,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1 h1:3yS6hw01X72jpJuAPGVOY+QFD9cpAETR/6Hq2WYKbpU=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1 h1:QDOdGCfrzuVLEess3id2a2B29oVZ9JXgJmUfwE7r/iI=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
diff --git a/sytest-whitelist b/sytest-whitelist
index 4bc7baefe..1628a6352 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -243,6 +243,9 @@ User can invite local user to room with version 2
 Remote user can backfill in a room with version 2
 Inbound federation accepts attempts to join v2 rooms from servers with support
 Outbound federation can send invites via v2 API
+Outbound federation can send invites via v1 API
+Inbound federation can receive invites via v1 API
+Inbound federation can receive invites via v2 API
 User can create and send/receive messages in a room with version 3
 local user can join room with version 3
 Remote user can backfill in a room with version 3

From c1d2382e6d7f459ddf911a16aac7d4e63d50838b Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 25 Jun 2020 18:05:41 +0100
Subject: [PATCH 08/23] Reject non-numeric ports (done in GMSL)

---
 go.mod           | 2 +-
 go.sum           | 2 ++
 sytest-whitelist | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 7d968b23e..b37a662ae 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible
diff --git a/go.sum b/go.sum
index 5a9547ef3..bfe5533d7 100644
--- a/go.sum
+++ b/go.sum
@@ -375,6 +375,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1 h1:3y
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1 h1:QDOdGCfrzuVLEess3id2a2B29oVZ9JXgJmUfwE7r/iI=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d h1:v1JS+JZWwAsqAc22TGWPbRDc6O5D6geSfV5Bb5wvYIs=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
diff --git a/sytest-whitelist b/sytest-whitelist
index 1628a6352..47fabc8a4 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -377,3 +377,4 @@ Outbound federation passes make_join failures through to the client
 Outbound federation correctly handles unsupported room versions
 Remote users may not join unfederated rooms
 Guest users denied access over federation if guest access prohibited
+Non-numeric ports in server names are rejected

From 4897beabeed3281f3e45a1426e6f1c9359e3152b Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Fri, 26 Jun 2020 11:07:52 +0100
Subject: [PATCH 09/23] Finish implementing retiring invites (#1166)

* Pass retired invites to the syncapi with the event ID of the invite

* Implement retire invite streaming

* Update whitelist
---
 roomserver/internal/perform_join.go         |  2 +-
 roomserver/internal/perform_leave.go        | 47 ++++++++-----
 roomserver/storage/interface.go             |  4 +-
 roomserver/storage/postgres/invite_table.go | 15 ++--
 roomserver/storage/shared/storage.go        |  2 +-
 roomserver/storage/sqlite3/invite_table.go  | 15 ++--
 roomserver/storage/tables/interface.go      |  4 +-
 syncapi/consumers/roomserver.go             |  7 +-
 syncapi/storage/interface.go                |  4 +-
 syncapi/storage/postgres/invites_table.go   | 37 ++++++----
 syncapi/storage/shared/syncserver.go        | 13 ++--
 syncapi/storage/sqlite3/invites_table.go    | 42 ++++++-----
 syncapi/storage/storage_test.go             | 77 +++++++++++++++++++++
 syncapi/storage/tables/interface.go         |  4 +-
 syncapi/types/types.go                      |  4 +-
 sytest-whitelist                            |  8 +++
 16 files changed, 204 insertions(+), 81 deletions(-)

diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go
index b594c2d87..1a4508893 100644
--- a/roomserver/internal/perform_join.go
+++ b/roomserver/internal/perform_join.go
@@ -155,7 +155,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID(
 	// where we might think we know about a room in the following
 	// section but don't know the latest state as all of our users
 	// have left.
-	isInvitePending, inviteSender, err := r.isInvitePending(ctx, req.RoomIDOrAlias, req.UserID)
+	isInvitePending, inviteSender, _, err := r.isInvitePending(ctx, req.RoomIDOrAlias, req.UserID)
 	if err == nil && isInvitePending {
 		// Check if there's an invite pending.
 		_, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender)
diff --git a/roomserver/internal/perform_leave.go b/roomserver/internal/perform_leave.go
index 880c8b203..a19d0da9f 100644
--- a/roomserver/internal/perform_leave.go
+++ b/roomserver/internal/perform_leave.go
@@ -9,6 +9,7 @@ import (
 	fsAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -38,9 +39,9 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID(
 ) error {
 	// If there's an invite outstanding for the room then respond to
 	// that.
-	isInvitePending, senderUser, err := r.isInvitePending(ctx, req.RoomID, req.UserID)
+	isInvitePending, senderUser, eventID, err := r.isInvitePending(ctx, req.RoomID, req.UserID)
 	if err == nil && isInvitePending {
-		return r.performRejectInvite(ctx, req, res, senderUser)
+		return r.performRejectInvite(ctx, req, res, senderUser, eventID)
 	}
 
 	// There's no invite pending, so first of all we want to find out
@@ -134,7 +135,7 @@ func (r *RoomserverInternalAPI) performRejectInvite(
 	ctx context.Context,
 	req *api.PerformLeaveRequest,
 	res *api.PerformLeaveResponse, // nolint:unparam
-	senderUser string,
+	senderUser, eventID string,
 ) error {
 	_, domain, err := gomatrixserverlib.SplitID('@', senderUser)
 	if err != nil {
@@ -152,56 +153,68 @@ func (r *RoomserverInternalAPI) performRejectInvite(
 		return err
 	}
 
-	// TODO: Withdraw the invite, so that the sync API etc are
+	// Withdraw the invite, so that the sync API etc are
 	// notified that we rejected it.
-
-	return nil
+	return r.WriteOutputEvents(req.RoomID, []api.OutputEvent{
+		{
+			Type: api.OutputTypeRetireInviteEvent,
+			RetireInviteEvent: &api.OutputRetireInviteEvent{
+				EventID:      eventID,
+				Membership:   "leave",
+				TargetUserID: req.UserID,
+			},
+		},
+	})
 }
 
 func (r *RoomserverInternalAPI) isInvitePending(
 	ctx context.Context,
 	roomID, userID string,
-) (bool, string, error) {
+) (bool, string, string, error) {
 	// Look up the room NID for the supplied room ID.
 	roomNID, err := r.DB.RoomNID(ctx, roomID)
 	if err != nil {
-		return false, "", fmt.Errorf("r.DB.RoomNID: %w", err)
+		return false, "", "", fmt.Errorf("r.DB.RoomNID: %w", err)
 	}
 
 	// Look up the state key NID for the supplied user ID.
 	targetUserNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{userID})
 	if err != nil {
-		return false, "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err)
+		return false, "", "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err)
 	}
 	targetUserNID, targetUserFound := targetUserNIDs[userID]
 	if !targetUserFound {
-		return false, "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs)
+		return false, "", "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs)
 	}
 
 	// Let's see if we have an event active for the user in the room. If
 	// we do then it will contain a server name that we can direct the
 	// send_leave to.
-	senderUserNIDs, err := r.DB.GetInvitesForUser(ctx, roomNID, targetUserNID)
+	senderUserNIDs, eventIDs, err := r.DB.GetInvitesForUser(ctx, roomNID, targetUserNID)
 	if err != nil {
-		return false, "", fmt.Errorf("r.DB.GetInvitesForUser: %w", err)
+		return false, "", "", fmt.Errorf("r.DB.GetInvitesForUser: %w", err)
 	}
 	if len(senderUserNIDs) == 0 {
-		return false, "", nil
+		return false, "", "", nil
+	}
+	userNIDToEventID := make(map[types.EventStateKeyNID]string)
+	for i, nid := range senderUserNIDs {
+		userNIDToEventID[nid] = eventIDs[i]
 	}
 
 	// Look up the user ID from the NID.
 	senderUsers, err := r.DB.EventStateKeys(ctx, senderUserNIDs)
 	if err != nil {
-		return false, "", fmt.Errorf("r.DB.EventStateKeys: %w", err)
+		return false, "", "", fmt.Errorf("r.DB.EventStateKeys: %w", err)
 	}
 	if len(senderUsers) == 0 {
-		return false, "", fmt.Errorf("no senderUsers")
+		return false, "", "", fmt.Errorf("no senderUsers")
 	}
 
 	senderUser, senderUserFound := senderUsers[senderUserNIDs[0]]
 	if !senderUserFound {
-		return false, "", fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers)
+		return false, "", "", fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers)
 	}
 
-	return true, senderUser, nil
+	return true, senderUser, userNIDToEventID[senderUserNIDs[0]], nil
 }
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 52e6a96b7..0c4e2e0b5 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -102,9 +102,9 @@ type Database interface {
 	// Returns an error if there was a problem talking to the database.
 	LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error)
 	// Look up the active invites targeting a user in a room and return the
-	// numeric state key IDs for the user IDs who sent them.
+	// numeric state key IDs for the user IDs who sent them along with the event IDs for the invites.
 	// Returns an error if there was a problem talking to the database.
-	GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error)
+	GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error)
 	// Save a given room alias with the room ID it refers to.
 	// Returns an error if there was a problem talking to the database.
 	SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error
diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go
index 048a094dc..bb7195164 100644
--- a/roomserver/storage/postgres/invite_table.go
+++ b/roomserver/storage/postgres/invite_table.go
@@ -62,7 +62,7 @@ const insertInviteEventSQL = "" +
 	" ON CONFLICT DO NOTHING"
 
 const selectInviteActiveForUserInRoomSQL = "" +
-	"SELECT sender_nid FROM roomserver_invites" +
+	"SELECT invite_event_id, sender_nid FROM roomserver_invites" +
 	" WHERE target_nid = $1 AND room_nid = $2" +
 	" AND NOT retired"
 
@@ -141,21 +141,24 @@ func (s *inviteStatements) UpdateInviteRetired(
 func (s *inviteStatements) SelectInviteActiveForUserInRoom(
 	ctx context.Context,
 	targetUserNID types.EventStateKeyNID, roomNID types.RoomNID,
-) ([]types.EventStateKeyNID, error) {
+) ([]types.EventStateKeyNID, []string, error) {
 	rows, err := s.selectInviteActiveForUserInRoomStmt.QueryContext(
 		ctx, targetUserNID, roomNID,
 	)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed")
 	var result []types.EventStateKeyNID
+	var eventIDs []string
 	for rows.Next() {
+		var inviteEventID string
 		var senderUserNID int64
-		if err := rows.Scan(&senderUserNID); err != nil {
-			return nil, err
+		if err := rows.Scan(&inviteEventID, &senderUserNID); err != nil {
+			return nil, nil, err
 		}
 		result = append(result, types.EventStateKeyNID(senderUserNID))
+		eventIDs = append(eventIDs, inviteEventID)
 	}
-	return result, rows.Err()
+	return result, eventIDs, rows.Err()
 }
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 2751cc557..e6d0e34e2 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -265,7 +265,7 @@ func (d *Database) GetInvitesForUser(
 	ctx context.Context,
 	roomNID types.RoomNID,
 	targetUserNID types.EventStateKeyNID,
-) (senderUserIDs []types.EventStateKeyNID, err error) {
+) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error) {
 	return d.InvitesTable.SelectInviteActiveForUserInRoom(ctx, targetUserNID, roomNID)
 }
 
diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go
index 21745d1b0..8b6cbe3fc 100644
--- a/roomserver/storage/sqlite3/invite_table.go
+++ b/roomserver/storage/sqlite3/invite_table.go
@@ -45,7 +45,7 @@ const insertInviteEventSQL = "" +
 	" ON CONFLICT DO NOTHING"
 
 const selectInviteActiveForUserInRoomSQL = "" +
-	"SELECT sender_nid FROM roomserver_invites" +
+	"SELECT invite_event_id, sender_nid FROM roomserver_invites" +
 	" WHERE target_nid = $1 AND room_nid = $2" +
 	" AND NOT retired"
 
@@ -133,21 +133,24 @@ func (s *inviteStatements) UpdateInviteRetired(
 func (s *inviteStatements) SelectInviteActiveForUserInRoom(
 	ctx context.Context,
 	targetUserNID types.EventStateKeyNID, roomNID types.RoomNID,
-) ([]types.EventStateKeyNID, error) {
+) ([]types.EventStateKeyNID, []string, error) {
 	rows, err := s.selectInviteActiveForUserInRoomStmt.QueryContext(
 		ctx, targetUserNID, roomNID,
 	)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed")
 	var result []types.EventStateKeyNID
+	var eventIDs []string
 	for rows.Next() {
+		var eventID string
 		var senderUserNID int64
-		if err := rows.Scan(&senderUserNID); err != nil {
-			return nil, err
+		if err := rows.Scan(&eventID, &senderUserNID); err != nil {
+			return nil, nil, err
 		}
 		result = append(result, types.EventStateKeyNID(senderUserNID))
+		eventIDs = append(eventIDs, eventID)
 	}
-	return result, nil
+	return result, eventIDs, nil
 }
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 11cff8a8b..3aa8c538c 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -100,8 +100,8 @@ type PreviousEvents interface {
 type Invites interface {
 	InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte) (bool, error)
 	UpdateInviteRetired(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) ([]string, error)
-	// SelectInviteActiveForUserInRoom returns a list of sender state key NIDs
-	SelectInviteActiveForUserInRoom(ctx context.Context, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, error)
+	// SelectInviteActiveForUserInRoom returns a list of sender state key NIDs and invite event IDs matching those nids.
+	SelectInviteActiveForUserInRoom(ctx context.Context, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, error)
 }
 
 type MembershipState int64
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 98be5bb73..af7f612b3 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -157,7 +157,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent(
 func (s *OutputRoomEventConsumer) onRetireInviteEvent(
 	ctx context.Context, msg api.OutputRetireInviteEvent,
 ) error {
-	err := s.db.RetireInviteEvent(ctx, msg.EventID)
+	sp, err := s.db.RetireInviteEvent(ctx, msg.EventID)
 	if err != nil {
 		// panic rather than continue with an inconsistent database
 		log.WithFields(log.Fields{
@@ -166,8 +166,9 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent(
 		}).Panicf("roomserver output log: remove invite failure")
 		return nil
 	}
-	// TODO: Notify any active sync requests that the invite has been retired.
-	// s.notifier.OnNewEvent(nil, msg.TargetUserID, syncStreamPos)
+	// Notify any active sync requests that the invite has been retired.
+	// Invites share the same stream counter as PDUs
+	s.notifier.OnNewEvent(nil, "", []string{msg.TargetUserID}, types.NewStreamToken(sp, 0))
 	return nil
 }
 
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 7b3bd6785..c693326b4 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -78,9 +78,9 @@ type Database interface {
 	// If the invite was successfully stored this returns the stream ID it was stored at.
 	// Returns an error if there was a problem communicating with the database.
 	AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error)
-	// RetireInviteEvent removes an old invite event from the database.
+	// RetireInviteEvent removes an old invite event from the database. Returns the new position of the retired invite.
 	// Returns an error if there was a problem communicating with the database.
-	RetireInviteEvent(ctx context.Context, inviteEventID string) error
+	RetireInviteEvent(ctx context.Context, inviteEventID string) (types.StreamPosition, error)
 	// SetTypingTimeoutCallback sets a callback function that is called right after
 	// a user is removed from the typing user list due to timeout.
 	SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn)
diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go
index 5031d64e5..530dc6452 100644
--- a/syncapi/storage/postgres/invites_table.go
+++ b/syncapi/storage/postgres/invites_table.go
@@ -33,7 +33,8 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events (
 	event_id TEXT NOT NULL,
 	room_id TEXT NOT NULL,
 	target_user_id TEXT NOT NULL,
-	headered_event_json TEXT NOT NULL
+	headered_event_json TEXT NOT NULL,
+	deleted BOOL NOT NULL
 );
 
 -- For looking up the invites for a given user.
@@ -47,14 +48,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx
 
 const insertInviteEventSQL = "" +
 	"INSERT INTO syncapi_invite_events (" +
-	" room_id, event_id, target_user_id, headered_event_json" +
-	") VALUES ($1, $2, $3, $4) RETURNING id"
+	" room_id, event_id, target_user_id, headered_event_json, deleted" +
+	") VALUES ($1, $2, $3, $4, FALSE) RETURNING id"
 
 const deleteInviteEventSQL = "" +
-	"DELETE FROM syncapi_invite_events WHERE event_id = $1"
+	"UPDATE syncapi_invite_events SET deleted=TRUE, id=nextval('syncapi_stream_id') WHERE event_id = $1 RETURNING id"
 
 const selectInviteEventsInRangeSQL = "" +
-	"SELECT room_id, headered_event_json FROM syncapi_invite_events" +
+	"SELECT room_id, headered_event_json, deleted FROM syncapi_invite_events" +
 	" WHERE target_user_id = $1 AND id > $2 AND id <= $3" +
 	" ORDER BY id DESC"
 
@@ -110,40 +111,46 @@ func (s *inviteEventsStatements) InsertInviteEvent(
 
 func (s *inviteEventsStatements) DeleteInviteEvent(
 	ctx context.Context, inviteEventID string,
-) error {
-	_, err := s.deleteInviteEventStmt.ExecContext(ctx, inviteEventID)
-	return err
+) (sp types.StreamPosition, err error) {
+	err = s.deleteInviteEventStmt.QueryRowContext(ctx, inviteEventID).Scan(&sp)
+	return
 }
 
 // selectInviteEventsInRange returns a map of room ID to invite event for the
 // active invites for the target user ID in the supplied range.
 func (s *inviteEventsStatements) SelectInviteEventsInRange(
 	ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range,
-) (map[string]gomatrixserverlib.HeaderedEvent, error) {
+) (map[string]gomatrixserverlib.HeaderedEvent, map[string]gomatrixserverlib.HeaderedEvent, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt)
 	rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High())
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed")
 	result := map[string]gomatrixserverlib.HeaderedEvent{}
+	retired := map[string]gomatrixserverlib.HeaderedEvent{}
 	for rows.Next() {
 		var (
 			roomID    string
 			eventJSON []byte
+			deleted   bool
 		)
-		if err = rows.Scan(&roomID, &eventJSON); err != nil {
-			return nil, err
+		if err = rows.Scan(&roomID, &eventJSON, &deleted); err != nil {
+			return nil, nil, err
 		}
 
 		var event gomatrixserverlib.HeaderedEvent
 		if err := json.Unmarshal(eventJSON, &event); err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
-		result[roomID] = event
+		if deleted {
+			retired[roomID] = event
+		} else {
+			result[roomID] = event
+		}
 	}
-	return result, rows.Err()
+	return result, retired, rows.Err()
 }
 
 func (s *inviteEventsStatements) SelectMaxInviteID(
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 74ae3eabd..f84dc341e 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -180,11 +180,8 @@ func (d *Database) AddInviteEvent(
 // Returns an error if there was a problem communicating with the database.
 func (d *Database) RetireInviteEvent(
 	ctx context.Context, inviteEventID string,
-) error {
-	// TODO: Record that invite has been retired in a stream so that we can
-	// notify the user in an incremental sync.
-	err := d.Invites.DeleteInviteEvent(ctx, inviteEventID)
-	return err
+) (types.StreamPosition, error) {
+	return d.Invites.DeleteInviteEvent(ctx, inviteEventID)
 }
 
 // GetAccountDataInRange returns all account data for a given user inserted or
@@ -724,7 +721,7 @@ func (d *Database) addInvitesToResponse(
 	r types.Range,
 	res *types.Response,
 ) error {
-	invites, err := d.Invites.SelectInviteEventsInRange(
+	invites, retiredInvites, err := d.Invites.SelectInviteEventsInRange(
 		ctx, txn, userID, r,
 	)
 	if err != nil {
@@ -734,6 +731,10 @@ func (d *Database) addInvitesToResponse(
 		ir := types.NewInviteResponse(inviteEvent)
 		res.Rooms.Invite[roomID] = *ir
 	}
+	for roomID := range retiredInvites {
+		lr := types.NewLeaveResponse()
+		res.Rooms.Leave[roomID] = *lr
+	}
 	return nil
 }
 
diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go
index bb58e3456..aa0513888 100644
--- a/syncapi/storage/sqlite3/invites_table.go
+++ b/syncapi/storage/sqlite3/invites_table.go
@@ -33,7 +33,8 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events (
 	event_id TEXT NOT NULL,
 	room_id TEXT NOT NULL,
 	target_user_id TEXT NOT NULL,
-	headered_event_json TEXT NOT NULL
+	headered_event_json TEXT NOT NULL,
+	deleted BOOL NOT NULL
 );
 
 CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx ON syncapi_invite_events (target_user_id, id);
@@ -42,14 +43,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx ON syncapi_invite_events
 
 const insertInviteEventSQL = "" +
 	"INSERT INTO syncapi_invite_events" +
-	" (id, room_id, event_id, target_user_id, headered_event_json)" +
-	" VALUES ($1, $2, $3, $4, $5)"
+	" (id, room_id, event_id, target_user_id, headered_event_json, deleted)" +
+	" VALUES ($1, $2, $3, $4, $5, false)"
 
 const deleteInviteEventSQL = "" +
-	"DELETE FROM syncapi_invite_events WHERE event_id = $1"
+	"UPDATE syncapi_invite_events SET deleted=true, id=$1 WHERE event_id = $2"
 
 const selectInviteEventsInRangeSQL = "" +
-	"SELECT room_id, headered_event_json FROM syncapi_invite_events" +
+	"SELECT room_id, headered_event_json, deleted FROM syncapi_invite_events" +
 	" WHERE target_user_id = $1 AND id > $2 AND id <= $3" +
 	" ORDER BY id DESC"
 
@@ -114,40 +115,49 @@ func (s *inviteEventsStatements) InsertInviteEvent(
 
 func (s *inviteEventsStatements) DeleteInviteEvent(
 	ctx context.Context, inviteEventID string,
-) error {
-	_, err := s.deleteInviteEventStmt.ExecContext(ctx, inviteEventID)
-	return err
+) (types.StreamPosition, error) {
+	streamPos, err := s.streamIDStatements.nextStreamID(ctx, nil)
+	if err != nil {
+		return streamPos, err
+	}
+	_, err = s.deleteInviteEventStmt.ExecContext(ctx, streamPos, inviteEventID)
+	return streamPos, err
 }
 
 // selectInviteEventsInRange returns a map of room ID to invite event for the
 // active invites for the target user ID in the supplied range.
 func (s *inviteEventsStatements) SelectInviteEventsInRange(
 	ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range,
-) (map[string]gomatrixserverlib.HeaderedEvent, error) {
+) (map[string]gomatrixserverlib.HeaderedEvent, map[string]gomatrixserverlib.HeaderedEvent, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt)
 	rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High())
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed")
 	result := map[string]gomatrixserverlib.HeaderedEvent{}
+	retired := map[string]gomatrixserverlib.HeaderedEvent{}
 	for rows.Next() {
 		var (
 			roomID    string
 			eventJSON []byte
+			deleted   bool
 		)
-		if err = rows.Scan(&roomID, &eventJSON); err != nil {
-			return nil, err
+		if err = rows.Scan(&roomID, &eventJSON, &deleted); err != nil {
+			return nil, nil, err
 		}
 
 		var event gomatrixserverlib.HeaderedEvent
 		if err := json.Unmarshal(eventJSON, &event); err != nil {
-			return nil, err
+			return nil, nil, err
+		}
+		if deleted {
+			retired[roomID] = event
+		} else {
+			result[roomID] = event
 		}
-
-		result[roomID] = event
 	}
-	return result, nil
+	return result, retired, nil
 }
 
 func (s *inviteEventsStatements) SelectMaxInviteID(
diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go
index 85084facb..feacbc18c 100644
--- a/syncapi/storage/storage_test.go
+++ b/syncapi/storage/storage_test.go
@@ -601,6 +601,83 @@ func TestSendToDeviceBehaviour(t *testing.T) {
 	}
 }
 
+func TestInviteBehaviour(t *testing.T) {
+	db := MustCreateDatabase(t)
+	inviteRoom1 := "!inviteRoom1:somewhere"
+	inviteEvent1 := MustCreateEvent(t, inviteRoom1, nil, &gomatrixserverlib.EventBuilder{
+		Content:  []byte(fmt.Sprintf(`{"membership":"invite"}`)),
+		Type:     "m.room.member",
+		StateKey: &testUserIDA,
+		Sender:   "@inviteUser1:somewhere",
+	})
+	inviteRoom2 := "!inviteRoom2:somewhere"
+	inviteEvent2 := MustCreateEvent(t, inviteRoom2, nil, &gomatrixserverlib.EventBuilder{
+		Content:  []byte(fmt.Sprintf(`{"membership":"invite"}`)),
+		Type:     "m.room.member",
+		StateKey: &testUserIDA,
+		Sender:   "@inviteUser2:somewhere",
+	})
+	for _, ev := range []gomatrixserverlib.HeaderedEvent{inviteEvent1, inviteEvent2} {
+		_, err := db.AddInviteEvent(ctx, ev)
+		if err != nil {
+			t.Fatalf("Failed to AddInviteEvent: %s", err)
+		}
+	}
+	latest, err := db.SyncPosition(ctx)
+	if err != nil {
+		t.Fatalf("failed to get SyncPosition: %s", err)
+	}
+	// both invite events should appear in a new sync
+	beforeRetireRes := types.NewResponse()
+	beforeRetireRes, err = db.IncrementalSync(ctx, beforeRetireRes, testUserDeviceA, types.NewStreamToken(0, 0), latest, 0, false)
+	if err != nil {
+		t.Fatalf("IncrementalSync failed: %s", err)
+	}
+	assertInvitedToRooms(t, beforeRetireRes, []string{inviteRoom1, inviteRoom2})
+
+	// retire one event: a fresh sync should just return 1 invite room
+	if _, err = db.RetireInviteEvent(ctx, inviteEvent1.EventID()); err != nil {
+		t.Fatalf("Failed to RetireInviteEvent: %s", err)
+	}
+	latest, err = db.SyncPosition(ctx)
+	if err != nil {
+		t.Fatalf("failed to get SyncPosition: %s", err)
+	}
+	res := types.NewResponse()
+	res, err = db.IncrementalSync(ctx, res, testUserDeviceA, types.NewStreamToken(0, 0), latest, 0, false)
+	if err != nil {
+		t.Fatalf("IncrementalSync failed: %s", err)
+	}
+	assertInvitedToRooms(t, res, []string{inviteRoom2})
+
+	// a sync after we have received both invites should result in a leave for the retired room
+	beforeRetireTok, err := types.NewStreamTokenFromString(beforeRetireRes.NextBatch)
+	if err != nil {
+		t.Fatalf("NewStreamTokenFromString cannot parse next batch '%s' : %s", beforeRetireRes.NextBatch, err)
+	}
+	res = types.NewResponse()
+	res, err = db.IncrementalSync(ctx, res, testUserDeviceA, beforeRetireTok, latest, 0, false)
+	if err != nil {
+		t.Fatalf("IncrementalSync failed: %s", err)
+	}
+	assertInvitedToRooms(t, res, []string{})
+	if _, ok := res.Rooms.Leave[inviteRoom1]; !ok {
+		t.Fatalf("IncrementalSync: expected to see room left after it was retired but it wasn't")
+	}
+}
+
+func assertInvitedToRooms(t *testing.T, res *types.Response, roomIDs []string) {
+	t.Helper()
+	if len(res.Rooms.Invite) != len(roomIDs) {
+		t.Fatalf("got %d invited rooms, want %d", len(res.Rooms.Invite), len(roomIDs))
+	}
+	for _, roomID := range roomIDs {
+		if _, ok := res.Rooms.Invite[roomID]; !ok {
+			t.Fatalf("missing room ID %s", roomID)
+		}
+	}
+}
+
 func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) {
 	if len(gots) != len(wants) {
 		t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants))
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 0b7d15951..246dc6955 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -32,9 +32,9 @@ type AccountData interface {
 
 type Invites interface {
 	InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent) (streamPos types.StreamPosition, err error)
-	DeleteInviteEvent(ctx context.Context, inviteEventID string) error
+	DeleteInviteEvent(ctx context.Context, inviteEventID string) (types.StreamPosition, error)
 	// SelectInviteEventsInRange returns a map of room ID to invite events.
-	SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range) (map[string]gomatrixserverlib.HeaderedEvent, error)
+	SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range) (invites map[string]gomatrixserverlib.HeaderedEvent, retired map[string]gomatrixserverlib.HeaderedEvent, err error)
 	SelectMaxInviteID(ctx context.Context, txn *sql.Tx) (id int64, err error)
 }
 
diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index 1094416a1..019f2e69b 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -290,10 +290,10 @@ type Response struct {
 	NextBatch   string `json:"next_batch"`
 	AccountData struct {
 		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"account_data"`
+	} `json:"account_data,omitempty"`
 	Presence struct {
 		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"presence"`
+	} `json:"presence,omitempty"`
 	Rooms struct {
 		Join   map[string]JoinResponse   `json:"join"`
 		Invite map[string]InviteResponse `json:"invite"`
diff --git a/sytest-whitelist b/sytest-whitelist
index 47fabc8a4..02677d38a 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -378,3 +378,11 @@ Outbound federation correctly handles unsupported room versions
 Remote users may not join unfederated rooms
 Guest users denied access over federation if guest access prohibited
 Non-numeric ports in server names are rejected
+Invited user can reject invite over federation
+Invited user can reject invite over federation for empty room
+Can reject invites over federation for rooms with version 1
+Can reject invites over federation for rooms with version 2
+Can reject invites over federation for rooms with version 3
+Can reject invites over federation for rooms with version 4
+Can reject invites over federation for rooms with version 5
+Can reject invites over federation for rooms with version 6

From 9592d53364b573f9cd6ae045b98c49779429fa5f Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 26 Jun 2020 11:34:25 +0100
Subject: [PATCH 10/23] Fix div 0 error and add new tests to list

---
 are-we-synapse-yet.list | 6 +++++-
 are-we-synapse-yet.py   | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list
index f59f80675..3876de549 100644
--- a/are-we-synapse-yet.list
+++ b/are-we-synapse-yet.list
@@ -853,4 +853,8 @@ fme Outbound federation will ignore a missing event with bad JSON for room versi
 fbk Outbound federation rejects backfill containing invalid JSON for events in room version 6
 jso Invalid JSON integers
 jso Invalid JSON floats
-jso Invalid JSON special values
\ No newline at end of file
+jso Invalid JSON special values
+inv Can invite users to invite-only rooms (2 subtests)
+plv setting 'm.room.name' respects room powerlevel (2 subtests)
+psh Messages that notify from another user increment notification_count
+psh Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count
\ No newline at end of file
diff --git a/are-we-synapse-yet.py b/are-we-synapse-yet.py
index 30979a129..8cd7ec9fb 100755
--- a/are-we-synapse-yet.py
+++ b/are-we-synapse-yet.py
@@ -159,6 +159,8 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
     total_tests = 0
     for gid, tests in gid_to_tests.items():
         group_total = len(tests)
+        if group_total == 0:
+            continue
         group_passing = 0
         test_names_and_marks = []
         for name, passing in tests.items():

From 164057a3be1e666d6fb68398d616da9a8a665a18 Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Fri, 26 Jun 2020 12:51:54 +0100
Subject: [PATCH 11/23] Honour event size limits and return 413 (#1167)

---
 clientapi/routing/sendevent.go | 11 +++++++++++
 go.mod                         |  2 +-
 go.sum                         |  2 ++
 sytest-whitelist               |  1 +
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index d8936f750..aba5f0d51 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -157,6 +157,17 @@ func generateSendEvent(
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.BadJSON(e.Error()),
 		}
+	} else if e, ok := err.(gomatrixserverlib.EventValidationError); ok {
+		if e.Code == gomatrixserverlib.EventValidationTooLarge {
+			return nil, &util.JSONResponse{
+				Code: http.StatusRequestEntityTooLarge,
+				JSON: jsonerror.BadJSON(e.Error()),
+			}
+		}
+		return nil, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(e.Error()),
+		}
 	} else if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("eventutil.BuildEvent failed")
 		resErr := jsonerror.InternalServerError()
diff --git a/go.mod b/go.mod
index b37a662ae..5f5a74a1b 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible
diff --git a/go.sum b/go.sum
index bfe5533d7..24c8d74a0 100644
--- a/go.sum
+++ b/go.sum
@@ -377,6 +377,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1 h1:QD
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d h1:v1JS+JZWwAsqAc22TGWPbRDc6O5D6geSfV5Bb5wvYIs=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328 h1:rz6aiTpUyNPRcWZBWUGDkQjI7lfeLdhzy+x/Pw2jha8=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
diff --git a/sytest-whitelist b/sytest-whitelist
index 02677d38a..857457cdf 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -386,3 +386,4 @@ Can reject invites over federation for rooms with version 3
 Can reject invites over federation for rooms with version 4
 Can reject invites over federation for rooms with version 5
 Can reject invites over federation for rooms with version 6
+Event size limits

From 1ad7219e4b6c71f64e4d44db17a6a8d729e6198a Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Fri, 26 Jun 2020 15:34:41 +0100
Subject: [PATCH 12/23] Implement /sync `limited` and read timeline limit from
 stored filters (#1168)

* Move filter table to syncapi where it is used

* Implement /sync `limited` and read timeline limit from stored filters

We now fully handle `room.timeline.limit` filters (in-line + stored) and
return the right value for `limited` syncs.

* Update whitelist

* Default to the default timeline limit if it's unset, also strip the extra event correctly

* Update whitelist
---
 clientapi/routing/routing.go                  | 20 ----------
 {clientapi => syncapi}/routing/filter.go      | 40 ++++++++++++++-----
 syncapi/routing/routing.go                    | 20 ++++++++++
 syncapi/storage/interface.go                  |  8 ++++
 .../storage}/postgres/filter_table.go         | 30 +++++++-------
 .../postgres/output_room_events_table.go      | 22 +++++++---
 syncapi/storage/postgres/syncserver.go        |  5 +++
 syncapi/storage/shared/syncserver.go          | 24 ++++++++---
 .../storage}/sqlite3/filter_table.go          | 30 +++++++-------
 .../sqlite3/output_room_events_table.go       | 21 +++++++---
 syncapi/storage/sqlite3/syncserver.go         |  5 +++
 syncapi/storage/tables/interface.go           |  9 ++++-
 syncapi/sync/notifier_test.go                 |  2 +-
 syncapi/sync/request.go                       | 33 ++++++++++-----
 syncapi/sync/requestpool.go                   |  2 +-
 sytest-whitelist                              |  6 +++
 userapi/storage/accounts/interface.go         |  2 -
 userapi/storage/accounts/postgres/storage.go  | 25 +-----------
 userapi/storage/accounts/sqlite3/storage.go   | 25 +-----------
 19 files changed, 194 insertions(+), 135 deletions(-)
 rename {clientapi => syncapi}/routing/filter.go (65%)
 rename {userapi/storage/accounts => syncapi/storage}/postgres/filter_table.go (83%)
 rename {userapi/storage/accounts => syncapi/storage}/sqlite3/filter_table.go (83%)

diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index eadcfd1ab..9dfff0f20 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -376,26 +376,6 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/user/{userId}/filter",
-		httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return PutFilter(req, device, accountDB, vars["userId"])
-		}),
-	).Methods(http.MethodPost, http.MethodOptions)
-
-	r0mux.Handle("/user/{userId}/filter/{filterId}",
-		httputil.MakeAuthAPI("get_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"])
-		}),
-	).Methods(http.MethodGet, http.MethodOptions)
-
 	// Riot user settings
 
 	r0mux.Handle("/profile/{userID}",
diff --git a/clientapi/routing/filter.go b/syncapi/routing/filter.go
similarity index 65%
rename from clientapi/routing/filter.go
rename to syncapi/routing/filter.go
index 6520e6e40..baa4d841c 100644
--- a/clientapi/routing/filter.go
+++ b/syncapi/routing/filter.go
@@ -15,19 +15,22 @@
 package routing
 
 import (
+	"encoding/json"
+	"io/ioutil"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/sync"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/tidwall/gjson"
 )
 
 // GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId}
 func GetFilter(
-	req *http.Request, device *api.Device, accountDB accounts.Database, userID string, filterID string,
+	req *http.Request, device *api.Device, syncDB storage.Database, userID string, filterID string,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -41,7 +44,7 @@ func GetFilter(
 		return jsonerror.InternalServerError()
 	}
 
-	filter, err := accountDB.GetFilter(req.Context(), localpart, filterID)
+	filter, err := syncDB.GetFilter(req.Context(), localpart, filterID)
 	if err != nil {
 		//TODO better error handling. This error message is *probably* right,
 		// but if there are obscure db errors, this will also be returned,
@@ -64,7 +67,7 @@ type filterResponse struct {
 
 //PutFilter implements POST /_matrix/client/r0/user/{userId}/filter
 func PutFilter(
-	req *http.Request, device *api.Device, accountDB accounts.Database, userID string,
+	req *http.Request, device *api.Device, syncDB storage.Database, userID string,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -81,8 +84,27 @@ func PutFilter(
 
 	var filter gomatrixserverlib.Filter
 
-	if reqErr := httputil.UnmarshalJSONRequest(req, &filter); reqErr != nil {
-		return *reqErr
+	defer req.Body.Close() // nolint:errcheck
+	body, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The request body could not be read. " + err.Error()),
+		}
+	}
+
+	if err = json.Unmarshal(body, &filter); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
+		}
+	}
+	// the filter `limit` is `int` which defaults to 0 if not set which is not what we want. We want to use the default
+	// limit if it is unset, which is what this does.
+	limitRes := gjson.GetBytes(body, "room.timeline.limit")
+	if !limitRes.Exists() {
+		util.GetLogger(req.Context()).Infof("missing timeline limit, using default")
+		filter.Room.Timeline.Limit = sync.DefaultTimelineLimit
 	}
 
 	// Validate generates a user-friendly error
@@ -93,9 +115,9 @@ func PutFilter(
 		}
 	}
 
-	filterID, err := accountDB.PutFilter(req.Context(), localpart, &filter)
+	filterID, err := syncDB.PutFilter(req.Context(), localpart, &filter)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.PutFilter failed")
+		util.GetLogger(req.Context()).WithError(err).Error("syncDB.PutFilter failed")
 		return jsonerror.InternalServerError()
 	}
 
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index 5744de05a..a98955c57 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -55,4 +55,24 @@ func Setup(
 		}
 		return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, rsAPI, cfg)
 	})).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/user/{userId}/filter",
+		httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return PutFilter(req, device, syncDB, vars["userId"])
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/user/{userId}/filter/{filterId}",
+		httputil.MakeAuthAPI("get_filter", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetFilter(req, device, syncDB, vars["userId"], vars["filterId"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
 }
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index c693326b4..c4dae4d09 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -128,4 +128,12 @@ type Database interface {
 	CleanSendToDeviceUpdates(ctx context.Context, toUpdate, toDelete []types.SendToDeviceNID, token types.StreamingToken) (err error)
 	// SendToDeviceUpdatesWaiting returns true if there are send-to-device updates waiting to be sent.
 	SendToDeviceUpdatesWaiting(ctx context.Context, userID, deviceID string) (bool, error)
+	// GetFilter looks up the filter associated with a given local user and filter ID.
+	// Returns a filter structure. Otherwise returns an error if no such filter exists
+	// or if there was an error talking to the database.
+	GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error)
+	// PutFilter puts the passed filter into the database.
+	// Returns the filterID as a string. Otherwise returns an error if something
+	// goes wrong.
+	PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error)
 }
diff --git a/userapi/storage/accounts/postgres/filter_table.go b/syncapi/storage/postgres/filter_table.go
similarity index 83%
rename from userapi/storage/accounts/postgres/filter_table.go
rename to syncapi/storage/postgres/filter_table.go
index c54e4bc42..beeb864ba 100644
--- a/userapi/storage/accounts/postgres/filter_table.go
+++ b/syncapi/storage/postgres/filter_table.go
@@ -19,12 +19,13 @@ import (
 	"database/sql"
 	"encoding/json"
 
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const filterSchema = `
 -- Stores data about filters
-CREATE TABLE IF NOT EXISTS account_filter (
+CREATE TABLE IF NOT EXISTS syncapi_filter (
 	-- The filter
 	filter TEXT NOT NULL,
 	-- The ID
@@ -35,17 +36,17 @@ CREATE TABLE IF NOT EXISTS account_filter (
 	PRIMARY KEY(id, localpart)
 );
 
-CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart);
+CREATE INDEX IF NOT EXISTS syncapi_filter_localpart ON syncapi_filter(localpart);
 `
 
 const selectFilterSQL = "" +
-	"SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2"
+	"SELECT filter FROM syncapi_filter WHERE localpart = $1 AND id = $2"
 
 const selectFilterIDByContentSQL = "" +
-	"SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2"
+	"SELECT id FROM syncapi_filter WHERE localpart = $1 AND filter = $2"
 
 const insertFilterSQL = "" +
-	"INSERT INTO account_filter (filter, id, localpart) VALUES ($1, DEFAULT, $2) RETURNING id"
+	"INSERT INTO syncapi_filter (filter, id, localpart) VALUES ($1, DEFAULT, $2) RETURNING id"
 
 type filterStatements struct {
 	selectFilterStmt            *sql.Stmt
@@ -53,24 +54,25 @@ type filterStatements struct {
 	insertFilterStmt            *sql.Stmt
 }
 
-func (s *filterStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(filterSchema)
+func NewPostgresFilterTable(db *sql.DB) (tables.Filter, error) {
+	_, err := db.Exec(filterSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
+	s := &filterStatements{}
 	if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil {
-		return
+		return nil, err
 	}
 	if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil {
-		return
+		return nil, err
 	}
 	if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil {
-		return
+		return nil, err
 	}
-	return
+	return s, nil
 }
 
-func (s *filterStatements) selectFilter(
+func (s *filterStatements) SelectFilter(
 	ctx context.Context, localpart string, filterID string,
 ) (*gomatrixserverlib.Filter, error) {
 	// Retrieve filter from database (stored as canonical JSON)
@@ -88,7 +90,7 @@ func (s *filterStatements) selectFilter(
 	return &filter, nil
 }
 
-func (s *filterStatements) insertFilter(
+func (s *filterStatements) InsertFilter(
 	ctx context.Context, filter *gomatrixserverlib.Filter, localpart string,
 ) (filterID string, err error) {
 	var existingFilterID string
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index f01b2eabd..c7c4dc63b 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -301,21 +301,21 @@ func (s *outputRoomEventsStatements) SelectRecentEvents(
 	ctx context.Context, txn *sql.Tx,
 	roomID string, r types.Range, limit int,
 	chronologicalOrder bool, onlySyncEvents bool,
-) ([]types.StreamEvent, error) {
+) ([]types.StreamEvent, bool, error) {
 	var stmt *sql.Stmt
 	if onlySyncEvents {
 		stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt)
 	} else {
 		stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt)
 	}
-	rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit)
+	rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit+1)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed")
 	events, err := rowsToStreamEvents(rows)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	if chronologicalOrder {
 		// The events need to be returned from oldest to latest, which isn't
@@ -325,7 +325,19 @@ func (s *outputRoomEventsStatements) SelectRecentEvents(
 			return events[i].StreamPosition < events[j].StreamPosition
 		})
 	}
-	return events, nil
+	// we queried for 1 more than the limit, so if we returned one more mark limited=true
+	limited := false
+	if len(events) > limit {
+		limited = true
+		// re-slice the extra (oldest) event out: in chronological order this is the first entry, else the last.
+		if chronologicalOrder {
+			events = events[1:]
+		} else {
+			events = events[:len(events)-1]
+		}
+	}
+
+	return events, limited, nil
 }
 
 // selectEarlyEvents returns the earliest events in the given room, starting
diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go
index 573586cc7..10c1b37c7 100644
--- a/syncapi/storage/postgres/syncserver.go
+++ b/syncapi/storage/postgres/syncserver.go
@@ -71,6 +71,10 @@ func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*S
 	if err != nil {
 		return nil, err
 	}
+	filter, err := NewPostgresFilterTable(d.db)
+	if err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		Invites:             invites,
@@ -79,6 +83,7 @@ func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*S
 		Topology:            topology,
 		CurrentRoomState:    currState,
 		BackwardExtremities: backwardExtremities,
+		Filter:              filter,
 		SendToDevice:        sendToDevice,
 		SendToDeviceWriter:  sqlutil.NewTransactionWriter(),
 		EDUCache:            cache.New(),
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index f84dc341e..01362ddd6 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -43,6 +43,7 @@ type Database struct {
 	CurrentRoomState    tables.CurrentRoomState
 	BackwardExtremities tables.BackwardsExtremities
 	SendToDevice        tables.SendToDevice
+	Filter              tables.Filter
 	SendToDeviceWriter  *sqlutil.TransactionWriter
 	EDUCache            *cache.EDUCache
 }
@@ -78,7 +79,7 @@ func (d *Database) GetEventsInStreamingRange(
 	}
 	if backwardOrdering {
 		// When using backward ordering, we want the most recent events first.
-		if events, err = d.OutputEvents.SelectRecentEvents(
+		if events, _, err = d.OutputEvents.SelectRecentEvents(
 			ctx, nil, roomID, r, limit, false, false,
 		); err != nil {
 			return
@@ -545,6 +546,18 @@ func (d *Database) addEDUDeltaToResponse(
 	return
 }
 
+func (d *Database) GetFilter(
+	ctx context.Context, localpart string, filterID string,
+) (*gomatrixserverlib.Filter, error) {
+	return d.Filter.SelectFilter(ctx, localpart, filterID)
+}
+
+func (d *Database) PutFilter(
+	ctx context.Context, localpart string, filter *gomatrixserverlib.Filter,
+) (string, error) {
+	return d.Filter.InsertFilter(ctx, filter, localpart)
+}
+
 func (d *Database) IncrementalSync(
 	ctx context.Context, res *types.Response,
 	device userapi.Device,
@@ -642,7 +655,8 @@ func (d *Database) getResponseWithPDUsForCompleteSync(
 		// TODO: When filters are added, we may need to call this multiple times to get enough events.
 		//       See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316
 		var recentStreamEvents []types.StreamEvent
-		recentStreamEvents, err = d.OutputEvents.SelectRecentEvents(
+		var limited bool
+		recentStreamEvents, limited, err = d.OutputEvents.SelectRecentEvents(
 			ctx, txn, roomID, r, numRecentEventsPerRoom, true, true,
 		)
 		if err != nil {
@@ -670,7 +684,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync(
 		jr := types.NewJoinResponse()
 		jr.Timeline.PrevBatch = prevBatchStr
 		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
-		jr.Timeline.Limited = true
+		jr.Timeline.Limited = limited
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync)
 		res.Rooms.Join[roomID] = *jr
 	}
@@ -776,7 +790,7 @@ func (d *Database) addRoomDeltaToResponse(
 		// This is all "okay" assuming history_visibility == "shared" which it is by default.
 		r.To = delta.membershipPos
 	}
-	recentStreamEvents, err := d.OutputEvents.SelectRecentEvents(
+	recentStreamEvents, limited, err := d.OutputEvents.SelectRecentEvents(
 		ctx, txn, delta.roomID, r,
 		numRecentEventsPerRoom, true, true,
 	)
@@ -796,7 +810,7 @@ func (d *Database) addRoomDeltaToResponse(
 
 		jr.Timeline.PrevBatch = prevBatch.String()
 		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
-		jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
+		jr.Timeline.Limited = limited
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync)
 		res.Rooms.Join[delta.roomID] = *jr
 	case gomatrixserverlib.Leave:
diff --git a/userapi/storage/accounts/sqlite3/filter_table.go b/syncapi/storage/sqlite3/filter_table.go
similarity index 83%
rename from userapi/storage/accounts/sqlite3/filter_table.go
rename to syncapi/storage/sqlite3/filter_table.go
index 7f1a0c249..8b26759dc 100644
--- a/userapi/storage/accounts/sqlite3/filter_table.go
+++ b/syncapi/storage/sqlite3/filter_table.go
@@ -20,12 +20,13 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const filterSchema = `
 -- Stores data about filters
-CREATE TABLE IF NOT EXISTS account_filter (
+CREATE TABLE IF NOT EXISTS syncapi_filter (
 	-- The filter
 	filter TEXT NOT NULL,
 	-- The ID
@@ -36,17 +37,17 @@ CREATE TABLE IF NOT EXISTS account_filter (
 	UNIQUE (id, localpart)
 );
 
-CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart);
+CREATE INDEX IF NOT EXISTS syncapi_filter_localpart ON syncapi_filter(localpart);
 `
 
 const selectFilterSQL = "" +
-	"SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2"
+	"SELECT filter FROM syncapi_filter WHERE localpart = $1 AND id = $2"
 
 const selectFilterIDByContentSQL = "" +
-	"SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2"
+	"SELECT id FROM syncapi_filter WHERE localpart = $1 AND filter = $2"
 
 const insertFilterSQL = "" +
-	"INSERT INTO account_filter (filter, localpart) VALUES ($1, $2)"
+	"INSERT INTO syncapi_filter (filter, localpart) VALUES ($1, $2)"
 
 type filterStatements struct {
 	selectFilterStmt            *sql.Stmt
@@ -54,24 +55,25 @@ type filterStatements struct {
 	insertFilterStmt            *sql.Stmt
 }
 
-func (s *filterStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(filterSchema)
+func NewSqliteFilterTable(db *sql.DB) (tables.Filter, error) {
+	_, err := db.Exec(filterSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
+	s := &filterStatements{}
 	if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil {
-		return
+		return nil, err
 	}
 	if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil {
-		return
+		return nil, err
 	}
 	if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil {
-		return
+		return nil, err
 	}
-	return
+	return s, nil
 }
 
-func (s *filterStatements) selectFilter(
+func (s *filterStatements) SelectFilter(
 	ctx context.Context, localpart string, filterID string,
 ) (*gomatrixserverlib.Filter, error) {
 	// Retrieve filter from database (stored as canonical JSON)
@@ -89,7 +91,7 @@ func (s *filterStatements) selectFilter(
 	return &filter, nil
 }
 
-func (s *filterStatements) insertFilter(
+func (s *filterStatements) InsertFilter(
 	ctx context.Context, filter *gomatrixserverlib.Filter, localpart string,
 ) (filterID string, err error) {
 	var existingFilterID string
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 367ab3c9a..0c909cc4d 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -311,7 +311,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents(
 	ctx context.Context, txn *sql.Tx,
 	roomID string, r types.Range, limit int,
 	chronologicalOrder bool, onlySyncEvents bool,
-) ([]types.StreamEvent, error) {
+) ([]types.StreamEvent, bool, error) {
 	var stmt *sql.Stmt
 	if onlySyncEvents {
 		stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt)
@@ -319,14 +319,14 @@ func (s *outputRoomEventsStatements) SelectRecentEvents(
 		stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt)
 	}
 
-	rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit)
+	rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit+1)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed")
 	events, err := rowsToStreamEvents(rows)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	if chronologicalOrder {
 		// The events need to be returned from oldest to latest, which isn't
@@ -336,7 +336,18 @@ func (s *outputRoomEventsStatements) SelectRecentEvents(
 			return events[i].StreamPosition < events[j].StreamPosition
 		})
 	}
-	return events, nil
+	// we queried for 1 more than the limit, so if we returned one more mark limited=true
+	limited := false
+	if len(events) > limit {
+		limited = true
+		// re-slice the extra (oldest) event out: in chronological order this is the first entry, else the last.
+		if chronologicalOrder {
+			events = events[1:]
+		} else {
+			events = events[:len(events)-1]
+		}
+	}
+	return events, limited, nil
 }
 
 func (s *outputRoomEventsStatements) SelectEarlyEvents(
diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go
index 51cdbe325..c85db5a4f 100644
--- a/syncapi/storage/sqlite3/syncserver.go
+++ b/syncapi/storage/sqlite3/syncserver.go
@@ -87,6 +87,10 @@ func (d *SyncServerDatasource) prepare() (err error) {
 	if err != nil {
 		return err
 	}
+	filter, err := NewSqliteFilterTable(d.db)
+	if err != nil {
+		return err
+	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		Invites:             invites,
@@ -95,6 +99,7 @@ func (d *SyncServerDatasource) prepare() (err error) {
 		BackwardExtremities: bwExtrem,
 		CurrentRoomState:    roomState,
 		Topology:            topology,
+		Filter:              filter,
 		SendToDevice:        sendToDevice,
 		SendToDeviceWriter:  sqlutil.NewTransactionWriter(),
 		EDUCache:            cache.New(),
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 246dc6955..4ac0be4ec 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -44,8 +44,8 @@ type Events interface {
 	InsertEvent(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool) (streamPos types.StreamPosition, err error)
 	// SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high.
 	// If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync.
-	// Returns up to `limit` events.
-	SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, error)
+	// Returns up to `limit` events. Returns `limited=true` if there are more events in this range but we hit the `limit`.
+	SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error)
 	// SelectEarlyEvents returns the earliest events in the given room.
 	SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int) ([]types.StreamEvent, error)
 	SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error)
@@ -133,3 +133,8 @@ type SendToDevice interface {
 	DeleteSendToDeviceMessages(ctx context.Context, txn *sql.Tx, nids []types.SendToDeviceNID) (err error)
 	CountSendToDeviceMessages(ctx context.Context, txn *sql.Tx, userID, deviceID string) (count int, err error)
 }
+
+type Filter interface {
+	SelectFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error)
+	InsertFilter(ctx context.Context, filter *gomatrixserverlib.Filter, localpart string) (filterID string, err error)
+}
diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go
index ecc4fcbfc..f2a368ec2 100644
--- a/syncapi/sync/notifier_test.go
+++ b/syncapi/sync/notifier_test.go
@@ -363,7 +363,7 @@ func newTestSyncRequest(userID, deviceID string, since types.StreamingToken) syn
 		timeout:       1 * time.Minute,
 		since:         &since,
 		wantFullState: false,
-		limit:         defaultTimelineLimit,
+		limit:         DefaultTimelineLimit,
 		log:           util.GetLogger(context.TODO()),
 		ctx:           context.TODO(),
 	}
diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go
index 5dd92c853..41b18aa10 100644
--- a/syncapi/sync/request.go
+++ b/syncapi/sync/request.go
@@ -21,14 +21,16 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	log "github.com/sirupsen/logrus"
 )
 
 const defaultSyncTimeout = time.Duration(0)
-const defaultTimelineLimit = 20
+const DefaultTimelineLimit = 20
 
 type filter struct {
 	Room struct {
@@ -49,7 +51,7 @@ type syncRequest struct {
 	log           *log.Entry
 }
 
-func newSyncRequest(req *http.Request, device userapi.Device) (*syncRequest, error) {
+func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Database) (*syncRequest, error) {
 	timeout := getTimeout(req.URL.Query().Get("timeout"))
 	fullState := req.URL.Query().Get("full_state")
 	wantFullState := fullState != "" && fullState != "false"
@@ -66,15 +68,28 @@ func newSyncRequest(req *http.Request, device userapi.Device) (*syncRequest, err
 		tok := types.NewStreamToken(0, 0)
 		since = &tok
 	}
-	timelineLimit := defaultTimelineLimit
+	timelineLimit := DefaultTimelineLimit
 	// TODO: read from stored filters too
 	filterQuery := req.URL.Query().Get("filter")
-	if filterQuery != "" && filterQuery[0] == '{' {
-		// attempt to parse the timeline limit at least
-		var f filter
-		err := json.Unmarshal([]byte(filterQuery), &f)
-		if err == nil && f.Room.Timeline.Limit != nil {
-			timelineLimit = *f.Room.Timeline.Limit
+	if filterQuery != "" {
+		if filterQuery[0] == '{' {
+			// attempt to parse the timeline limit at least
+			var f filter
+			err := json.Unmarshal([]byte(filterQuery), &f)
+			if err == nil && f.Room.Timeline.Limit != nil {
+				timelineLimit = *f.Room.Timeline.Limit
+			}
+		} else {
+			// attempt to load the filter ID
+			localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+			if err != nil {
+				util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
+				return nil, err
+			}
+			f, err := syncDB.GetFilter(req.Context(), localpart, filterQuery)
+			if err == nil {
+				timelineLimit = f.Room.Timeline.Limit
+			}
 		}
 	}
 	// TODO: Additional query params: set_presence, filter
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index 743c63a62..196d446a2 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -49,7 +49,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 	var syncData *types.Response
 
 	// Extract values from request
-	syncReq, err := newSyncRequest(req, *device)
+	syncReq, err := newSyncRequest(req, *device, rp.db)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
diff --git a/sytest-whitelist b/sytest-whitelist
index 857457cdf..d055e75a2 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -387,3 +387,9 @@ Can reject invites over federation for rooms with version 4
 Can reject invites over federation for rooms with version 5
 Can reject invites over federation for rooms with version 6
 Event size limits
+Can sync a room with a single message
+Can sync a room with a message with a transaction id
+A full_state incremental update returns only recent timeline
+A prev_batch token can be used in the v1 messages API
+We don't send redundant membership state across incremental syncs by default
+Typing notifications don't leak
diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go
index c6692879b..9ed33e1b9 100644
--- a/userapi/storage/accounts/interface.go
+++ b/userapi/storage/accounts/interface.go
@@ -52,8 +52,6 @@ type Database interface {
 	RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error)
 	GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error)
 	GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error)
-	GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error)
-	PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error)
 	CheckAccountAvailability(ctx context.Context, localpart string) (bool, error)
 	GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error)
 }
diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go
index e55099800..f0b11bfdb 100644
--- a/userapi/storage/accounts/postgres/storage.go
+++ b/userapi/storage/accounts/postgres/storage.go
@@ -40,7 +40,6 @@ type Database struct {
 	memberships  membershipStatements
 	accountDatas accountDataStatements
 	threepids    threepidStatements
-	filter       filterStatements
 	serverName   gomatrixserverlib.ServerName
 }
 
@@ -75,11 +74,7 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serve
 	if err = t.prepare(db); err != nil {
 		return nil, err
 	}
-	f := filterStatements{}
-	if err = f.prepare(db); err != nil {
-		return nil, err
-	}
-	return &Database{db, partitions, a, p, m, ac, t, f, serverName}, nil
+	return &Database{db, partitions, a, p, m, ac, t, serverName}, nil
 }
 
 // GetAccountByPassword returns the account associated with the given localpart and password.
@@ -396,24 +391,6 @@ func (d *Database) GetThreePIDsForLocalpart(
 	return d.threepids.selectThreePIDsForLocalpart(ctx, localpart)
 }
 
-// GetFilter looks up the filter associated with a given local user and filter ID.
-// Returns a filter structure. Otherwise returns an error if no such filter exists
-// or if there was an error talking to the database.
-func (d *Database) GetFilter(
-	ctx context.Context, localpart string, filterID string,
-) (*gomatrixserverlib.Filter, error) {
-	return d.filter.selectFilter(ctx, localpart, filterID)
-}
-
-// PutFilter puts the passed filter into the database.
-// Returns the filterID as a string. Otherwise returns an error if something
-// goes wrong.
-func (d *Database) PutFilter(
-	ctx context.Context, localpart string, filter *gomatrixserverlib.Filter,
-) (string, error) {
-	return d.filter.insertFilter(ctx, filter, localpart)
-}
-
 // CheckAccountAvailability checks if the username/localpart is already present
 // in the database.
 // If the DB returns sql.ErrNoRows the Localpart isn't taken.
diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go
index d84f25b1f..e965df4f9 100644
--- a/userapi/storage/accounts/sqlite3/storage.go
+++ b/userapi/storage/accounts/sqlite3/storage.go
@@ -39,7 +39,6 @@ type Database struct {
 	memberships  membershipStatements
 	accountDatas accountDataStatements
 	threepids    threepidStatements
-	filter       filterStatements
 	serverName   gomatrixserverlib.ServerName
 
 	createAccountMu sync.Mutex
@@ -80,11 +79,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName)
 	if err = t.prepare(db); err != nil {
 		return nil, err
 	}
-	f := filterStatements{}
-	if err = f.prepare(db); err != nil {
-		return nil, err
-	}
-	return &Database{db, partitions, a, p, m, ac, t, f, serverName, sync.Mutex{}}, nil
+	return &Database{db, partitions, a, p, m, ac, t, serverName, sync.Mutex{}}, nil
 }
 
 // GetAccountByPassword returns the account associated with the given localpart and password.
@@ -410,24 +405,6 @@ func (d *Database) GetThreePIDsForLocalpart(
 	return d.threepids.selectThreePIDsForLocalpart(ctx, localpart)
 }
 
-// GetFilter looks up the filter associated with a given local user and filter ID.
-// Returns a filter structure. Otherwise returns an error if no such filter exists
-// or if there was an error talking to the database.
-func (d *Database) GetFilter(
-	ctx context.Context, localpart string, filterID string,
-) (*gomatrixserverlib.Filter, error) {
-	return d.filter.selectFilter(ctx, localpart, filterID)
-}
-
-// PutFilter puts the passed filter into the database.
-// Returns the filterID as a string. Otherwise returns an error if something
-// goes wrong.
-func (d *Database) PutFilter(
-	ctx context.Context, localpart string, filter *gomatrixserverlib.Filter,
-) (string, error) {
-	return d.filter.insertFilter(ctx, filter, localpart)
-}
-
 // CheckAccountAvailability checks if the username/localpart is already present
 // in the database.
 // If the DB returns sql.ErrNoRows the Localpart isn't taken.

From 4cf45d1ce99c98f41808fba41b00212c21099bdc Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 29 Jun 2020 14:39:21 +0100
Subject: [PATCH 13/23] Don't include current state in
 processEventWithMissingState (#1126)

* Don't include current state in processEventWithMissingState

* Remove lookupCurrentState as not needed

Co-authored-by: Kegsay <kegan@matrix.org>
---
 federationapi/routing/send.go | 27 +--------------------------
 1 file changed, 1 insertion(+), 26 deletions(-)

diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 53f951951..680eaccd3 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -404,8 +404,7 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer
 	// at this point we know we're going to have a gap: we need to work out the room state at the new backwards extremity.
 	// security: we have to do state resolution on the new backwards extremity (TODO: WHY)
 	// Therefore, we cannot just query /state_ids with this event to get the state before. Instead, we need to query
-	// the state AFTER all the prev_events for this event, then mix in our current room state and apply state resolution
-	// to that to get the state before the event.
+	// the state AFTER all the prev_events for this event, then apply state resolution to that to get the state before the event.
 	var states []*gomatrixserverlib.RespState
 	needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*backwardsExtremity}).Tuples()
 	for _, prevEventID := range backwardsExtremity.PrevEventIDs() {
@@ -417,13 +416,6 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer
 		}
 		states = append(states, prevState)
 	}
-	// mix in the current room state
-	currState, err := t.lookupCurrentState(backwardsExtremity)
-	if err != nil {
-		util.GetLogger(t.context).WithError(err).Errorf("Failed to lookup current room state")
-		return err
-	}
-	states = append(states, currState)
 	resolvedState, err := t.resolveStatesAndCheck(roomVersion, states, backwardsExtremity)
 	if err != nil {
 		util.GetLogger(t.context).WithError(err).Errorf("Failed to resolve state conflicts for event %s", backwardsExtremity.EventID())
@@ -526,23 +518,6 @@ func (t *txnReq) lookupStateAfterEventLocally(roomID, eventID string, needed []g
 	}
 }
 
-func (t *txnReq) lookupCurrentState(newEvent *gomatrixserverlib.Event) (*gomatrixserverlib.RespState, error) {
-	// Ask the roomserver for information about this room
-	queryReq := api.QueryLatestEventsAndStateRequest{
-		RoomID:       newEvent.RoomID(),
-		StateToFetch: gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*newEvent}).Tuples(),
-	}
-	var queryRes api.QueryLatestEventsAndStateResponse
-	if err := t.rsAPI.QueryLatestEventsAndState(t.context, &queryReq, &queryRes); err != nil {
-		return nil, fmt.Errorf("lookupCurrentState rsAPI.QueryLatestEventsAndState: %w", err)
-	}
-	evs := gomatrixserverlib.UnwrapEventHeaders(queryRes.StateEvents)
-	return &gomatrixserverlib.RespState{
-		StateEvents: evs,
-		AuthEvents:  evs,
-	}, nil
-}
-
 // lookuptStateBeforeEvent returns the room state before the event e, which is just /state_ids and/or /state depending on what
 // the server supports.
 func (t *txnReq) lookupStateBeforeEvent(roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (

From 3a18b7fc7814755868a9847eb99fb465c9317017 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 30 Jun 2020 10:26:26 +0100
Subject: [PATCH 14/23] Update awsy list

---
 are-we-synapse-yet.list | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list
index 3876de549..239b0ebf4 100644
--- a/are-we-synapse-yet.list
+++ b/are-we-synapse-yet.list
@@ -91,6 +91,7 @@ snd PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id
 get GET /rooms/:room_id/messages returns a message
 get GET /rooms/:room_id/messages lazy loads members correctly 
 typ PUT /rooms/:room_id/typing/:user_id sets typing notification 
+typ Typing notifications don't leak (3 subtests)
 rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels
 rst PUT /rooms/:room_id/state/m.room.power_levels can set levels
 rst PUT power_levels should not explode if the old power levels were empty
@@ -857,4 +858,4 @@ jso Invalid JSON special values
 inv Can invite users to invite-only rooms (2 subtests)
 plv setting 'm.room.name' respects room powerlevel (2 subtests)
 psh Messages that notify from another user increment notification_count
-psh Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count
\ No newline at end of file
+psh Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count

From ca5bbffd8d987b220c8f8eb888a2fc9b9cef104c Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Tue, 30 Jun 2020 10:37:21 +0100
Subject: [PATCH 15/23] Add a new component: currentstateserver (#1171)

* Add a new component: currentstateserver

- Add a skeleton for it, with databases and a single query method.
- Add integration tests for it.
- Add listen/address fields in the config (breaking as this will force people to specify this to validate)

Not currently hooked up to anything yet.

* Unbreak config tests

* Add current_state to sample config

* comments
---
 currentstateserver/api/api.go                 |  64 ++++++
 currentstateserver/consumers/roomserver.go    | 140 ++++++++++++
 currentstateserver/currentstateserver.go      |  51 +++++
 currentstateserver/currentstateserver_test.go | 180 +++++++++++++++
 currentstateserver/internal/api.go            |  41 ++++
 currentstateserver/inthttp/client.go          |  62 ++++++
 currentstateserver/inthttp/server.go          |  41 ++++
 currentstateserver/storage/interface.go       |  32 +++
 .../postgres/current_room_state_table.go      | 205 ++++++++++++++++++
 .../storage/postgres/storage.go               |  35 +++
 currentstateserver/storage/shared/storage.go  |  65 ++++++
 .../sqlite3/current_room_state_table.go       | 201 +++++++++++++++++
 currentstateserver/storage/sqlite3/storage.go |  39 ++++
 currentstateserver/storage/storage.go         |  41 ++++
 currentstateserver/storage/storage_wasm.go    |  42 ++++
 .../storage/tables/interface.go               |  31 +++
 dendrite-config.yaml                          |   2 +
 internal/config/config.go                     |  22 +-
 internal/config/config_test.go                |   2 +
 19 files changed, 1293 insertions(+), 3 deletions(-)
 create mode 100644 currentstateserver/api/api.go
 create mode 100644 currentstateserver/consumers/roomserver.go
 create mode 100644 currentstateserver/currentstateserver.go
 create mode 100644 currentstateserver/currentstateserver_test.go
 create mode 100644 currentstateserver/internal/api.go
 create mode 100644 currentstateserver/inthttp/client.go
 create mode 100644 currentstateserver/inthttp/server.go
 create mode 100644 currentstateserver/storage/interface.go
 create mode 100644 currentstateserver/storage/postgres/current_room_state_table.go
 create mode 100644 currentstateserver/storage/postgres/storage.go
 create mode 100644 currentstateserver/storage/shared/storage.go
 create mode 100644 currentstateserver/storage/sqlite3/current_room_state_table.go
 create mode 100644 currentstateserver/storage/sqlite3/storage.go
 create mode 100644 currentstateserver/storage/storage.go
 create mode 100644 currentstateserver/storage/storage_wasm.go
 create mode 100644 currentstateserver/storage/tables/interface.go

diff --git a/currentstateserver/api/api.go b/currentstateserver/api/api.go
new file mode 100644
index 000000000..10433722c
--- /dev/null
+++ b/currentstateserver/api/api.go
@@ -0,0 +1,64 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type CurrentStateInternalAPI interface {
+	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
+}
+
+type QueryCurrentStateRequest struct {
+	RoomID      string
+	StateTuples []gomatrixserverlib.StateKeyTuple
+}
+
+type QueryCurrentStateResponse struct {
+	StateEvents map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent
+}
+
+// MarshalJSON stringifies the StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
+func (r *QueryCurrentStateResponse) MarshalJSON() ([]byte, error) {
+	se := make(map[string]gomatrixserverlib.HeaderedEvent, len(r.StateEvents))
+	for k, v := range r.StateEvents {
+		// use 0x1F (unit separator) as the delimiter between type/state key,
+		se[fmt.Sprintf("%s\x1F%s", k.EventType, k.StateKey)] = v
+	}
+	return json.Marshal(se)
+}
+
+func (r *QueryCurrentStateResponse) UnmarshalJSON(data []byte) error {
+	res := make(map[string]gomatrixserverlib.HeaderedEvent)
+	err := json.Unmarshal(data, &res)
+	if err != nil {
+		return err
+	}
+	r.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent, len(res))
+	for k, v := range res {
+		fields := strings.Split(k, "\x1F")
+		r.StateEvents[gomatrixserverlib.StateKeyTuple{
+			EventType: fields[0],
+			StateKey:  fields[1],
+		}] = v
+	}
+	return nil
+}
diff --git a/currentstateserver/consumers/roomserver.go b/currentstateserver/consumers/roomserver.go
new file mode 100644
index 000000000..9e2694b0c
--- /dev/null
+++ b/currentstateserver/consumers/roomserver.go
@@ -0,0 +1,140 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/Shopify/sarama"
+	"github.com/matrix-org/dendrite/currentstateserver/storage"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	log "github.com/sirupsen/logrus"
+)
+
+type OutputRoomEventConsumer struct {
+	rsConsumer *internal.ContinualConsumer
+	db         storage.Database
+}
+
+func NewOutputRoomEventConsumer(topicName string, kafkaConsumer sarama.Consumer, store storage.Database) *OutputRoomEventConsumer {
+	consumer := &internal.ContinualConsumer{
+		Topic:          topicName,
+		Consumer:       kafkaConsumer,
+		PartitionStore: store,
+	}
+	s := &OutputRoomEventConsumer{
+		rsConsumer: consumer,
+		db:         store,
+	}
+	consumer.ProcessMessage = s.onMessage
+
+	return s
+}
+
+func (c *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
+	// Parse out the event JSON
+	var output api.OutputEvent
+	if err := json.Unmarshal(msg.Value, &output); err != nil {
+		// If the message was invalid, log it and move on to the next message in the stream
+		log.WithError(err).Errorf("roomserver output log: message parse failure")
+		return nil
+	}
+
+	switch output.Type {
+	case api.OutputTypeNewRoomEvent:
+		return c.onNewRoomEvent(context.TODO(), *output.NewRoomEvent)
+	case api.OutputTypeNewInviteEvent:
+	case api.OutputTypeRetireInviteEvent:
+	default:
+		log.WithField("type", output.Type).Debug(
+			"roomserver output log: ignoring unknown output type",
+		)
+	}
+	return nil
+}
+
+func (c *OutputRoomEventConsumer) onNewRoomEvent(
+	ctx context.Context, msg api.OutputNewRoomEvent,
+) error {
+	ev := msg.Event
+
+	addsStateEvents := msg.AddsState()
+
+	ev, err := c.updateStateEvent(ev)
+	if err != nil {
+		return err
+	}
+
+	for i := range addsStateEvents {
+		addsStateEvents[i], err = c.updateStateEvent(addsStateEvents[i])
+		if err != nil {
+			return err
+		}
+	}
+
+	err = c.db.StoreStateEvents(
+		ctx,
+		addsStateEvents,
+		msg.RemovesStateEventIDs,
+	)
+	if err != nil {
+		// panic rather than continue with an inconsistent database
+		log.WithFields(log.Fields{
+			"event":      string(ev.JSON()),
+			log.ErrorKey: err,
+			"add":        msg.AddsStateEventIDs,
+			"del":        msg.RemovesStateEventIDs,
+		}).Panicf("roomserver output log: write event failure")
+	}
+	return nil
+}
+
+// Start consuming from room servers
+func (c *OutputRoomEventConsumer) Start() error {
+	return c.rsConsumer.Start()
+}
+
+func (c *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.HeaderedEvent) (gomatrixserverlib.HeaderedEvent, error) {
+	var stateKey string
+	if event.StateKey() == nil {
+		stateKey = ""
+	} else {
+		stateKey = *event.StateKey()
+	}
+
+	prevEvent, err := c.db.GetStateEvent(
+		context.TODO(), event.RoomID(), event.Type(), stateKey,
+	)
+	if err != nil {
+		return event, err
+	}
+
+	if prevEvent == nil {
+		return event, nil
+	}
+
+	prev := types.PrevEventRef{
+		PrevContent:   prevEvent.Content(),
+		ReplacesState: prevEvent.EventID(),
+		PrevSender:    prevEvent.Sender(),
+	}
+
+	event.Event, err = event.SetUnsigned(prev)
+	return event, err
+}
diff --git a/currentstateserver/currentstateserver.go b/currentstateserver/currentstateserver.go
new file mode 100644
index 000000000..07d5e54ad
--- /dev/null
+++ b/currentstateserver/currentstateserver.go
@@ -0,0 +1,51 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package currentstateserver
+
+import (
+	"github.com/Shopify/sarama"
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/currentstateserver/api"
+	"github.com/matrix-org/dendrite/currentstateserver/consumers"
+	"github.com/matrix-org/dendrite/currentstateserver/internal"
+	"github.com/matrix-org/dendrite/currentstateserver/inthttp"
+	"github.com/matrix-org/dendrite/currentstateserver/storage"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/sirupsen/logrus"
+)
+
+// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
+// on the given input API.
+func AddInternalRoutes(router *mux.Router, intAPI api.CurrentStateInternalAPI) {
+	inthttp.AddRoutes(router, intAPI)
+}
+
+// NewInternalAPI returns a concrete implementation of the internal API. Callers
+// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
+func NewInternalAPI(cfg *config.Dendrite, consumer sarama.Consumer) api.CurrentStateInternalAPI {
+	csDB, err := storage.NewDatabase(string(cfg.Database.CurrentState), cfg.DbProperties())
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to open database")
+	}
+	roomConsumer := consumers.NewOutputRoomEventConsumer(
+		string(cfg.Kafka.Topics.OutputRoomEvent), consumer, csDB,
+	)
+	if err = roomConsumer.Start(); err != nil {
+		logrus.WithError(err).Panicf("failed to start room server consumer")
+	}
+	return &internal.CurrentStateInternalAPI{
+		DB: csDB,
+	}
+}
diff --git a/currentstateserver/currentstateserver_test.go b/currentstateserver/currentstateserver_test.go
new file mode 100644
index 000000000..95ca609b4
--- /dev/null
+++ b/currentstateserver/currentstateserver_test.go
@@ -0,0 +1,180 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package currentstateserver
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/currentstateserver/api"
+	"github.com/matrix-org/dendrite/currentstateserver/inthttp"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/test"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/naffka"
+)
+
+var (
+	testRoomVersion = gomatrixserverlib.RoomVersionV1
+	testData        = []json.RawMessage{
+		[]byte(`{"auth_events":[],"content":{"creator":"@userid:kaer.morhen"},"depth":0,"event_id":"$0ok8ynDp7kjc95e3:kaer.morhen","hashes":{"sha256":"17kPoH+h0Dk4Omn7Sus0qMb6+oGcf+CZFEgDhv7UKWs"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"jP4a04f5/F10Pw95FPpdCyKAO44JOwUQ/MZOOeA/RTU1Dn+AHPMzGSaZnuGjRr/xQuADt+I3ctb5ZQfLKNzHDw"}},"state_key":"","type":"m.room.create"}`),
+		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"content":{"membership":"join"},"depth":1,"event_id":"$LEwEu0kxrtu5fOiS:kaer.morhen","hashes":{"sha256":"B7M88PhXf3vd1LaFtjQutFu4x/w7fHD28XKZ4sAsJTo"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"p2vqmuJn7ZBRImctSaKbXCAxCcBlIjPH9JHte1ouIUGy84gpu4eLipOvSBCLL26hXfC0Zrm4WUto6Hr+ohdrCg"}},"state_key":"@userid:kaer.morhen","type":"m.room.member"}`),
+		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"join_rule":"public"},"depth":2,"event_id":"$SMHlqUrNhhBBRLeN:kaer.morhen","hashes":{"sha256":"vIuJQvmMjrGxshAkj1SXe0C4RqvMbv4ZADDw9pFCWqQ"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"hBMsb3Qppo3RaqqAl4JyTgaiWEbW5hlckATky6PrHun+F3YM203TzG7w9clwuQU5F5pZoB1a6nw+to0hN90FAw"}},"state_key":"","type":"m.room.join_rules"}`),
+		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"history_visibility":"shared"},"depth":3,"event_id":"$6F1yGIbO0J7TM93h:kaer.morhen","hashes":{"sha256":"Mr23GKSlZW7UCCYLgOWawI2Sg6KIoMjUWO2TDenuOgw"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$SMHlqUrNhhBBRLeN:kaer.morhen",{"sha256":"SylzE8U02I+6eyEHgL+FlU0L5YdqrVp8OOlxKS9VQW0"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sHLKrFI3hKGrEJfpMVZSDS3LvLasQsy50CTsOwru9XTVxgRsPo6wozNtRVjxo1J3Rk18RC9JppovmQ5VR5EcDw"}},"state_key":"","type":"m.room.history_visibility"}`),
+		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"ban":50,"events":null,"events_default":0,"invite":0,"kick":50,"redact":50,"state_default":50,"users":null,"users_default":0},"depth":4,"event_id":"$UKNe10XzYzG0TeA9:kaer.morhen","hashes":{"sha256":"ngbP3yja9U5dlckKerUs/fSOhtKxZMCVvsfhPURSS28"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$6F1yGIbO0J7TM93h:kaer.morhen",{"sha256":"A4CucrKSoWX4IaJXhq02mBg1sxIyZEftbC+5p3fZAvk"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"zOmwlP01QL3yFchzuR9WHvogOoBZA3oVtNIF3lM0ZfDnqlSYZB9sns27G/4HVq0k7alaK7ZE3oGoCrVnMkPNCw"}},"state_key":"","type":"m.room.power_levels"}`),
+		// messages
+		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":5,"event_id":"$gl2T9l3qm0kUbiIJ:kaer.morhen","hashes":{"sha256":"Qx3nRMHLDPSL5hBAzuX84FiSSP0K0Kju2iFoBWH4Za8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$UKNe10XzYzG0TeA9:kaer.morhen",{"sha256":"KtSRyMjt0ZSjsv2koixTRCxIRCGoOp6QrKscsW97XRo"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sqDgv3EG7ml5VREzmT9aZeBpS4gAPNIaIeJOwqjDhY0GPU/BcpX5wY4R7hYLrNe5cChgV+eFy/GWm1Zfg5FfDg"}},"type":"m.room.message"}`),
+		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":6,"event_id":"$MYSbs8m4rEbsCWXD:kaer.morhen","hashes":{"sha256":"kgbYM7v4Ud2YaBsjBTolM4ySg6rHcJNYI6nWhMSdFUA"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$gl2T9l3qm0kUbiIJ:kaer.morhen",{"sha256":"C/rD04h9wGxRdN2G/IBfrgoE1UovzLZ+uskwaKZ37/Q"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"x0UoKh968jj/F5l1/R7Ew0T6CTKuew3PLNHASNxqck/bkNe8yYQiDHXRr+kZxObeqPZZTpaF1+EI+bLU9W8GDQ"}},"type":"m.room.message"}`),
+		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":7,"event_id":"$N5x9WJkl9ClPrAEg:kaer.morhen","hashes":{"sha256":"FWM8oz4yquTunRZ67qlW2gzPDzdWfBP6RPHXhK1I/x8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$MYSbs8m4rEbsCWXD:kaer.morhen",{"sha256":"fatqgW+SE8mb2wFn3UN+drmluoD4UJ/EcSrL6Ur9q1M"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"Y+LX/xcyufoXMOIoqQBNOzy6lZfUGB1ffgXIrSugk6obMiyAsiRejHQN/pciZXsHKxMJLYRFAz4zSJoS/LGPAA"}},"type":"m.room.message"}`),
+	}
+	testEvents      = []gomatrixserverlib.HeaderedEvent{}
+	testStateEvents = make(map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent)
+
+	kafkaTopic = "room_events"
+)
+
+func init() {
+	for _, j := range testData {
+		e, err := gomatrixserverlib.NewEventFromTrustedJSON(j, false, testRoomVersion)
+		if err != nil {
+			panic("cannot load test data: " + err.Error())
+		}
+		h := e.Headered(testRoomVersion)
+		testEvents = append(testEvents, h)
+		if e.StateKey() != nil {
+			testStateEvents[gomatrixserverlib.StateKeyTuple{
+				EventType: e.Type(),
+				StateKey:  *e.StateKey(),
+			}] = h
+		}
+	}
+}
+
+func MustWriteOutputEvent(t *testing.T, producer sarama.SyncProducer, out *roomserverAPI.OutputNewRoomEvent) error {
+	value, err := json.Marshal(roomserverAPI.OutputEvent{
+		Type:         roomserverAPI.OutputTypeNewRoomEvent,
+		NewRoomEvent: out,
+	})
+	if err != nil {
+		t.Fatalf("failed to marshal output event: %s", err)
+	}
+	_, _, err = producer.SendMessage(&sarama.ProducerMessage{
+		Topic: kafkaTopic,
+		Key:   sarama.StringEncoder(out.Event.RoomID()),
+		Value: sarama.ByteEncoder(value),
+	})
+	if err != nil {
+		t.Fatalf("failed to send message: %s", err)
+	}
+	return nil
+}
+
+func MustMakeInternalAPI(t *testing.T) (api.CurrentStateInternalAPI, sarama.SyncProducer) {
+	cfg := &config.Dendrite{}
+	cfg.Kafka.Topics.OutputRoomEvent = config.Topic(kafkaTopic)
+	cfg.Database.CurrentState = config.DataSource("file::memory:")
+	db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), "file::memory:", nil)
+	if err != nil {
+		t.Fatalf("Failed to open naffka database: %s", err)
+	}
+	naffkaDB, err := naffka.NewSqliteDatabase(db)
+	if err != nil {
+		t.Fatalf("Failed to setup naffka database: %s", err)
+	}
+	naff, err := naffka.New(naffkaDB)
+	if err != nil {
+		t.Fatalf("Failed to create naffka consumer: %s", err)
+	}
+	return NewInternalAPI(cfg, naff), naff
+}
+
+func TestQueryCurrentState(t *testing.T) {
+	currStateAPI, producer := MustMakeInternalAPI(t)
+	plTuple := gomatrixserverlib.StateKeyTuple{
+		EventType: "m.room.power_levels",
+		StateKey:  "",
+	}
+	plEvent := testEvents[4]
+	MustWriteOutputEvent(t, producer, &roomserverAPI.OutputNewRoomEvent{
+		Event:             plEvent,
+		AddsStateEventIDs: []string{plEvent.EventID()},
+	})
+	// we have no good way to know /when/ the server has consumed the event
+	time.Sleep(100 * time.Millisecond)
+
+	testCases := []struct {
+		req     api.QueryCurrentStateRequest
+		wantRes api.QueryCurrentStateResponse
+		wantErr error
+	}{
+		{
+			req: api.QueryCurrentStateRequest{
+				RoomID: plEvent.RoomID(),
+				StateTuples: []gomatrixserverlib.StateKeyTuple{
+					plTuple,
+				},
+			},
+			wantRes: api.QueryCurrentStateResponse{
+				StateEvents: map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent{
+					plTuple: plEvent,
+				},
+			},
+		},
+	}
+
+	runCases := func(testAPI api.CurrentStateInternalAPI) {
+		for _, tc := range testCases {
+			var gotRes api.QueryCurrentStateResponse
+			gotErr := testAPI.QueryCurrentState(context.TODO(), &tc.req, &gotRes)
+			if tc.wantErr == nil && gotErr != nil || tc.wantErr != nil && gotErr == nil {
+				t.Errorf("QueryCurrentState error, got %s want %s", gotErr, tc.wantErr)
+				continue
+			}
+			for tuple, wantEvent := range tc.wantRes.StateEvents {
+				gotEvent, ok := gotRes.StateEvents[tuple]
+				if !ok {
+					t.Errorf("QueryCurrentState want tuple %+v but it is missing from the response", tuple)
+					continue
+				}
+				if !reflect.DeepEqual(gotEvent.JSON(), wantEvent.JSON()) {
+					t.Errorf("QueryCurrentState tuple %+v got event JSON %s want %s", tuple, string(gotEvent.JSON()), string(wantEvent.JSON()))
+				}
+			}
+		}
+	}
+	t.Run("HTTP API", func(t *testing.T) {
+		router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
+		AddInternalRoutes(router, currStateAPI)
+		apiURL, cancel := test.ListenAndServe(t, router, false)
+		defer cancel()
+		httpAPI, err := inthttp.NewCurrentStateAPIClient(apiURL, &http.Client{})
+		if err != nil {
+			t.Fatalf("failed to create HTTP client")
+		}
+		runCases(httpAPI)
+	})
+	t.Run("Monolith", func(t *testing.T) {
+		runCases(currStateAPI)
+	})
+}
diff --git a/currentstateserver/internal/api.go b/currentstateserver/internal/api.go
new file mode 100644
index 000000000..d83a7a0f9
--- /dev/null
+++ b/currentstateserver/internal/api.go
@@ -0,0 +1,41 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/currentstateserver/api"
+	"github.com/matrix-org/dendrite/currentstateserver/storage"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type CurrentStateInternalAPI struct {
+	DB storage.Database
+}
+
+func (a *CurrentStateInternalAPI) QueryCurrentState(ctx context.Context, req *api.QueryCurrentStateRequest, res *api.QueryCurrentStateResponse) error {
+	res.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent)
+	for _, tuple := range req.StateTuples {
+		ev, err := a.DB.GetStateEvent(ctx, req.RoomID, tuple.EventType, tuple.StateKey)
+		if err != nil {
+			return err
+		}
+		if ev != nil {
+			res.StateEvents[tuple] = *ev
+		}
+	}
+	return nil
+}
diff --git a/currentstateserver/inthttp/client.go b/currentstateserver/inthttp/client.go
new file mode 100644
index 000000000..2267685a1
--- /dev/null
+++ b/currentstateserver/inthttp/client.go
@@ -0,0 +1,62 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package inthttp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/currentstateserver/api"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/opentracing/opentracing-go"
+)
+
+// HTTP paths for the internal HTTP APIs
+const (
+	QueryCurrentStatePath = "/currentstateserver/queryCurrentState"
+)
+
+// NewCurrentStateAPIClient creates a CurrentStateInternalAPI implemented by talking to a HTTP POST API.
+// If httpClient is nil an error is returned
+func NewCurrentStateAPIClient(
+	apiURL string,
+	httpClient *http.Client,
+) (api.CurrentStateInternalAPI, error) {
+	if httpClient == nil {
+		return nil, errors.New("NewCurrentStateAPIClient: httpClient is <nil>")
+	}
+	return &httpCurrentStateInternalAPI{
+		apiURL:     apiURL,
+		httpClient: httpClient,
+	}, nil
+}
+
+type httpCurrentStateInternalAPI struct {
+	apiURL     string
+	httpClient *http.Client
+}
+
+func (h *httpCurrentStateInternalAPI) QueryCurrentState(
+	ctx context.Context,
+	request *api.QueryCurrentStateRequest,
+	response *api.QueryCurrentStateResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryCurrentState")
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryCurrentStatePath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
diff --git a/currentstateserver/inthttp/server.go b/currentstateserver/inthttp/server.go
new file mode 100644
index 000000000..83bac6ebb
--- /dev/null
+++ b/currentstateserver/inthttp/server.go
@@ -0,0 +1,41 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package inthttp
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/currentstateserver/api"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/util"
+)
+
+func AddRoutes(internalAPIMux *mux.Router, intAPI api.CurrentStateInternalAPI) {
+	internalAPIMux.Handle(QueryCurrentStatePath,
+		httputil.MakeInternalAPI("queryCurrentState", func(req *http.Request) util.JSONResponse {
+			request := api.QueryCurrentStateRequest{}
+			response := api.QueryCurrentStateResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := intAPI.QueryCurrentState(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+}
diff --git a/currentstateserver/storage/interface.go b/currentstateserver/storage/interface.go
new file mode 100644
index 000000000..488df9e2b
--- /dev/null
+++ b/currentstateserver/storage/interface.go
@@ -0,0 +1,32 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type Database interface {
+	internal.PartitionStorer
+	// StoreStateEvents updates the database with new events from the roomserver.
+	StoreStateEvents(ctx context.Context, addStateEvents []gomatrixserverlib.HeaderedEvent, removeStateEventIDs []string) error
+	// GetStateEvent returns the state event of a given type for a given room with a given state key
+	// If no event could be found, returns nil
+	// If there was an issue during the retrieval, returns an error
+	GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
+}
diff --git a/currentstateserver/storage/postgres/current_room_state_table.go b/currentstateserver/storage/postgres/current_room_state_table.go
new file mode 100644
index 000000000..255be42a6
--- /dev/null
+++ b/currentstateserver/storage/postgres/current_room_state_table.go
@@ -0,0 +1,205 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const currentRoomStateSchema = `
+-- Stores the current room state for every room.
+CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
+    -- The 'room_id' key for the state event.
+    room_id TEXT NOT NULL,
+    -- The state event ID
+    event_id TEXT NOT NULL,
+    -- The state event type e.g 'm.room.member'
+    type TEXT NOT NULL,
+    -- The 'sender' property of the event.
+    sender TEXT NOT NULL,
+    -- The state_key value for this state event e.g ''
+    state_key TEXT NOT NULL,
+    -- The JSON for the event. Stored as TEXT because this should be valid UTF-8.
+    headered_event_json TEXT NOT NULL,
+    -- The 'content.membership' value if this event is an m.room.member event. For other
+    -- events, this will be NULL.
+    membership TEXT,
+    -- Clobber based on 3-uple of room_id, type and state_key
+    CONSTRAINT currentstate_current_room_state_unique UNIQUE (room_id, type, state_key)
+);
+-- for event deletion
+CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender);
+-- for querying membership states of users
+CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave';
+`
+
+const upsertRoomStateSQL = "" +
+	"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, membership)" +
+	" VALUES ($1, $2, $3, $4, $5, $6, $7)" +
+	" ON CONFLICT ON CONSTRAINT currentstate_room_state_unique" +
+	" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, membership = $7"
+
+const deleteRoomStateByEventIDSQL = "" +
+	"DELETE FROM currentstate_current_room_state WHERE event_id = $1"
+
+const selectRoomIDsWithMembershipSQL = "" +
+	"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
+
+const selectStateEventSQL = "" +
+	"SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
+
+const selectEventsWithEventIDsSQL = "" +
+	"SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id = ANY($1)"
+
+type currentRoomStateStatements struct {
+	upsertRoomStateStmt             *sql.Stmt
+	deleteRoomStateByEventIDStmt    *sql.Stmt
+	selectRoomIDsWithMembershipStmt *sql.Stmt
+	selectEventsWithEventIDsStmt    *sql.Stmt
+	selectStateEventStmt            *sql.Stmt
+}
+
+func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
+	s := &currentRoomStateStatements{}
+	_, err := db.Exec(currentRoomStateSchema)
+	if err != nil {
+		return nil, err
+	}
+	if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil {
+		return nil, err
+	}
+	if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil {
+		return nil, err
+	}
+	if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil {
+		return nil, err
+	}
+	if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil {
+		return nil, err
+	}
+	if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
+func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
+	ctx context.Context,
+	txn *sql.Tx,
+	userID string,
+	membership string,
+) ([]string, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
+	rows, err := stmt.QueryContext(ctx, userID, membership)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed")
+
+	var result []string
+	for rows.Next() {
+		var roomID string
+		if err := rows.Scan(&roomID); err != nil {
+			return nil, err
+		}
+		result = append(result, roomID)
+	}
+	return result, rows.Err()
+}
+
+func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
+	ctx context.Context, txn *sql.Tx, eventID string,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt)
+	_, err := stmt.ExecContext(ctx, eventID)
+	return err
+}
+
+func (s *currentRoomStateStatements) UpsertRoomState(
+	ctx context.Context, txn *sql.Tx,
+	event gomatrixserverlib.HeaderedEvent, membership *string,
+) error {
+	headeredJSON, err := json.Marshal(event)
+	if err != nil {
+		return err
+	}
+
+	// upsert state event
+	stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt)
+	_, err = stmt.ExecContext(
+		ctx,
+		event.RoomID(),
+		event.EventID(),
+		event.Type(),
+		event.Sender(),
+		*event.StateKey(),
+		headeredJSON,
+		membership,
+	)
+	return err
+}
+
+func (s *currentRoomStateStatements) SelectEventsWithEventIDs(
+	ctx context.Context, txn *sql.Tx, eventIDs []string,
+) ([]gomatrixserverlib.HeaderedEvent, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectEventsWithEventIDsStmt)
+	rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs))
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed")
+	result := []gomatrixserverlib.HeaderedEvent{}
+	for rows.Next() {
+		var eventBytes []byte
+		if err := rows.Scan(&eventBytes); err != nil {
+			return nil, err
+		}
+		// TODO: Handle redacted events
+		var ev gomatrixserverlib.HeaderedEvent
+		if err := json.Unmarshal(eventBytes, &ev); err != nil {
+			return nil, err
+		}
+		result = append(result, ev)
+	}
+	return result, rows.Err()
+}
+
+func (s *currentRoomStateStatements) SelectStateEvent(
+	ctx context.Context, roomID, evType, stateKey string,
+) (*gomatrixserverlib.HeaderedEvent, error) {
+	stmt := s.selectStateEventStmt
+	var res []byte
+	err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var ev gomatrixserverlib.HeaderedEvent
+	if err = json.Unmarshal(res, &ev); err != nil {
+		return nil, err
+	}
+	return &ev, err
+}
diff --git a/currentstateserver/storage/postgres/storage.go b/currentstateserver/storage/postgres/storage.go
new file mode 100644
index 000000000..f8edb94e6
--- /dev/null
+++ b/currentstateserver/storage/postgres/storage.go
@@ -0,0 +1,35 @@
+package postgres
+
+import (
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/currentstateserver/storage/shared"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+type Database struct {
+	shared.Database
+	db *sql.DB
+	sqlutil.PartitionOffsetStatements
+}
+
+// NewDatabase creates a new sync server database
+func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
+	var d Database
+	var err error
+	if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil {
+		return nil, err
+	}
+	if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil {
+		return nil, err
+	}
+	currRoomState, err := NewPostgresCurrentRoomStateTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	d.Database = shared.Database{
+		DB:               d.db,
+		CurrentRoomState: currRoomState,
+	}
+	return &d, nil
+}
diff --git a/currentstateserver/storage/shared/storage.go b/currentstateserver/storage/shared/storage.go
new file mode 100644
index 000000000..976190cb8
--- /dev/null
+++ b/currentstateserver/storage/shared/storage.go
@@ -0,0 +1,65 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type Database struct {
+	DB               *sql.DB
+	CurrentRoomState tables.CurrentRoomState
+}
+
+func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) {
+	return d.CurrentRoomState.SelectStateEvent(ctx, roomID, evType, stateKey)
+}
+
+func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatrixserverlib.HeaderedEvent,
+	removeStateEventIDs []string) error {
+	return sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error {
+		// remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add.
+		for _, eventID := range removeStateEventIDs {
+			if err := d.CurrentRoomState.DeleteRoomStateByEventID(ctx, txn, eventID); err != nil {
+				return err
+			}
+		}
+
+		for _, event := range addStateEvents {
+			if event.StateKey() == nil {
+				// ignore non state events
+				continue
+			}
+			var membership *string
+			if event.Type() == "m.room.member" {
+				value, err := event.Membership()
+				if err != nil {
+					return err
+				}
+				membership = &value
+			}
+
+			if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
diff --git a/currentstateserver/storage/sqlite3/current_room_state_table.go b/currentstateserver/storage/sqlite3/current_room_state_table.go
new file mode 100644
index 000000000..c18193276
--- /dev/null
+++ b/currentstateserver/storage/sqlite3/current_room_state_table.go
@@ -0,0 +1,201 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"strings"
+
+	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const currentRoomStateSchema = `
+-- Stores the current room state for every room.
+CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    sender TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    headered_event_json TEXT NOT NULL,
+    membership TEXT,
+    UNIQUE (room_id, type, state_key)
+);
+-- for event deletion
+CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender);
+-- for querying membership states of users
+-- CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave';
+`
+
+const upsertRoomStateSQL = "" +
+	"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, membership)" +
+	" VALUES ($1, $2, $3, $4, $5, $6, $7)" +
+	" ON CONFLICT (event_id, room_id, type, sender)" +
+	" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, membership = $7"
+
+const deleteRoomStateByEventIDSQL = "" +
+	"DELETE FROM currentstate_current_room_state WHERE event_id = $1"
+
+const selectRoomIDsWithMembershipSQL = "" +
+	"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
+
+const selectStateEventSQL = "" +
+	"SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
+
+const selectEventsWithEventIDsSQL = "" +
+	// TODO: The session_id and transaction_id blanks are here because otherwise
+	// the rowsToStreamEvents expects there to be exactly five columns. We need to
+	// figure out if these really need to be in the DB, and if so, we need a
+	// better permanent fix for this. - neilalexander, 2 Jan 2020
+	"SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" +
+	" FROM currentstate_current_room_state WHERE event_id IN ($1)"
+
+type currentRoomStateStatements struct {
+	upsertRoomStateStmt             *sql.Stmt
+	deleteRoomStateByEventIDStmt    *sql.Stmt
+	selectRoomIDsWithMembershipStmt *sql.Stmt
+	selectStateEventStmt            *sql.Stmt
+}
+
+func NewSqliteCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
+	s := &currentRoomStateStatements{}
+	_, err := db.Exec(currentRoomStateSchema)
+	if err != nil {
+		return nil, err
+	}
+	if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil {
+		return nil, err
+	}
+	if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil {
+		return nil, err
+	}
+	if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil {
+		return nil, err
+	}
+	if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
+func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
+	ctx context.Context,
+	txn *sql.Tx,
+	userID string,
+	membership string, // nolint: unparam
+) ([]string, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
+	rows, err := stmt.QueryContext(ctx, userID, membership)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed")
+
+	var result []string
+	for rows.Next() {
+		var roomID string
+		if err := rows.Scan(&roomID); err != nil {
+			return nil, err
+		}
+		result = append(result, roomID)
+	}
+	return result, nil
+}
+
+func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
+	ctx context.Context, txn *sql.Tx, eventID string,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt)
+	_, err := stmt.ExecContext(ctx, eventID)
+	return err
+}
+
+func (s *currentRoomStateStatements) UpsertRoomState(
+	ctx context.Context, txn *sql.Tx,
+	event gomatrixserverlib.HeaderedEvent, membership *string,
+) error {
+	headeredJSON, err := json.Marshal(event)
+	if err != nil {
+		return err
+	}
+
+	// upsert state event
+	stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt)
+	_, err = stmt.ExecContext(
+		ctx,
+		event.RoomID(),
+		event.EventID(),
+		event.Type(),
+		event.Sender(),
+		*event.StateKey(),
+		headeredJSON,
+		membership,
+	)
+	return err
+}
+
+func (s *currentRoomStateStatements) SelectEventsWithEventIDs(
+	ctx context.Context, txn *sql.Tx, eventIDs []string,
+) ([]gomatrixserverlib.HeaderedEvent, error) {
+	iEventIDs := make([]interface{}, len(eventIDs))
+	for k, v := range eventIDs {
+		iEventIDs[k] = v
+	}
+	query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1)
+	rows, err := txn.QueryContext(ctx, query, iEventIDs...)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed")
+	result := []gomatrixserverlib.HeaderedEvent{}
+	for rows.Next() {
+		var eventBytes []byte
+		if err := rows.Scan(&eventBytes); err != nil {
+			return nil, err
+		}
+		// TODO: Handle redacted events
+		var ev gomatrixserverlib.HeaderedEvent
+		if err := json.Unmarshal(eventBytes, &ev); err != nil {
+			return nil, err
+		}
+		result = append(result, ev)
+	}
+	return result, nil
+}
+
+func (s *currentRoomStateStatements) SelectStateEvent(
+	ctx context.Context, roomID, evType, stateKey string,
+) (*gomatrixserverlib.HeaderedEvent, error) {
+	stmt := s.selectStateEventStmt
+	var res []byte
+	err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var ev gomatrixserverlib.HeaderedEvent
+	if err = json.Unmarshal(res, &ev); err != nil {
+		return nil, err
+	}
+	return &ev, err
+}
diff --git a/currentstateserver/storage/sqlite3/storage.go b/currentstateserver/storage/sqlite3/storage.go
new file mode 100644
index 000000000..6975e40ba
--- /dev/null
+++ b/currentstateserver/storage/sqlite3/storage.go
@@ -0,0 +1,39 @@
+package sqlite3
+
+import (
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/currentstateserver/storage/shared"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+type Database struct {
+	shared.Database
+	db *sql.DB
+	sqlutil.PartitionOffsetStatements
+}
+
+// NewDatabase creates a new sync server database
+// nolint: gocyclo
+func NewDatabase(dataSourceName string) (*Database, error) {
+	var d Database
+	cs, err := sqlutil.ParseFileURI(dataSourceName)
+	if err != nil {
+		return nil, err
+	}
+	if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
+		return nil, err
+	}
+	if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil {
+		return nil, err
+	}
+	currRoomState, err := NewSqliteCurrentRoomStateTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	d.Database = shared.Database{
+		DB:               d.db,
+		CurrentRoomState: currRoomState,
+	}
+	return &d, nil
+}
diff --git a/currentstateserver/storage/storage.go b/currentstateserver/storage/storage.go
new file mode 100644
index 000000000..ad04cf414
--- /dev/null
+++ b/currentstateserver/storage/storage.go
@@ -0,0 +1,41 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !wasm
+
+package storage
+
+import (
+	"net/url"
+
+	"github.com/matrix-org/dendrite/currentstateserver/storage/postgres"
+	"github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+// NewDatabase opens a database connection.
+func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
+	uri, err := url.Parse(dataSourceName)
+	if err != nil {
+		return postgres.NewDatabase(dataSourceName, dbProperties)
+	}
+	switch uri.Scheme {
+	case "postgres":
+		return postgres.NewDatabase(dataSourceName, dbProperties)
+	case "file":
+		return sqlite3.NewDatabase(dataSourceName)
+	default:
+		return postgres.NewDatabase(dataSourceName, dbProperties)
+	}
+}
diff --git a/currentstateserver/storage/storage_wasm.go b/currentstateserver/storage/storage_wasm.go
new file mode 100644
index 000000000..aa46c44df
--- /dev/null
+++ b/currentstateserver/storage/storage_wasm.go
@@ -0,0 +1,42 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+// NewDatabase opens a database connection.
+func NewDatabase(
+	dataSourceName string,
+	dbProperties sqlutil.DbProperties, // nolint:unparam
+) (Database, error) {
+	uri, err := url.Parse(dataSourceName)
+	if err != nil {
+		return nil, fmt.Errorf("Cannot use postgres implementation")
+	}
+	switch uri.Scheme {
+	case "postgres":
+		return nil, fmt.Errorf("Cannot use postgres implementation")
+	case "file":
+		return sqlite3.NewDatabase(dataSourceName)
+	default:
+		return nil, fmt.Errorf("Cannot use postgres implementation")
+	}
+}
diff --git a/currentstateserver/storage/tables/interface.go b/currentstateserver/storage/tables/interface.go
new file mode 100644
index 000000000..d2e560a21
--- /dev/null
+++ b/currentstateserver/storage/tables/interface.go
@@ -0,0 +1,31 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type CurrentRoomState interface {
+	SelectStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
+	SelectEventsWithEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error)
+	UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, membership *string) error
+	DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error
+	// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
+	SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error)
+}
diff --git a/dendrite-config.yaml b/dendrite-config.yaml
index 73bfec247..70c8f7958 100644
--- a/dendrite-config.yaml
+++ b/dendrite-config.yaml
@@ -121,6 +121,7 @@ database:
     federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable"
     appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable"
     public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable"
+    current_state: "postgres://dendrite:itsasecret@localhost/dendrite_currentstate?sslmode=disable"
     max_open_conns: 100
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -143,6 +144,7 @@ listen:
     key_server: "localhost:7779"
     server_key_api: "localhost:7780"
     user_api: "localhost:7781"
+    current_state_server: "localhost:7782"
 
 # The configuration for tracing the dendrite components.
 tracing:
diff --git a/internal/config/config.go b/internal/config/config.go
index baa82be23..8275fc478 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -160,10 +160,13 @@ type Dendrite struct {
 	// Postgres Config
 	Database struct {
 		// The Account database stores the login details and account information
-		// for local users. It is accessed by the ClientAPI.
+		// for local users. It is accessed by the UserAPI.
 		Account DataSource `yaml:"account"`
+		// The CurrentState database stores the current state of all rooms.
+		// It is accessed by the CurrentStateServer.
+		CurrentState DataSource `yaml:"current_state"`
 		// The Device database stores session information for the devices of logged
-		// in local users. It is accessed by the ClientAPI, the MediaAPI and the SyncAPI.
+		// in local users. It is accessed by the UserAPI.
 		Device DataSource `yaml:"device"`
 		// The MediaAPI database stores information about files uploaded and downloaded
 		// by local users. It is only accessed by the MediaAPI.
@@ -222,6 +225,7 @@ type Dendrite struct {
 	Bind struct {
 		MediaAPI         Address `yaml:"media_api"`
 		ClientAPI        Address `yaml:"client_api"`
+		CurrentState     Address `yaml:"current_state_server"`
 		FederationAPI    Address `yaml:"federation_api"`
 		ServerKeyAPI     Address `yaml:"server_key_api"`
 		AppServiceAPI    Address `yaml:"appservice_api"`
@@ -238,6 +242,7 @@ type Dendrite struct {
 	Listen struct {
 		MediaAPI         Address `yaml:"media_api"`
 		ClientAPI        Address `yaml:"client_api"`
+		CurrentState     Address `yaml:"current_state_server"`
 		FederationAPI    Address `yaml:"federation_api"`
 		ServerKeyAPI     Address `yaml:"server_key_api"`
 		AppServiceAPI    Address `yaml:"appservice_api"`
@@ -601,6 +606,7 @@ func (config *Dendrite) checkDatabase(configErrs *configErrors) {
 	checkNotEmpty(configErrs, "database.media_api", string(config.Database.MediaAPI))
 	checkNotEmpty(configErrs, "database.sync_api", string(config.Database.SyncAPI))
 	checkNotEmpty(configErrs, "database.room_server", string(config.Database.RoomServer))
+	checkNotEmpty(configErrs, "database.current_state", string(config.Database.CurrentState))
 }
 
 // checkListen verifies the parameters listen.* are valid.
@@ -613,6 +619,7 @@ func (config *Dendrite) checkListen(configErrs *configErrors) {
 	checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer))
 	checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer))
 	checkNotEmpty(configErrs, "listen.user_api", string(config.Listen.UserAPI))
+	checkNotEmpty(configErrs, "listen.current_state_server", string(config.Listen.CurrentState))
 }
 
 // checkLogging verifies the parameters logging.* are valid.
@@ -735,6 +742,15 @@ func (config *Dendrite) UserAPIURL() string {
 	return "http://" + string(config.Listen.UserAPI)
 }
 
+// CurrentStateAPIURL returns an HTTP URL for where the currentstateserver is listening.
+func (config *Dendrite) CurrentStateAPIURL() string {
+	// Hard code the currentstateserver to talk HTTP for now.
+	// If we support HTTPS we need to think of a practical way to do certificate validation.
+	// People setting up servers shouldn't need to get a certificate valid for the public
+	// internet for an internal API.
+	return "http://" + string(config.Listen.CurrentState)
+}
+
 // EDUServerURL returns an HTTP URL for where the EDU server is listening.
 func (config *Dendrite) EDUServerURL() string {
 	// Hard code the EDU server to talk HTTP for now.
@@ -753,7 +769,7 @@ func (config *Dendrite) FederationSenderURL() string {
 	return "http://" + string(config.Listen.FederationSender)
 }
 
-// FederationSenderURL returns an HTTP URL for where the federation sender is listening.
+// ServerKeyAPIURL returns an HTTP URL for where the federation sender is listening.
 func (config *Dendrite) ServerKeyAPIURL() string {
 	// Hard code the server key API server to talk HTTP for now.
 	// If we support HTTPS we need to think of a practical way to do certificate validation.
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 9a543e763..9b776a50f 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -55,6 +55,7 @@ database:
   sync_api: "postgresql:///syn_api"
   room_server: "postgresql:///room_server"
   appservice: "postgresql:///appservice"
+  current_state: "postgresql:///current_state"
 listen:
   room_server: "localhost:7770"
   client_api: "localhost:7771"
@@ -64,6 +65,7 @@ listen:
   appservice_api: "localhost:7777"
   edu_server: "localhost:7778"
   user_api: "localhost:7779"
+  current_state_server: "localhost:7775"
 logging:
   - type: "file"
     level: "info"

From 6f49758b90d655d9c2bb9170da2ea1d0a2bdd664 Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Tue, 30 Jun 2020 13:34:59 +0100
Subject: [PATCH 16/23] Remove membership table from account DB  (#1172)

* Remove membership table from account DB

And make code which needs that data use the currentstate server

* Unbreak tests; use a membership enum for space
---
 clientapi/clientapi.go                        |  14 +-
 clientapi/consumers/roomserver.go             |  92 ----------
 clientapi/routing/memberships.go              |  20 +--
 clientapi/routing/profile.go                  |  37 ++--
 clientapi/routing/routing.go                  |  10 +-
 clientapi/routing/sendtyping.go               |  36 ++--
 cmd/dendrite-client-api-server/main.go        |   5 +-
 cmd/dendrite-demo-libp2p/main.go              |   3 +
 cmd/dendrite-demo-yggdrasil/main.go           |   4 +
 cmd/dendrite-monolith-server/main.go          |   4 +
 cmd/dendritejs/main.go                        |   4 +
 currentstateserver/api/api.go                 |  22 ++-
 currentstateserver/currentstateserver_test.go |   4 +-
 currentstateserver/internal/api.go            |  13 +-
 currentstateserver/inthttp/client.go          |  13 ++
 currentstateserver/inthttp/server.go          |  13 ++
 currentstateserver/storage/interface.go       |   2 +
 .../postgres/current_room_state_table.go      |  23 +--
 currentstateserver/storage/shared/storage.go  |  21 ++-
 .../sqlite3/current_room_state_table.go       |  10 +-
 .../storage/tables/interface.go               |  17 +-
 internal/setup/base.go                        |  11 ++
 internal/setup/monolith.go                    |   6 +-
 userapi/storage/accounts/interface.go         |   5 -
 .../accounts/postgres/membership_table.go     | 159 ------------------
 userapi/storage/accounts/postgres/storage.go  | 113 +------------
 .../accounts/sqlite3/membership_table.go      | 159 ------------------
 userapi/storage/accounts/sqlite3/storage.go   | 113 +------------
 28 files changed, 211 insertions(+), 722 deletions(-)
 delete mode 100644 clientapi/consumers/roomserver.go
 delete mode 100644 userapi/storage/accounts/postgres/membership_table.go
 delete mode 100644 userapi/storage/accounts/sqlite3/membership_table.go

diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index 174eb1bf1..8ea84249a 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -18,9 +18,9 @@ import (
 	"github.com/Shopify/sarama"
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/consumers"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/routing"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
@@ -30,14 +30,12 @@ import (
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
 )
 
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
 func AddPublicRoutes(
 	router *mux.Router,
 	cfg *config.Dendrite,
-	consumer sarama.Consumer,
 	producer sarama.SyncProducer,
 	deviceDB devices.Database,
 	accountsDB accounts.Database,
@@ -45,6 +43,7 @@ func AddPublicRoutes(
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	eduInputAPI eduServerAPI.EDUServerInputAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI,
 	transactionsCache *transactions.Cache,
 	fsAPI federationSenderAPI.FederationSenderInternalAPI,
 	userAPI userapi.UserInternalAPI,
@@ -54,16 +53,9 @@ func AddPublicRoutes(
 		Topic:    string(cfg.Kafka.Topics.OutputClientData),
 	}
 
-	roomEventConsumer := consumers.NewOutputRoomEventConsumer(
-		cfg, consumer, accountsDB, rsAPI,
-	)
-	if err := roomEventConsumer.Start(); err != nil {
-		logrus.WithError(err).Panicf("failed to start room server consumer")
-	}
-
 	routing.Setup(
 		router, cfg, eduInputAPI, rsAPI, asAPI,
 		accountsDB, deviceDB, userAPI, federation,
-		syncProducer, transactionsCache, fsAPI,
+		syncProducer, transactionsCache, fsAPI, stateAPI,
 	)
 }
diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go
deleted file mode 100644
index beeda042b..000000000
--- a/clientapi/consumers/roomserver.go
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package consumers
-
-import (
-	"context"
-	"encoding/json"
-
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/Shopify/sarama"
-	log "github.com/sirupsen/logrus"
-)
-
-// OutputRoomEventConsumer consumes events that originated in the room server.
-type OutputRoomEventConsumer struct {
-	rsAPI      api.RoomserverInternalAPI
-	rsConsumer *internal.ContinualConsumer
-	db         accounts.Database
-	serverName string
-}
-
-// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
-func NewOutputRoomEventConsumer(
-	cfg *config.Dendrite,
-	kafkaConsumer sarama.Consumer,
-	store accounts.Database,
-	rsAPI api.RoomserverInternalAPI,
-) *OutputRoomEventConsumer {
-
-	consumer := internal.ContinualConsumer{
-		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
-		Consumer:       kafkaConsumer,
-		PartitionStore: store,
-	}
-	s := &OutputRoomEventConsumer{
-		rsConsumer: &consumer,
-		db:         store,
-		rsAPI:      rsAPI,
-		serverName: string(cfg.Matrix.ServerName),
-	}
-	consumer.ProcessMessage = s.onMessage
-
-	return s
-}
-
-// Start consuming from room servers
-func (s *OutputRoomEventConsumer) Start() error {
-	return s.rsConsumer.Start()
-}
-
-// onMessage is called when the sync server receives a new event from the room server output log.
-// It is not safe for this function to be called from multiple goroutines, or else the
-// sync stream position may race and be incorrectly calculated.
-func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
-	// Parse out the event JSON
-	var output api.OutputEvent
-	if err := json.Unmarshal(msg.Value, &output); err != nil {
-		// If the message was invalid, log it and move on to the next message in the stream
-		log.WithError(err).Errorf("roomserver output log: message parse failure")
-		return nil
-	}
-
-	if output.Type != api.OutputTypeNewRoomEvent {
-		log.WithField("type", output.Type).Debug(
-			"roomserver output log: ignoring unknown output type",
-		)
-		return nil
-	}
-
-	return s.db.UpdateMemberships(
-		context.TODO(),
-		gomatrixserverlib.UnwrapEventHeaders(output.NewRoomEvent.AddsState()),
-		output.NewRoomEvent.RemovesStateEventIDs,
-	)
-}
diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go
index 1c9800b66..9c4cf7497 100644
--- a/clientapi/routing/memberships.go
+++ b/clientapi/routing/memberships.go
@@ -18,9 +18,8 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
-
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -95,20 +94,19 @@ func GetMemberships(
 func GetJoinedRooms(
 	req *http.Request,
 	device *userapi.Device,
-	accountsDB accounts.Database,
+	stateAPI currentstateAPI.CurrentStateInternalAPI,
 ) util.JSONResponse {
-	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+	var res currentstateAPI.QueryRoomsForUserResponse
+	err := stateAPI.QueryRoomsForUser(req.Context(), &currentstateAPI.QueryRoomsForUserRequest{
+		UserID:         device.UserID,
+		WantMembership: "join",
+	}, &res)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
-	}
-	joinedRooms, err := accountsDB.GetRoomIDsByLocalPart(req.Context(), localpart)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountsDB.GetRoomIDsByLocalPart failed")
+		util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
 		return jsonerror.InternalServerError()
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: getJoinedRoomsResponse{joinedRooms},
+		JSON: getJoinedRoomsResponse{res.RoomIDs},
 	}
 }
diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go
index 7c2cd19bc..1df4c9b33 100644
--- a/clientapi/routing/profile.go
+++ b/clientapi/routing/profile.go
@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -93,8 +94,8 @@ func GetAvatarURL(
 // SetAvatarURL implements PUT /profile/{userID}/avatar_url
 // nolint:gocyclo
 func SetAvatarURL(
-	req *http.Request, accountDB accounts.Database, device *userapi.Device,
-	userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
+	req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI,
+	device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -139,9 +140,13 @@ func SetAvatarURL(
 		return jsonerror.InternalServerError()
 	}
 
-	memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart)
+	var res currentstateAPI.QueryRoomsForUserResponse
+	err = stateAPI.QueryRoomsForUser(req.Context(), &currentstateAPI.QueryRoomsForUserRequest{
+		UserID:         device.UserID,
+		WantMembership: "join",
+	}, &res)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipsByLocalpart failed")
+		util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -152,7 +157,7 @@ func SetAvatarURL(
 	}
 
 	events, err := buildMembershipEvents(
-		req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
+		req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI,
 	)
 	switch e := err.(type) {
 	case nil:
@@ -207,8 +212,8 @@ func GetDisplayName(
 // SetDisplayName implements PUT /profile/{userID}/displayname
 // nolint:gocyclo
 func SetDisplayName(
-	req *http.Request, accountDB accounts.Database, device *userapi.Device,
-	userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
+	req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI,
+	device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -253,9 +258,13 @@ func SetDisplayName(
 		return jsonerror.InternalServerError()
 	}
 
-	memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart)
+	var res currentstateAPI.QueryRoomsForUserResponse
+	err = stateAPI.QueryRoomsForUser(req.Context(), &currentstateAPI.QueryRoomsForUserRequest{
+		UserID:         device.UserID,
+		WantMembership: "join",
+	}, &res)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipsByLocalpart failed")
+		util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -266,7 +275,7 @@ func SetDisplayName(
 	}
 
 	events, err := buildMembershipEvents(
-		req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
+		req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI,
 	)
 	switch e := err.(type) {
 	case nil:
@@ -335,14 +344,14 @@ func getProfile(
 
 func buildMembershipEvents(
 	ctx context.Context,
-	memberships []authtypes.Membership,
+	roomIDs []string,
 	newProfile authtypes.Profile, userID string, cfg *config.Dendrite,
 	evTime time.Time, rsAPI api.RoomserverInternalAPI,
 ) ([]gomatrixserverlib.HeaderedEvent, error) {
 	evs := []gomatrixserverlib.HeaderedEvent{}
 
-	for _, membership := range memberships {
-		verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID}
+	for _, roomID := range roomIDs {
+		verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
 		if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
 			return []gomatrixserverlib.HeaderedEvent{}, err
@@ -350,7 +359,7 @@ func buildMembershipEvents(
 
 		builder := gomatrixserverlib.EventBuilder{
 			Sender:   userID,
-			RoomID:   membership.RoomID,
+			RoomID:   roomID,
 			Type:     "m.room.member",
 			StateKey: &userID,
 		}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 9dfff0f20..deaa7b329 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -23,6 +23,7 @@ import (
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
@@ -58,6 +59,7 @@ func Setup(
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
 	federationSender federationSenderAPI.FederationSenderInternalAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI,
 ) {
 
 	publicAPIMux.Handle("/client/versions",
@@ -98,7 +100,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/joined_rooms",
 		httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			return GetJoinedRooms(req, device, accountDB)
+			return GetJoinedRooms(req, device, stateAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/join",
@@ -307,7 +309,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI)
+			return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI, stateAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
@@ -404,7 +406,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetAvatarURL(req, accountDB, device, vars["userID"], cfg, rsAPI)
+			return SetAvatarURL(req, accountDB, stateAPI, device, vars["userID"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
@@ -426,7 +428,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetDisplayName(req, accountDB, device, vars["userID"], cfg, rsAPI)
+			return SetDisplayName(req, accountDB, stateAPI, device, vars["userID"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go
index 9b6a0b39b..54a822860 100644
--- a/clientapi/routing/sendtyping.go
+++ b/clientapi/routing/sendtyping.go
@@ -13,15 +13,15 @@
 package routing
 
 import (
-	"database/sql"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/userutil"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
@@ -36,6 +36,7 @@ func SendTyping(
 	req *http.Request, device *userapi.Device, roomID string,
 	userID string, accountDB accounts.Database,
 	eduAPI api.EDUServerInputAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI,
 ) util.JSONResponse {
 	if device.UserID != userID {
 		return util.JSONResponse{
@@ -44,23 +45,38 @@ func SendTyping(
 		}
 	}
 
-	localpart, err := userutil.ParseUsernameParam(userID, nil)
+	// Verify that the user is a member of this room
+	tuple := gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomMember,
+		StateKey:  userID,
+	}
+	var res currentstateAPI.QueryCurrentStateResponse
+	err := stateAPI.QueryCurrentState(req.Context(), &currentstateAPI.QueryCurrentStateRequest{
+		RoomID:      roomID,
+		StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
+	}, &res)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("userutil.ParseUsernameParam failed")
+		util.GetLogger(req.Context()).WithError(err).Error("QueryCurrentState failed")
 		return jsonerror.InternalServerError()
 	}
-
-	// Verify that the user is a member of this room
-	_, err = accountDB.GetMembershipInRoomByLocalpart(req.Context(), localpart, roomID)
-	if err == sql.ErrNoRows {
+	ev := res.StateEvents[tuple]
+	if ev == nil {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("User not in this room"),
 		}
-	} else if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipInRoomByLocalPart failed")
+	}
+	membership, err := ev.Membership()
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("Member event isn't valid")
 		return jsonerror.InternalServerError()
 	}
+	if membership != gomatrixserverlib.Join {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("User not in this room"),
+		}
+	}
 
 	// parse the incoming http request
 	var r typingContentJSON
diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go
index fe5f30a0e..f46dae502 100644
--- a/cmd/dendrite-client-api-server/main.go
+++ b/cmd/dendrite-client-api-server/main.go
@@ -35,10 +35,11 @@ func main() {
 	fsAPI := base.FederationSenderHTTPClient()
 	eduInputAPI := base.EDUServerClient()
 	userAPI := base.UserAPIClient()
+	stateAPI := base.CurrentStateAPIClient()
 
 	clientapi.AddPublicRoutes(
-		base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation,
-		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI,
+		base.PublicAPIMux, base.Cfg, base.KafkaProducer, deviceDB, accountDB, federation,
+		rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))
diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go
index 356ab5a7f..b7e86b77c 100644
--- a/cmd/dendrite-demo-libp2p/main.go
+++ b/cmd/dendrite-demo-libp2p/main.go
@@ -30,6 +30,7 @@ import (
 	p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery"
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage"
+	"github.com/matrix-org/dendrite/currentstateserver"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/internal/config"
@@ -166,6 +167,7 @@ func main() {
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
+	stateAPI := currentstateserver.NewInternalAPI(base.Base.Cfg, base.Base.KafkaConsumer)
 
 	monolith := setup.Monolith{
 		Config:        base.Base.Cfg,
@@ -182,6 +184,7 @@ func main() {
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
 		ServerKeyAPI:        serverKeyAPI,
+		StateAPI:            stateAPI,
 		UserAPI:             userAPI,
 
 		PublicRoomsDB: publicRoomsDB,
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index db05ecb76..5de674021 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -27,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
+	"github.com/matrix-org/dendrite/currentstateserver"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
@@ -115,6 +116,8 @@ func main() {
 
 	embed.Embed(base.BaseMux, *instancePort, "Yggdrasil Demo")
 
+	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+
 	monolith := setup.Monolith{
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
@@ -130,6 +133,7 @@ func main() {
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
 		UserAPI:             userAPI,
+		StateAPI:            stateAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 
 		PublicRoomsDB: publicRoomsDB,
diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go
index 339bbe699..905eda2ba 100644
--- a/cmd/dendrite-monolith-server/main.go
+++ b/cmd/dendrite-monolith-server/main.go
@@ -20,6 +20,7 @@ import (
 	"os"
 
 	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/currentstateserver"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
@@ -122,6 +123,8 @@ func main() {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
 
+	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+
 	monolith := setup.Monolith{
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
@@ -137,6 +140,7 @@ func main() {
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
 		ServerKeyAPI:        serverKeyAPI,
+		StateAPI:            stateAPI,
 		UserAPI:             userAPI,
 
 		PublicRoomsDB: publicRoomsDB,
diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go
index 883b0fad0..11f339b0f 100644
--- a/cmd/dendritejs/main.go
+++ b/cmd/dendritejs/main.go
@@ -22,6 +22,7 @@ import (
 	"syscall/js"
 
 	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/currentstateserver"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
@@ -218,6 +219,8 @@ func main() {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
 
+	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+
 	monolith := setup.Monolith{
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
@@ -232,6 +235,7 @@ func main() {
 		EDUInternalAPI:      eduInputAPI,
 		FederationSenderAPI: fedSenderAPI,
 		RoomserverAPI:       rsAPI,
+		StateAPI:            stateAPI,
 		UserAPI:             userAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 
diff --git a/currentstateserver/api/api.go b/currentstateserver/api/api.go
index 10433722c..b16306ab0 100644
--- a/currentstateserver/api/api.go
+++ b/currentstateserver/api/api.go
@@ -24,7 +24,21 @@ import (
 )
 
 type CurrentStateInternalAPI interface {
+	// QueryCurrentState retrieves the requested state events. If state events are not found, they will be missing from
+	// the response.
 	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
+	// QueryRoomsForUser retrieves a list of room IDs matching the given query.
+	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
+}
+
+type QueryRoomsForUserRequest struct {
+	UserID string
+	// The desired membership of the user. If this is the empty string then no rooms are returned.
+	WantMembership string
+}
+
+type QueryRoomsForUserResponse struct {
+	RoomIDs []string
 }
 
 type QueryCurrentStateRequest struct {
@@ -33,12 +47,12 @@ type QueryCurrentStateRequest struct {
 }
 
 type QueryCurrentStateResponse struct {
-	StateEvents map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent
+	StateEvents map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent
 }
 
 // MarshalJSON stringifies the StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
 func (r *QueryCurrentStateResponse) MarshalJSON() ([]byte, error) {
-	se := make(map[string]gomatrixserverlib.HeaderedEvent, len(r.StateEvents))
+	se := make(map[string]*gomatrixserverlib.HeaderedEvent, len(r.StateEvents))
 	for k, v := range r.StateEvents {
 		// use 0x1F (unit separator) as the delimiter between type/state key,
 		se[fmt.Sprintf("%s\x1F%s", k.EventType, k.StateKey)] = v
@@ -47,12 +61,12 @@ func (r *QueryCurrentStateResponse) MarshalJSON() ([]byte, error) {
 }
 
 func (r *QueryCurrentStateResponse) UnmarshalJSON(data []byte) error {
-	res := make(map[string]gomatrixserverlib.HeaderedEvent)
+	res := make(map[string]*gomatrixserverlib.HeaderedEvent)
 	err := json.Unmarshal(data, &res)
 	if err != nil {
 		return err
 	}
-	r.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent, len(res))
+	r.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent, len(res))
 	for k, v := range res {
 		fields := strings.Split(k, "\x1F")
 		r.StateEvents[gomatrixserverlib.StateKeyTuple{
diff --git a/currentstateserver/currentstateserver_test.go b/currentstateserver/currentstateserver_test.go
index 95ca609b4..a0627fea7 100644
--- a/currentstateserver/currentstateserver_test.go
+++ b/currentstateserver/currentstateserver_test.go
@@ -136,8 +136,8 @@ func TestQueryCurrentState(t *testing.T) {
 				},
 			},
 			wantRes: api.QueryCurrentStateResponse{
-				StateEvents: map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent{
-					plTuple: plEvent,
+				StateEvents: map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent{
+					plTuple: &plEvent,
 				},
 			},
 		},
diff --git a/currentstateserver/internal/api.go b/currentstateserver/internal/api.go
index d83a7a0f9..85fbf51ef 100644
--- a/currentstateserver/internal/api.go
+++ b/currentstateserver/internal/api.go
@@ -27,15 +27,24 @@ type CurrentStateInternalAPI struct {
 }
 
 func (a *CurrentStateInternalAPI) QueryCurrentState(ctx context.Context, req *api.QueryCurrentStateRequest, res *api.QueryCurrentStateResponse) error {
-	res.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent)
+	res.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent)
 	for _, tuple := range req.StateTuples {
 		ev, err := a.DB.GetStateEvent(ctx, req.RoomID, tuple.EventType, tuple.StateKey)
 		if err != nil {
 			return err
 		}
 		if ev != nil {
-			res.StateEvents[tuple] = *ev
+			res.StateEvents[tuple] = ev
 		}
 	}
 	return nil
 }
+
+func (a *CurrentStateInternalAPI) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error {
+	roomIDs, err := a.DB.GetRoomsByMembership(ctx, req.UserID, req.WantMembership)
+	if err != nil {
+		return err
+	}
+	res.RoomIDs = roomIDs
+	return nil
+}
diff --git a/currentstateserver/inthttp/client.go b/currentstateserver/inthttp/client.go
index 2267685a1..6fd9907bd 100644
--- a/currentstateserver/inthttp/client.go
+++ b/currentstateserver/inthttp/client.go
@@ -27,6 +27,7 @@ import (
 // HTTP paths for the internal HTTP APIs
 const (
 	QueryCurrentStatePath = "/currentstateserver/queryCurrentState"
+	QueryRoomsForUserPath = "/currentstateserver/queryRoomsForUser"
 )
 
 // NewCurrentStateAPIClient creates a CurrentStateInternalAPI implemented by talking to a HTTP POST API.
@@ -60,3 +61,15 @@ func (h *httpCurrentStateInternalAPI) QueryCurrentState(
 	apiURL := h.apiURL + QueryCurrentStatePath
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
+
+func (h *httpCurrentStateInternalAPI) QueryRoomsForUser(
+	ctx context.Context,
+	request *api.QueryRoomsForUserRequest,
+	response *api.QueryRoomsForUserResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomsForUser")
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryRoomsForUserPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
diff --git a/currentstateserver/inthttp/server.go b/currentstateserver/inthttp/server.go
index 83bac6ebb..fa7ecb22e 100644
--- a/currentstateserver/inthttp/server.go
+++ b/currentstateserver/inthttp/server.go
@@ -38,4 +38,17 @@ func AddRoutes(internalAPIMux *mux.Router, intAPI api.CurrentStateInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(QueryRoomsForUserPath,
+		httputil.MakeInternalAPI("queryRoomsForUser", func(req *http.Request) util.JSONResponse {
+			request := api.QueryRoomsForUserRequest{}
+			response := api.QueryRoomsForUserResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := intAPI.QueryRoomsForUser(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }
diff --git a/currentstateserver/storage/interface.go b/currentstateserver/storage/interface.go
index 488df9e2b..dbf223f33 100644
--- a/currentstateserver/storage/interface.go
+++ b/currentstateserver/storage/interface.go
@@ -29,4 +29,6 @@ type Database interface {
 	// If no event could be found, returns nil
 	// If there was an issue during the retrieval, returns an error
 	GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
+	// GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key).
+	GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error)
 }
diff --git a/currentstateserver/storage/postgres/current_room_state_table.go b/currentstateserver/storage/postgres/current_room_state_table.go
index 255be42a6..95621913b 100644
--- a/currentstateserver/storage/postgres/current_room_state_table.go
+++ b/currentstateserver/storage/postgres/current_room_state_table.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
+	"strconv"
 
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
@@ -26,7 +27,9 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-const currentRoomStateSchema = `
+var leaveEnum = strconv.Itoa(tables.MembershipToEnum["leave"])
+
+var currentRoomStateSchema = `
 -- Stores the current room state for every room.
 CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
     -- The 'room_id' key for the state event.
@@ -41,22 +44,22 @@ CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
     state_key TEXT NOT NULL,
     -- The JSON for the event. Stored as TEXT because this should be valid UTF-8.
     headered_event_json TEXT NOT NULL,
-    -- The 'content.membership' value if this event is an m.room.member event. For other
-    -- events, this will be NULL.
-    membership TEXT,
+    -- The 'content.membership' enum value if this event is an m.room.member event.
+    membership SMALLINT NOT NULL DEFAULT 0,
     -- Clobber based on 3-uple of room_id, type and state_key
     CONSTRAINT currentstate_current_room_state_unique UNIQUE (room_id, type, state_key)
 );
 -- for event deletion
 CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender);
 -- for querying membership states of users
-CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave';
+CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership)
+WHERE membership IS NOT NULL AND membership != ` + leaveEnum + `;
 `
 
 const upsertRoomStateSQL = "" +
 	"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, membership)" +
 	" VALUES ($1, $2, $3, $4, $5, $6, $7)" +
-	" ON CONFLICT ON CONSTRAINT currentstate_room_state_unique" +
+	" ON CONFLICT ON CONSTRAINT currentstate_current_room_state_unique" +
 	" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, membership = $7"
 
 const deleteRoomStateByEventIDSQL = "" +
@@ -108,10 +111,10 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
 	ctx context.Context,
 	txn *sql.Tx,
 	userID string,
-	membership string,
+	membershipEnum int,
 ) ([]string, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
-	rows, err := stmt.QueryContext(ctx, userID, membership)
+	rows, err := stmt.QueryContext(ctx, userID, membershipEnum)
 	if err != nil {
 		return nil, err
 	}
@@ -138,7 +141,7 @@ func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
 
 func (s *currentRoomStateStatements) UpsertRoomState(
 	ctx context.Context, txn *sql.Tx,
-	event gomatrixserverlib.HeaderedEvent, membership *string,
+	event gomatrixserverlib.HeaderedEvent, membershipEnum int,
 ) error {
 	headeredJSON, err := json.Marshal(event)
 	if err != nil {
@@ -155,7 +158,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
 		event.Sender(),
 		*event.StateKey(),
 		headeredJSON,
-		membership,
+		membershipEnum,
 	)
 	return err
 }
diff --git a/currentstateserver/storage/shared/storage.go b/currentstateserver/storage/shared/storage.go
index 976190cb8..d78b3e0ed 100644
--- a/currentstateserver/storage/shared/storage.go
+++ b/currentstateserver/storage/shared/storage.go
@@ -17,6 +17,7 @@ package shared
 import (
 	"context"
 	"database/sql"
+	"fmt"
 
 	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -47,19 +48,31 @@ func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatr
 				// ignore non state events
 				continue
 			}
-			var membership *string
+			var membershipEnum int
 			if event.Type() == "m.room.member" {
-				value, err := event.Membership()
+				membership, err := event.Membership()
 				if err != nil {
 					return err
 				}
-				membership = &value
+				enum, ok := tables.MembershipToEnum[membership]
+				if !ok {
+					return fmt.Errorf("unknown membership: %s", membership)
+				}
+				membershipEnum = enum
 			}
 
-			if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership); err != nil {
+			if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membershipEnum); err != nil {
 				return err
 			}
 		}
 		return nil
 	})
 }
+
+func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) {
+	enum, ok := tables.MembershipToEnum[membership]
+	if !ok {
+		return nil, fmt.Errorf("unknown membership: %s", membership)
+	}
+	return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, enum)
+}
diff --git a/currentstateserver/storage/sqlite3/current_room_state_table.go b/currentstateserver/storage/sqlite3/current_room_state_table.go
index c18193276..2e2b0e423 100644
--- a/currentstateserver/storage/sqlite3/current_room_state_table.go
+++ b/currentstateserver/storage/sqlite3/current_room_state_table.go
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
     sender TEXT NOT NULL,
     state_key TEXT NOT NULL,
     headered_event_json TEXT NOT NULL,
-    membership TEXT,
+    membership INTEGER NOT NULL DEFAULT 0,
     UNIQUE (room_id, type, state_key)
 );
 -- for event deletion
@@ -100,10 +100,10 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
 	ctx context.Context,
 	txn *sql.Tx,
 	userID string,
-	membership string, // nolint: unparam
+	membershipEnum int,
 ) ([]string, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
-	rows, err := stmt.QueryContext(ctx, userID, membership)
+	rows, err := stmt.QueryContext(ctx, userID, membershipEnum)
 	if err != nil {
 		return nil, err
 	}
@@ -130,7 +130,7 @@ func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
 
 func (s *currentRoomStateStatements) UpsertRoomState(
 	ctx context.Context, txn *sql.Tx,
-	event gomatrixserverlib.HeaderedEvent, membership *string,
+	event gomatrixserverlib.HeaderedEvent, membershipEnum int,
 ) error {
 	headeredJSON, err := json.Marshal(event)
 	if err != nil {
@@ -147,7 +147,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
 		event.Sender(),
 		*event.StateKey(),
 		headeredJSON,
-		membership,
+		membershipEnum,
 	)
 	return err
 }
diff --git a/currentstateserver/storage/tables/interface.go b/currentstateserver/storage/tables/interface.go
index d2e560a21..f2c8b14ed 100644
--- a/currentstateserver/storage/tables/interface.go
+++ b/currentstateserver/storage/tables/interface.go
@@ -21,11 +21,24 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
+var MembershipToEnum = map[string]int{
+	gomatrixserverlib.Invite: 1,
+	gomatrixserverlib.Join:   2,
+	gomatrixserverlib.Leave:  3,
+	gomatrixserverlib.Ban:    4,
+}
+var EnumToMembership = map[int]string{
+	1: gomatrixserverlib.Invite,
+	2: gomatrixserverlib.Join,
+	3: gomatrixserverlib.Leave,
+	4: gomatrixserverlib.Ban,
+}
+
 type CurrentRoomState interface {
 	SelectStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
 	SelectEventsWithEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error)
-	UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, membership *string) error
+	UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, membershipEnum int) error
 	DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error
 	// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
-	SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error)
+	SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membershipEnum int) ([]string, error)
 }
diff --git a/internal/setup/base.go b/internal/setup/base.go
index 66424a609..ddf8e0fad 100644
--- a/internal/setup/base.go
+++ b/internal/setup/base.go
@@ -22,6 +22,7 @@ import (
 	"net/url"
 	"time"
 
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -37,6 +38,7 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	asinthttp "github.com/matrix-org/dendrite/appservice/inthttp"
+	currentstateinthttp "github.com/matrix-org/dendrite/currentstateserver/inthttp"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	eduinthttp "github.com/matrix-org/dendrite/eduserver/inthttp"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
@@ -171,6 +173,15 @@ func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI {
 	return userAPI
 }
 
+// CurrentStateAPIClient returns CurrentStateInternalAPI for hitting the currentstateserver over HTTP.
+func (b *BaseDendrite) CurrentStateAPIClient() currentstateAPI.CurrentStateInternalAPI {
+	stateAPI, err := currentstateinthttp.NewCurrentStateAPIClient(b.Cfg.CurrentStateAPIURL(), b.httpClient)
+	if err != nil {
+		logrus.WithError(err).Panic("UserAPIClient failed", b.httpClient)
+	}
+	return stateAPI
+}
+
 // EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP
 func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI {
 	e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient)
diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go
index 24bee9502..86275e28d 100644
--- a/internal/setup/monolith.go
+++ b/internal/setup/monolith.go
@@ -19,6 +19,7 @@ import (
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/federationapi"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
@@ -56,6 +57,7 @@ type Monolith struct {
 	RoomserverAPI       roomserverAPI.RoomserverInternalAPI
 	ServerKeyAPI        serverKeyAPI.ServerKeyInternalAPI
 	UserAPI             userapi.UserInternalAPI
+	StateAPI            currentstateAPI.CurrentStateInternalAPI
 
 	// TODO: can we remove this? It's weird that we are required the database
 	// yet every other component can do that on its own. libp2p-demo uses a custom
@@ -69,9 +71,9 @@ type Monolith struct {
 // AddAllPublicRoutes attaches all public paths to the given router
 func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) {
 	clientapi.AddPublicRoutes(
-		publicMux, m.Config, m.KafkaConsumer, m.KafkaProducer, m.DeviceDB, m.AccountDB,
+		publicMux, m.Config, m.KafkaProducer, m.DeviceDB, m.AccountDB,
 		m.FedClient, m.RoomserverAPI,
-		m.EDUInternalAPI, m.AppserviceAPI, transactions.New(),
+		m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(),
 		m.FederationSenderAPI, m.UserAPI,
 	)
 
diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go
index 9ed33e1b9..6f6caf111 100644
--- a/userapi/storage/accounts/interface.go
+++ b/userapi/storage/accounts/interface.go
@@ -22,7 +22,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
@@ -36,10 +35,6 @@ type Database interface {
 	// account already exists, it will return nil, ErrUserExists.
 	CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*api.Account, error)
 	CreateGuestAccount(ctx context.Context) (*api.Account, error)
-	UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error
-	GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error)
-	GetRoomIDsByLocalPart(ctx context.Context, localpart string) ([]string, error)
-	GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error)
 	SaveAccountData(ctx context.Context, localpart, roomID, dataType string, content json.RawMessage) error
 	GetAccountData(ctx context.Context, localpart string) (global map[string]json.RawMessage, rooms map[string]map[string]json.RawMessage, err error)
 	// GetAccountDataByType returns account data matching a given
diff --git a/userapi/storage/accounts/postgres/membership_table.go b/userapi/storage/accounts/postgres/membership_table.go
deleted file mode 100644
index 623530acc..000000000
--- a/userapi/storage/accounts/postgres/membership_table.go
+++ /dev/null
@@ -1,159 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"context"
-	"database/sql"
-
-	"github.com/lib/pq"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/internal"
-)
-
-const membershipSchema = `
--- Stores data about users memberships to rooms.
-CREATE TABLE IF NOT EXISTS account_memberships (
-    -- The Matrix user ID localpart for the member
-    localpart TEXT NOT NULL,
-    -- The room this user is a member of
-    room_id TEXT NOT NULL,
-    -- The ID of the join membership event
-    event_id TEXT NOT NULL,
-
-    -- A user can only be member of a room once
-    PRIMARY KEY (localpart, room_id)
-);
-
--- Use index to process deletion by ID more efficiently
-CREATE UNIQUE INDEX IF NOT EXISTS account_membership_event_id ON account_memberships(event_id);
-`
-
-const insertMembershipSQL = `
-	INSERT INTO account_memberships(localpart, room_id, event_id) VALUES ($1, $2, $3)
-	ON CONFLICT (localpart, room_id) DO UPDATE SET event_id = EXCLUDED.event_id
-`
-
-const selectMembershipsByLocalpartSQL = "" +
-	"SELECT room_id, event_id FROM account_memberships WHERE localpart = $1"
-
-const selectMembershipInRoomByLocalpartSQL = "" +
-	"SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2"
-
-const selectRoomIDsByLocalPartSQL = "" +
-	"SELECT room_id FROM account_memberships WHERE localpart = $1"
-
-const deleteMembershipsByEventIDsSQL = "" +
-	"DELETE FROM account_memberships WHERE event_id = ANY($1)"
-
-type membershipStatements struct {
-	deleteMembershipsByEventIDsStmt       *sql.Stmt
-	insertMembershipStmt                  *sql.Stmt
-	selectMembershipInRoomByLocalpartStmt *sql.Stmt
-	selectMembershipsByLocalpartStmt      *sql.Stmt
-	selectRoomIDsByLocalPartStmt          *sql.Stmt
-}
-
-func (s *membershipStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(membershipSchema)
-	if err != nil {
-		return
-	}
-	if s.deleteMembershipsByEventIDsStmt, err = db.Prepare(deleteMembershipsByEventIDsSQL); err != nil {
-		return
-	}
-	if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil {
-		return
-	}
-	if s.selectMembershipInRoomByLocalpartStmt, err = db.Prepare(selectMembershipInRoomByLocalpartSQL); err != nil {
-		return
-	}
-	if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil {
-		return
-	}
-	if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil {
-		return
-	}
-	return
-}
-
-func (s *membershipStatements) insertMembership(
-	ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string,
-) (err error) {
-	stmt := txn.Stmt(s.insertMembershipStmt)
-	_, err = stmt.ExecContext(ctx, localpart, roomID, eventID)
-	return
-}
-
-func (s *membershipStatements) deleteMembershipsByEventIDs(
-	ctx context.Context, txn *sql.Tx, eventIDs []string,
-) (err error) {
-	stmt := txn.Stmt(s.deleteMembershipsByEventIDsStmt)
-	_, err = stmt.ExecContext(ctx, pq.StringArray(eventIDs))
-	return
-}
-
-func (s *membershipStatements) selectMembershipInRoomByLocalpart(
-	ctx context.Context, localpart, roomID string,
-) (authtypes.Membership, error) {
-	membership := authtypes.Membership{Localpart: localpart, RoomID: roomID}
-	stmt := s.selectMembershipInRoomByLocalpartStmt
-	err := stmt.QueryRowContext(ctx, localpart, roomID).Scan(&membership.EventID)
-
-	return membership, err
-}
-
-func (s *membershipStatements) selectMembershipsByLocalpart(
-	ctx context.Context, localpart string,
-) (memberships []authtypes.Membership, err error) {
-	stmt := s.selectMembershipsByLocalpartStmt
-	rows, err := stmt.QueryContext(ctx, localpart)
-	if err != nil {
-		return
-	}
-
-	memberships = []authtypes.Membership{}
-
-	defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed")
-	for rows.Next() {
-		var m authtypes.Membership
-		m.Localpart = localpart
-		if err = rows.Scan(&m.RoomID, &m.EventID); err != nil {
-			return
-		}
-		memberships = append(memberships, m)
-	}
-	return memberships, rows.Err()
-}
-
-func (s *membershipStatements) selectRoomIDsByLocalPart(
-	ctx context.Context, localPart string,
-) ([]string, error) {
-	stmt := s.selectRoomIDsByLocalPartStmt
-	rows, err := stmt.QueryContext(ctx, localPart)
-	if err != nil {
-		return nil, err
-	}
-	roomIDs := []string{}
-	defer rows.Close() // nolint: errcheck
-	for rows.Next() {
-		var roomID string
-		if err = rows.Scan(&roomID); err != nil {
-			return nil, err
-		}
-		roomIDs = append(roomIDs, roomID)
-	}
-	return roomIDs, rows.Err()
-}
diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go
index f0b11bfdb..c76b92f10 100644
--- a/userapi/storage/accounts/postgres/storage.go
+++ b/userapi/storage/accounts/postgres/storage.go
@@ -37,7 +37,6 @@ type Database struct {
 	sqlutil.PartitionOffsetStatements
 	accounts     accountsStatements
 	profiles     profilesStatements
-	memberships  membershipStatements
 	accountDatas accountDataStatements
 	threepids    threepidStatements
 	serverName   gomatrixserverlib.ServerName
@@ -62,10 +61,6 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serve
 	if err = p.prepare(db); err != nil {
 		return nil, err
 	}
-	m := membershipStatements{}
-	if err = m.prepare(db); err != nil {
-		return nil, err
-	}
 	ac := accountDataStatements{}
 	if err = ac.prepare(db); err != nil {
 		return nil, err
@@ -74,7 +69,7 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serve
 	if err = t.prepare(db); err != nil {
 		return nil, err
 	}
-	return &Database{db, partitions, a, p, m, ac, t, serverName}, nil
+	return &Database{db, partitions, a, p, ac, t, serverName}, nil
 }
 
 // GetAccountByPassword returns the account associated with the given localpart and password.
@@ -179,112 +174,6 @@ func (d *Database) createAccount(
 	return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID)
 }
 
-// SaveMembership saves the user matching a given localpart as a member of a given
-// room. It also stores the ID of the membership event.
-// If a membership already exists between the user and the room, or if the
-// insert fails, returns the SQL error
-func (d *Database) saveMembership(
-	ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string,
-) error {
-	return d.memberships.insertMembership(ctx, txn, localpart, roomID, eventID)
-}
-
-// removeMembershipsByEventIDs removes the memberships corresponding to the
-// `join` membership events IDs in the eventIDs slice.
-// If the removal fails, or if there is no membership to remove, returns an error
-func (d *Database) removeMembershipsByEventIDs(
-	ctx context.Context, txn *sql.Tx, eventIDs []string,
-) error {
-	return d.memberships.deleteMembershipsByEventIDs(ctx, txn, eventIDs)
-}
-
-// UpdateMemberships adds the "join" membership events included in a given state
-// events array, and removes those which ID is included in a given array of events
-// IDs. All of the process is run in a transaction, which commits only once/if every
-// insertion and deletion has been successfully processed.
-// Returns a SQL error if there was an issue with any part of the process
-func (d *Database) UpdateMemberships(
-	ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string,
-) error {
-	return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
-		if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil {
-			return err
-		}
-
-		for _, event := range eventsToAdd {
-			if err := d.newMembership(ctx, txn, event); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	})
-}
-
-// GetMembershipInRoomByLocalpart returns the membership for an user
-// matching the given localpart if he is a member of the room matching roomID,
-// if not sql.ErrNoRows is returned.
-// If there was an issue during the retrieval, returns the SQL error
-func (d *Database) GetMembershipInRoomByLocalpart(
-	ctx context.Context, localpart, roomID string,
-) (authtypes.Membership, error) {
-	return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID)
-}
-
-// GetRoomIDsByLocalPart returns an array containing the room ids of all
-// the rooms a user matching a given localpart is a member of
-// If no membership match the given localpart, returns an empty array
-// If there was an issue during the retrieval, returns the SQL error
-func (d *Database) GetRoomIDsByLocalPart(
-	ctx context.Context, localpart string,
-) ([]string, error) {
-	return d.memberships.selectRoomIDsByLocalPart(ctx, localpart)
-}
-
-// GetMembershipsByLocalpart returns an array containing the memberships for all
-// the rooms a user matching a given localpart is a member of
-// If no membership match the given localpart, returns an empty array
-// If there was an issue during the retrieval, returns the SQL error
-func (d *Database) GetMembershipsByLocalpart(
-	ctx context.Context, localpart string,
-) (memberships []authtypes.Membership, err error) {
-	return d.memberships.selectMembershipsByLocalpart(ctx, localpart)
-}
-
-// newMembership saves a new membership in the database.
-// If the event isn't a valid m.room.member event with type `join`, does nothing.
-// If an error occurred, returns the SQL error
-func (d *Database) newMembership(
-	ctx context.Context, txn *sql.Tx, ev gomatrixserverlib.Event,
-) error {
-	if ev.Type() == "m.room.member" && ev.StateKey() != nil {
-		localpart, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey())
-		if err != nil {
-			return err
-		}
-
-		// We only want state events from local users
-		if string(serverName) != string(d.serverName) {
-			return nil
-		}
-
-		eventID := ev.EventID()
-		roomID := ev.RoomID()
-		membership, err := ev.Membership()
-		if err != nil {
-			return err
-		}
-
-		// Only "join" membership events can be considered as new memberships
-		if membership == gomatrixserverlib.Join {
-			if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
 // SaveAccountData saves new account data for a given user and a given room.
 // If the account data is not specific to a room, the room ID should be an empty string
 // If an account data already exists for a given set (user, room, data type), it will
diff --git a/userapi/storage/accounts/sqlite3/membership_table.go b/userapi/storage/accounts/sqlite3/membership_table.go
deleted file mode 100644
index 67958f27d..000000000
--- a/userapi/storage/accounts/sqlite3/membership_table.go
+++ /dev/null
@@ -1,159 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-	"strings"
-
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-)
-
-const membershipSchema = `
--- Stores data about users memberships to rooms.
-CREATE TABLE IF NOT EXISTS account_memberships (
-    -- The Matrix user ID localpart for the member
-    localpart TEXT NOT NULL,
-    -- The room this user is a member of
-    room_id TEXT NOT NULL,
-    -- The ID of the join membership event
-    event_id TEXT NOT NULL,
-
-    -- A user can only be member of a room once
-    PRIMARY KEY (localpart, room_id),
-
-		UNIQUE (event_id)
-);
-`
-
-const insertMembershipSQL = `
-	INSERT INTO account_memberships(localpart, room_id, event_id) VALUES ($1, $2, $3)
-	ON CONFLICT (localpart, room_id) DO UPDATE SET event_id = EXCLUDED.event_id
-`
-
-const selectMembershipsByLocalpartSQL = "" +
-	"SELECT room_id, event_id FROM account_memberships WHERE localpart = $1"
-
-const selectMembershipInRoomByLocalpartSQL = "" +
-	"SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2"
-
-const selectRoomIDsByLocalPartSQL = "" +
-	"SELECT room_id FROM account_memberships WHERE localpart = $1"
-
-const deleteMembershipsByEventIDsSQL = "" +
-	"DELETE FROM account_memberships WHERE event_id IN ($1)"
-
-type membershipStatements struct {
-	insertMembershipStmt                  *sql.Stmt
-	selectMembershipInRoomByLocalpartStmt *sql.Stmt
-	selectMembershipsByLocalpartStmt      *sql.Stmt
-	selectRoomIDsByLocalPartStmt          *sql.Stmt
-}
-
-func (s *membershipStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(membershipSchema)
-	if err != nil {
-		return
-	}
-	if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil {
-		return
-	}
-	if s.selectMembershipInRoomByLocalpartStmt, err = db.Prepare(selectMembershipInRoomByLocalpartSQL); err != nil {
-		return
-	}
-	if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil {
-		return
-	}
-	if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil {
-		return
-	}
-	return
-}
-
-func (s *membershipStatements) insertMembership(
-	ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string,
-) (err error) {
-	stmt := txn.Stmt(s.insertMembershipStmt)
-	_, err = stmt.ExecContext(ctx, localpart, roomID, eventID)
-	return
-}
-
-func (s *membershipStatements) deleteMembershipsByEventIDs(
-	ctx context.Context, txn *sql.Tx, eventIDs []string,
-) (err error) {
-	sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(eventIDs)), 1)
-	iEventIDs := make([]interface{}, len(eventIDs))
-	for i, e := range eventIDs {
-		iEventIDs[i] = e
-	}
-	_, err = txn.ExecContext(ctx, sqlStr, iEventIDs...)
-	return
-}
-
-func (s *membershipStatements) selectMembershipInRoomByLocalpart(
-	ctx context.Context, localpart, roomID string,
-) (authtypes.Membership, error) {
-	membership := authtypes.Membership{Localpart: localpart, RoomID: roomID}
-	stmt := s.selectMembershipInRoomByLocalpartStmt
-	err := stmt.QueryRowContext(ctx, localpart, roomID).Scan(&membership.EventID)
-
-	return membership, err
-}
-
-func (s *membershipStatements) selectMembershipsByLocalpart(
-	ctx context.Context, localpart string,
-) (memberships []authtypes.Membership, err error) {
-	stmt := s.selectMembershipsByLocalpartStmt
-	rows, err := stmt.QueryContext(ctx, localpart)
-	if err != nil {
-		return
-	}
-
-	memberships = []authtypes.Membership{}
-
-	defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed")
-	for rows.Next() {
-		var m authtypes.Membership
-		m.Localpart = localpart
-		if err := rows.Scan(&m.RoomID, &m.EventID); err != nil {
-			return nil, err
-		}
-		memberships = append(memberships, m)
-	}
-
-	return
-}
-func (s *membershipStatements) selectRoomIDsByLocalPart(
-	ctx context.Context, localPart string,
-) ([]string, error) {
-	stmt := s.selectRoomIDsByLocalPartStmt
-	rows, err := stmt.QueryContext(ctx, localPart)
-	if err != nil {
-		return nil, err
-	}
-	roomIDs := []string{}
-	defer rows.Close() // nolint: errcheck
-	for rows.Next() {
-		var roomID string
-		if err = rows.Scan(&roomID); err != nil {
-			return nil, err
-		}
-		roomIDs = append(roomIDs, roomID)
-	}
-	return roomIDs, rows.Err()
-}
diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go
index e965df4f9..72b27c8bf 100644
--- a/userapi/storage/accounts/sqlite3/storage.go
+++ b/userapi/storage/accounts/sqlite3/storage.go
@@ -36,7 +36,6 @@ type Database struct {
 	sqlutil.PartitionOffsetStatements
 	accounts     accountsStatements
 	profiles     profilesStatements
-	memberships  membershipStatements
 	accountDatas accountDataStatements
 	threepids    threepidStatements
 	serverName   gomatrixserverlib.ServerName
@@ -67,10 +66,6 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName)
 	if err = p.prepare(db); err != nil {
 		return nil, err
 	}
-	m := membershipStatements{}
-	if err = m.prepare(db); err != nil {
-		return nil, err
-	}
 	ac := accountDataStatements{}
 	if err = ac.prepare(db); err != nil {
 		return nil, err
@@ -79,7 +74,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName)
 	if err = t.prepare(db); err != nil {
 		return nil, err
 	}
-	return &Database{db, partitions, a, p, m, ac, t, serverName, sync.Mutex{}}, nil
+	return &Database{db, partitions, a, p, ac, t, serverName, sync.Mutex{}}, nil
 }
 
 // GetAccountByPassword returns the account associated with the given localpart and password.
@@ -193,112 +188,6 @@ func (d *Database) createAccount(
 	return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID)
 }
 
-// SaveMembership saves the user matching a given localpart as a member of a given
-// room. It also stores the ID of the membership event.
-// If a membership already exists between the user and the room, or if the
-// insert fails, returns the SQL error
-func (d *Database) saveMembership(
-	ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string,
-) error {
-	return d.memberships.insertMembership(ctx, txn, localpart, roomID, eventID)
-}
-
-// removeMembershipsByEventIDs removes the memberships corresponding to the
-// `join` membership events IDs in the eventIDs slice.
-// If the removal fails, or if there is no membership to remove, returns an error
-func (d *Database) removeMembershipsByEventIDs(
-	ctx context.Context, txn *sql.Tx, eventIDs []string,
-) error {
-	return d.memberships.deleteMembershipsByEventIDs(ctx, txn, eventIDs)
-}
-
-// UpdateMemberships adds the "join" membership events included in a given state
-// events array, and removes those which ID is included in a given array of events
-// IDs. All of the process is run in a transaction, which commits only once/if every
-// insertion and deletion has been successfully processed.
-// Returns a SQL error if there was an issue with any part of the process
-func (d *Database) UpdateMemberships(
-	ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string,
-) error {
-	return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
-		if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil {
-			return err
-		}
-
-		for _, event := range eventsToAdd {
-			if err := d.newMembership(ctx, txn, event); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	})
-}
-
-// GetMembershipInRoomByLocalpart returns the membership for an user
-// matching the given localpart if he is a member of the room matching roomID,
-// if not sql.ErrNoRows is returned.
-// If there was an issue during the retrieval, returns the SQL error
-func (d *Database) GetMembershipInRoomByLocalpart(
-	ctx context.Context, localpart, roomID string,
-) (authtypes.Membership, error) {
-	return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID)
-}
-
-// GetMembershipsByLocalpart returns an array containing the memberships for all
-// the rooms a user matching a given localpart is a member of
-// If no membership match the given localpart, returns an empty array
-// If there was an issue during the retrieval, returns the SQL error
-func (d *Database) GetMembershipsByLocalpart(
-	ctx context.Context, localpart string,
-) (memberships []authtypes.Membership, err error) {
-	return d.memberships.selectMembershipsByLocalpart(ctx, localpart)
-}
-
-// GetRoomIDsByLocalPart returns an array containing the room ids of all
-// the rooms a user matching a given localpart is a member of
-// If no membership match the given localpart, returns an empty array
-// If there was an issue during the retrieval, returns the SQL error
-func (d *Database) GetRoomIDsByLocalPart(
-	ctx context.Context, localpart string,
-) ([]string, error) {
-	return d.memberships.selectRoomIDsByLocalPart(ctx, localpart)
-}
-
-// newMembership saves a new membership in the database.
-// If the event isn't a valid m.room.member event with type `join`, does nothing.
-// If an error occurred, returns the SQL error
-func (d *Database) newMembership(
-	ctx context.Context, txn *sql.Tx, ev gomatrixserverlib.Event,
-) error {
-	if ev.Type() == "m.room.member" && ev.StateKey() != nil {
-		localpart, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey())
-		if err != nil {
-			return err
-		}
-
-		// We only want state events from local users
-		if string(serverName) != string(d.serverName) {
-			return nil
-		}
-
-		eventID := ev.EventID()
-		roomID := ev.RoomID()
-		membership, err := ev.Membership()
-		if err != nil {
-			return err
-		}
-
-		// Only "join" membership events can be considered as new memberships
-		if membership == gomatrixserverlib.Join {
-			if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
 // SaveAccountData saves new account data for a given user and a given room.
 // If the account data is not specific to a room, the room ID should be an empty string
 // If an account data already exists for a given set (user, room, data type), it will

From 42dd96242574866378bb95d92bc0c7fdf3dbabf6 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 1 Jul 2020 11:46:38 +0100
Subject: [PATCH 17/23] Persistent federation sender queues (PDUs) (#1173)

* Initial work on persistent queues

* Update index for event ID and server name

* Put things into database (postgres for now)

* Duplicate postgres code into sqlite for now just to stop build errors, will fix SQLite soon

* Fix table name

* Fix index

* Fix table name

* Use RETURNING because LastInsertID is not supported by postgres

* Use functions

* Marshal headered event

* Don't error on now rows

* Don't block if there are PDUs waiting

* Try to tidy up JSON

* Debug logging

* Fix query, use transactions in postgres

* Clean up

* Rehydrate more opportunistically

* Fix SQLite

* remove unused types

* Review comments

* Shuffle things around a bit

* Clean up transaction properly

* Don't send empty transactions

* Reduce unnecessary retries

* Count PDUs to make more resilient

* Don't stop when there is work to be done

* Try to limit wakeups

* well this is tedious

* Fix race in incomplete transactions

* Thread safety on transaction ID/count
---
 cmd/dendrite-demo-libp2p/main.go              |   3 +-
 cmd/dendrite-demo-yggdrasil/main.go           |   3 +-
 federationsender/federationsender.go          |   3 +-
 federationsender/queue/destinationqueue.go    | 267 +++++++++++-------
 federationsender/queue/queue.go               |  21 +-
 federationsender/storage/interface.go         |   5 +
 .../storage/postgres/queue_json_table.go      | 111 ++++++++
 .../storage/postgres/queue_pdus_table.go      | 169 +++++++++++
 federationsender/storage/postgres/storage.go  | 135 +++++++++
 .../storage/sqlite3/queue_json_table.go       | 132 +++++++++
 .../storage/sqlite3/queue_pdus_table.go       | 167 +++++++++++
 federationsender/storage/sqlite3/storage.go   | 135 +++++++++
 12 files changed, 1049 insertions(+), 102 deletions(-)
 create mode 100644 federationsender/storage/postgres/queue_json_table.go
 create mode 100644 federationsender/storage/postgres/queue_pdus_table.go
 create mode 100644 federationsender/storage/sqlite3/queue_json_table.go
 create mode 100644 federationsender/storage/sqlite3/queue_pdus_table.go

diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go
index b7e86b77c..4bb7a96c2 100644
--- a/cmd/dendrite-demo-libp2p/main.go
+++ b/cmd/dendrite-demo-libp2p/main.go
@@ -130,8 +130,9 @@ func main() {
 	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
 	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
 	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName))
+	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicrooms.db", *instanceName))
 	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
+	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 5de674021..cef34c7ed 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -74,7 +74,8 @@ func main() {
 	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
 	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
 	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName))
+	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicrooms.db", *instanceName))
+	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
 	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go
index 10ac51c8a..acf524146 100644
--- a/federationsender/federationsender.go
+++ b/federationsender/federationsender.go
@@ -50,7 +50,8 @@ func NewInternalAPI(
 
 	statistics := &types.Statistics{}
 	queues := queue.NewOutgoingQueues(
-		base.Cfg.Matrix.ServerName, federation, rsAPI, statistics, &queue.SigningInfo{
+		federationSenderDB, base.Cfg.Matrix.ServerName, federation, rsAPI, statistics,
+		&queue.SigningInfo{
 			KeyID:      base.Cfg.Matrix.KeyID,
 			PrivateKey: base.Cfg.Matrix.PrivateKey,
 			ServerName: base.Cfg.Matrix.ServerName,
diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go
index 4449f9e63..a736b3852 100644
--- a/federationsender/queue/destinationqueue.go
+++ b/federationsender/queue/destinationqueue.go
@@ -18,8 +18,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationsender/storage"
 	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrix"
@@ -29,11 +31,14 @@ import (
 	"go.uber.org/atomic"
 )
 
+const maxPDUsPerTransaction = 50
+
 // destinationQueue is a queue of events for a single destination.
 // It is responsible for sending the events to the destination and
 // ensures that only one request is in flight to a given destination
 // at a time.
 type destinationQueue struct {
+	db                 storage.Database
 	signing            *SigningInfo
 	rsAPI              api.RoomserverInternalAPI
 	client             *gomatrixserverlib.FederationClient     // federation client
@@ -42,13 +47,15 @@ type destinationQueue struct {
 	running            atomic.Bool                             // is the queue worker running?
 	backingOff         atomic.Bool                             // true if we're backing off
 	statistics         *types.ServerStatistics                 // statistics about this remote server
-	incomingPDUs       chan *gomatrixserverlib.HeaderedEvent   // PDUs to send
-	incomingEDUs       chan *gomatrixserverlib.EDU             // EDUs to send
 	incomingInvites    chan *gomatrixserverlib.InviteV2Request // invites to send
-	lastTransactionIDs []gomatrixserverlib.TransactionID       // last transaction ID
-	pendingPDUs        []*gomatrixserverlib.HeaderedEvent      // owned by backgroundSend
+	incomingEDUs       chan *gomatrixserverlib.EDU             // EDUs to send
+	transactionIDMutex sync.Mutex                              // protects transactionID
+	transactionID      gomatrixserverlib.TransactionID         // last transaction ID
+	transactionCount   atomic.Int32                            // how many events in this transaction so far
+	pendingPDUs        atomic.Int32                            // how many PDUs are waiting to be sent
 	pendingEDUs        []*gomatrixserverlib.EDU                // owned by backgroundSend
 	pendingInvites     []*gomatrixserverlib.InviteV2Request    // owned by backgroundSend
+	wakeServerCh       chan bool                               // interrupts idle wait
 	retryServerCh      chan bool                               // interrupts backoff
 }
 
@@ -79,15 +86,47 @@ func (oq *destinationQueue) retry() {
 // Send event adds the event to the pending queue for the destination.
 // If the queue is empty then it starts a background goroutine to
 // start sending events to that destination.
-func (oq *destinationQueue) sendEvent(ev *gomatrixserverlib.HeaderedEvent) {
+func (oq *destinationQueue) sendEvent(nid int64) {
 	if oq.statistics.Blacklisted() {
 		// If the destination is blacklisted then drop the event.
 		return
 	}
+	// Create a transaction ID. We'll either do this if we don't have
+	// one made up yet, or if we've exceeded the number of maximum
+	// events allowed in a single tranaction. We'll reset the counter
+	// when we do.
+	oq.transactionIDMutex.Lock()
+	if oq.transactionID == "" || oq.transactionCount.Load() >= maxPDUsPerTransaction {
+		now := gomatrixserverlib.AsTimestamp(time.Now())
+		oq.transactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount()))
+		oq.transactionCount.Store(0)
+	}
+	oq.transactionIDMutex.Unlock()
+	// Create a database entry that associates the given PDU NID with
+	// this destination queue. We'll then be able to retrieve the PDU
+	// later.
+	if err := oq.db.AssociatePDUWithDestination(
+		context.TODO(),
+		oq.transactionID, // the current transaction ID
+		oq.destination,   // the destination server name
+		[]int64{nid},     // NID from federationsender_queue_json table
+	); err != nil {
+		log.WithError(err).Errorf("failed to associate PDU NID %d with destination %q", nid, oq.destination)
+		return
+	}
+	// We've successfully added a PDU to the transaction so increase
+	// the counter.
+	oq.transactionCount.Add(1)
+	// If the queue isn't running at this point then start it.
 	if !oq.running.Load() {
 		go oq.backgroundSend()
 	}
-	oq.incomingPDUs <- ev
+	// Signal that we've sent a new PDU. This will cause the queue to
+	// wake up if it's asleep. The return to the Add function will only
+	// be 1 if the previous value was 0, e.g. nothing was waiting before.
+	if oq.pendingPDUs.Add(1) == 1 {
+		oq.wakeServerCh <- true
+	}
 }
 
 // sendEDU adds the EDU event to the pending queue for the destination.
@@ -129,56 +168,48 @@ func (oq *destinationQueue) backgroundSend() {
 	defer oq.running.Store(false)
 
 	for {
-		// Wait either for incoming events, or until we hit an
-		// idle timeout.
-		select {
-		case pdu := <-oq.incomingPDUs:
-			// Ordering of PDUs is important so we add them to the end
-			// of the queue and they will all be added to transactions
-			// in order.
-			oq.pendingPDUs = append(oq.pendingPDUs, pdu)
-			// If there are any more things waiting in the channel queue
-			// then read them. This is safe because we guarantee only
-			// having one goroutine per destination queue, so the channel
-			// isn't being consumed anywhere else.
-			for len(oq.incomingPDUs) > 0 {
-				oq.pendingPDUs = append(oq.pendingPDUs, <-oq.incomingPDUs)
+		// If we have nothing to do then wait either for incoming events, or
+		// until we hit an idle timeout.
+		if oq.pendingPDUs.Load() == 0 && len(oq.pendingEDUs) == 0 && len(oq.pendingInvites) == 0 {
+			select {
+			case <-oq.wakeServerCh:
+				// We were woken up because there are new PDUs waiting in the
+				// database.
+			case edu := <-oq.incomingEDUs:
+				// EDUs are handled in-memory for now. We will try to keep
+				// the ordering intact.
+				// TODO: Certain EDU types need persistence, e.g. send-to-device
+				oq.pendingEDUs = append(oq.pendingEDUs, edu)
+				// If there are any more things waiting in the channel queue
+				// then read them. This is safe because we guarantee only
+				// having one goroutine per destination queue, so the channel
+				// isn't being consumed anywhere else.
+				for len(oq.incomingEDUs) > 0 {
+					oq.pendingEDUs = append(oq.pendingEDUs, <-oq.incomingEDUs)
+				}
+			case invite := <-oq.incomingInvites:
+				// There's no strict ordering requirement for invites like
+				// there is for transactions, so we put the invite onto the
+				// front of the queue. This means that if an invite that is
+				// stuck failing already, that it won't block our new invite
+				// from being sent.
+				oq.pendingInvites = append(
+					[]*gomatrixserverlib.InviteV2Request{invite},
+					oq.pendingInvites...,
+				)
+				// If there are any more things waiting in the channel queue
+				// then read them. This is safe because we guarantee only
+				// having one goroutine per destination queue, so the channel
+				// isn't being consumed anywhere else.
+				for len(oq.incomingInvites) > 0 {
+					oq.pendingInvites = append(oq.pendingInvites, <-oq.incomingInvites)
+				}
+			case <-time.After(time.Second * 30):
+				// The worker is idle so stop the goroutine. It'll get
+				// restarted automatically the next time we have an event to
+				// send.
+				return
 			}
-		case edu := <-oq.incomingEDUs:
-			// Likewise for EDUs, although we should probably not try
-			// too hard with some EDUs (like typing notifications) after
-			// a certain amount of time has passed.
-			// TODO: think about EDU expiry some more
-			oq.pendingEDUs = append(oq.pendingEDUs, edu)
-			// If there are any more things waiting in the channel queue
-			// then read them. This is safe because we guarantee only
-			// having one goroutine per destination queue, so the channel
-			// isn't being consumed anywhere else.
-			for len(oq.incomingEDUs) > 0 {
-				oq.pendingEDUs = append(oq.pendingEDUs, <-oq.incomingEDUs)
-			}
-		case invite := <-oq.incomingInvites:
-			// There's no strict ordering requirement for invites like
-			// there is for transactions, so we put the invite onto the
-			// front of the queue. This means that if an invite that is
-			// stuck failing already, that it won't block our new invite
-			// from being sent.
-			oq.pendingInvites = append(
-				[]*gomatrixserverlib.InviteV2Request{invite},
-				oq.pendingInvites...,
-			)
-			// If there are any more things waiting in the channel queue
-			// then read them. This is safe because we guarantee only
-			// having one goroutine per destination queue, so the channel
-			// isn't being consumed anywhere else.
-			for len(oq.incomingInvites) > 0 {
-				oq.pendingInvites = append(oq.pendingInvites, <-oq.incomingInvites)
-			}
-		case <-time.After(time.Second * 30):
-			// The worker is idle so stop the goroutine. It'll
-			// get restarted automatically the next time we
-			// get an event.
-			return
 		}
 
 		// If we are backing off this server then wait for the
@@ -193,47 +224,31 @@ func (oq *destinationQueue) backgroundSend() {
 			oq.backingOff.Store(false)
 		}
 
-		// How many things do we have waiting?
-		numPDUs := len(oq.pendingPDUs)
-		numEDUs := len(oq.pendingEDUs)
-		numInvites := len(oq.pendingInvites)
-
 		// If we have pending PDUs or EDUs then construct a transaction.
-		if numPDUs > 0 || numEDUs > 0 {
+		if oq.pendingPDUs.Load() > 0 || len(oq.pendingEDUs) > 0 {
 			// Try sending the next transaction and see what happens.
-			transaction, terr := oq.nextTransaction(oq.pendingPDUs, oq.pendingEDUs, oq.statistics.SuccessCount())
+			transaction, terr := oq.nextTransaction(oq.pendingEDUs)
 			if terr != nil {
 				// We failed to send the transaction.
 				if giveUp := oq.statistics.Failure(); giveUp {
-					// It's been suggested that we should give up because
-					// the backoff has exceeded a maximum allowable value.
+					// It's been suggested that we should give up because the backoff
+					// has exceeded a maximum allowable value. Clean up the in-memory
+					// buffers at this point. The PDU clean-up is already on a defer.
+					oq.cleanPendingEDUs()
+					oq.cleanPendingInvites()
 					return
 				}
 			} else if transaction {
 				// If we successfully sent the transaction then clear out
-				// the pending events and EDUs.
+				// the pending events and EDUs, and wipe our transaction ID.
 				oq.statistics.Success()
-				// Reallocate so that the underlying arrays can be GC'd, as
-				// opposed to growing forever.
-				for i := 0; i < numPDUs; i++ {
-					oq.pendingPDUs[i] = nil
-				}
-				for i := 0; i < numEDUs; i++ {
-					oq.pendingEDUs[i] = nil
-				}
-				oq.pendingPDUs = append(
-					[]*gomatrixserverlib.HeaderedEvent{},
-					oq.pendingPDUs[numPDUs:]...,
-				)
-				oq.pendingEDUs = append(
-					[]*gomatrixserverlib.EDU{},
-					oq.pendingEDUs[numEDUs:]...,
-				)
+				// Clean up the in-memory buffers.
+				oq.cleanPendingEDUs()
 			}
 		}
 
 		// Try sending the next invite and see what happens.
-		if numInvites > 0 {
+		if len(oq.pendingInvites) > 0 {
 			sent, ierr := oq.nextInvites(oq.pendingInvites)
 			if ierr != nil {
 				// We failed to send the transaction so increase the
@@ -249,59 +264,117 @@ func (oq *destinationQueue) backgroundSend() {
 				oq.statistics.Success()
 				// Reallocate so that the underlying array can be GC'd, as
 				// opposed to growing forever.
-				oq.pendingInvites = append(
-					[]*gomatrixserverlib.InviteV2Request{},
-					oq.pendingInvites[sent:]...,
-				)
+				oq.cleanPendingInvites()
 			}
 		}
 	}
 }
 
+// cleanPendingEDUs cleans out the pending EDU buffer, removing
+// all references so that the underlying objects can be GC'd.
+func (oq *destinationQueue) cleanPendingEDUs() {
+	for i := 0; i < len(oq.pendingEDUs); i++ {
+		oq.pendingEDUs[i] = nil
+	}
+	oq.pendingEDUs = []*gomatrixserverlib.EDU{}
+}
+
+// cleanPendingInvites cleans out the pending invite buffer,
+// removing all references so that the underlying objects can
+// be GC'd.
+func (oq *destinationQueue) cleanPendingInvites() {
+	for i := 0; i < len(oq.pendingInvites); i++ {
+		oq.pendingInvites[i] = nil
+	}
+	oq.pendingInvites = []*gomatrixserverlib.InviteV2Request{}
+}
+
 // nextTransaction creates a new transaction from the pending event
 // queue and sends it. Returns true if a transaction was sent or
 // false otherwise.
 func (oq *destinationQueue) nextTransaction(
-	pendingPDUs []*gomatrixserverlib.HeaderedEvent,
 	pendingEDUs []*gomatrixserverlib.EDU,
-	sentCounter uint32,
 ) (bool, error) {
+	// Before we do anything, we need to roll over the transaction
+	// ID that is being used to coalesce events into the next TX.
+	// Otherwise it's possible that we'll pick up an incomplete
+	// transaction and end up nuking the rest of the events at the
+	// cleanup stage.
+	oq.transactionIDMutex.Lock()
+	oq.transactionID = ""
+	oq.transactionIDMutex.Unlock()
+	oq.transactionCount.Store(0)
+
+	// Create the transaction.
 	t := gomatrixserverlib.Transaction{
 		PDUs: []json.RawMessage{},
 		EDUs: []gomatrixserverlib.EDU{},
 	}
-	now := gomatrixserverlib.AsTimestamp(time.Now())
-	t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, sentCounter))
 	t.Origin = oq.origin
 	t.Destination = oq.destination
-	t.OriginServerTS = now
-	t.PreviousIDs = oq.lastTransactionIDs
-	if t.PreviousIDs == nil {
-		t.PreviousIDs = []gomatrixserverlib.TransactionID{}
+	t.OriginServerTS = gomatrixserverlib.AsTimestamp(time.Now())
+
+	// Ask the database for any pending PDUs from the next transaction.
+	// maxPDUsPerTransaction is an upper limit but we probably won't
+	// actually retrieve that many events.
+	txid, pdus, err := oq.db.GetNextTransactionPDUs(
+		context.TODO(),        // context
+		oq.destination,        // server name
+		maxPDUsPerTransaction, // max events to retrieve
+	)
+	if err != nil {
+		log.WithError(err).Errorf("failed to get next transaction PDUs for server %q", oq.destination)
+		return false, fmt.Errorf("oq.db.GetNextTransactionPDUs: %w", err)
 	}
 
-	oq.lastTransactionIDs = []gomatrixserverlib.TransactionID{t.TransactionID}
+	// If we didn't get anything from the database and there are no
+	// pending EDUs then there's nothing to do - stop here.
+	if len(pdus) == 0 && len(pendingEDUs) == 0 {
+		return false, nil
+	}
 
-	for _, pdu := range pendingPDUs {
+	// Pick out the transaction ID from the database. If we didn't
+	// get a transaction ID (i.e. because there are no PDUs but only
+	// EDUs) then generate a transaction ID.
+	t.TransactionID = txid
+	if t.TransactionID == "" {
+		now := gomatrixserverlib.AsTimestamp(time.Now())
+		t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount()))
+	}
+
+	// Go through PDUs that we retrieved from the database, if any,
+	// and add them into the transaction.
+	for _, pdu := range pdus {
 		// Append the JSON of the event, since this is a json.RawMessage type in the
 		// gomatrixserverlib.Transaction struct
 		t.PDUs = append(t.PDUs, (*pdu).JSON())
 	}
 
+	// Do the same for pending EDUS in the queue.
 	for _, edu := range pendingEDUs {
 		t.EDUs = append(t.EDUs, *edu)
 	}
 
 	logrus.WithField("server_name", oq.destination).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs))
 
+	// Try to send the transaction to the destination server.
 	// TODO: we should check for 500-ish fails vs 400-ish here,
 	// since we shouldn't queue things indefinitely in response
 	// to a 400-ish error
-	_, err := oq.client.SendTransaction(context.TODO(), t)
+	_, err = oq.client.SendTransaction(context.TODO(), t)
 	switch e := err.(type) {
 	case nil:
 		// No error was returned so the transaction looks to have
 		// been successfully sent.
+		oq.pendingPDUs.Sub(int32(len(t.PDUs)))
+		// Clean up the transaction in the database.
+		if err = oq.db.CleanTransactionPDUs(
+			context.TODO(),
+			t.Destination,
+			t.TransactionID,
+		); err != nil {
+			log.WithError(err).Errorf("failed to clean transaction %q for server %q", t.TransactionID, t.Destination)
+		}
 		return true, nil
 	case gomatrix.HTTPError:
 		// We received a HTTP error back. In this instance we only
diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go
index 240343559..492d5f553 100644
--- a/federationsender/queue/queue.go
+++ b/federationsender/queue/queue.go
@@ -15,10 +15,13 @@
 package queue
 
 import (
+	"context"
 	"crypto/ed25519"
+	"encoding/json"
 	"fmt"
 	"sync"
 
+	"github.com/matrix-org/dendrite/federationsender/storage"
 	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -29,6 +32,7 @@ import (
 // OutgoingQueues is a collection of queues for sending transactions to other
 // matrix servers
 type OutgoingQueues struct {
+	db          storage.Database
 	rsAPI       api.RoomserverInternalAPI
 	origin      gomatrixserverlib.ServerName
 	client      *gomatrixserverlib.FederationClient
@@ -40,6 +44,7 @@ type OutgoingQueues struct {
 
 // NewOutgoingQueues makes a new OutgoingQueues
 func NewOutgoingQueues(
+	db storage.Database,
 	origin gomatrixserverlib.ServerName,
 	client *gomatrixserverlib.FederationClient,
 	rsAPI api.RoomserverInternalAPI,
@@ -47,6 +52,7 @@ func NewOutgoingQueues(
 	signing *SigningInfo,
 ) *OutgoingQueues {
 	return &OutgoingQueues{
+		db:         db,
 		rsAPI:      rsAPI,
 		origin:     origin,
 		client:     client,
@@ -76,14 +82,15 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d
 	oq := oqs.queues[destination]
 	if oq == nil {
 		oq = &destinationQueue{
+			db:              oqs.db,
 			rsAPI:           oqs.rsAPI,
 			origin:          oqs.origin,
 			destination:     destination,
 			client:          oqs.client,
 			statistics:      oqs.statistics.ForServer(destination),
-			incomingPDUs:    make(chan *gomatrixserverlib.HeaderedEvent, 128),
 			incomingEDUs:    make(chan *gomatrixserverlib.EDU, 128),
 			incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128),
+			wakeServerCh:    make(chan bool, 128),
 			retryServerCh:   make(chan bool),
 			signing:         oqs.signing,
 		}
@@ -115,8 +122,18 @@ func (oqs *OutgoingQueues) SendEvent(
 		"destinations": destinations, "event": ev.EventID(),
 	}).Info("Sending event")
 
+	headeredJSON, err := json.Marshal(ev)
+	if err != nil {
+		return fmt.Errorf("json.Marshal: %w", err)
+	}
+
+	nid, err := oqs.db.StoreJSON(context.TODO(), string(headeredJSON))
+	if err != nil {
+		return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
+	}
+
 	for _, destination := range destinations {
-		oqs.getQueue(destination).sendEvent(ev)
+		oqs.getQueue(destination).sendEvent(nid)
 	}
 
 	return nil
diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go
index be195382b..f4df93fa4 100644
--- a/federationsender/storage/interface.go
+++ b/federationsender/storage/interface.go
@@ -19,10 +19,15 @@ import (
 
 	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
 	internal.PartitionStorer
 	UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error)
 	GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error)
+	StoreJSON(ctx context.Context, js string) (int64, error)
+	AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, nids []int64) error
+	GetNextTransactionPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (gomatrixserverlib.TransactionID, []*gomatrixserverlib.HeaderedEvent, error)
+	CleanTransactionPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, transactionID gomatrixserverlib.TransactionID) error
 }
diff --git a/federationsender/storage/postgres/queue_json_table.go b/federationsender/storage/postgres/queue_json_table.go
new file mode 100644
index 000000000..eac2ea988
--- /dev/null
+++ b/federationsender/storage/postgres/queue_json_table.go
@@ -0,0 +1,111 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+const queueJSONSchema = `
+-- The federationsender_queue_json table contains event contents that
+-- we failed to send. 
+CREATE TABLE IF NOT EXISTS federationsender_queue_json (
+	-- The JSON NID. This allows the federationsender_queue_retry table to
+	-- cross-reference to find the JSON blob.
+	json_nid BIGSERIAL,
+	-- The JSON body. Text so that we preserve UTF-8.
+	json_body TEXT NOT NULL
+);
+`
+
+const insertJSONSQL = "" +
+	"INSERT INTO federationsender_queue_json (json_body)" +
+	" VALUES ($1)" +
+	" RETURNING json_nid"
+
+const deleteJSONSQL = "" +
+	"DELETE FROM federationsender_queue_json WHERE json_nid = ANY($1)"
+
+const selectJSONSQL = "" +
+	"SELECT json_nid, json_body FROM federationsender_queue_json" +
+	" WHERE json_nid = ANY($1)"
+
+type queueJSONStatements struct {
+	insertJSONStmt *sql.Stmt
+	deleteJSONStmt *sql.Stmt
+	selectJSONStmt *sql.Stmt
+}
+
+func (s *queueJSONStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(queueJSONSchema)
+	if err != nil {
+		return
+	}
+	if s.insertJSONStmt, err = db.Prepare(insertJSONSQL); err != nil {
+		return
+	}
+	if s.deleteJSONStmt, err = db.Prepare(deleteJSONSQL); err != nil {
+		return
+	}
+	if s.selectJSONStmt, err = db.Prepare(selectJSONSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *queueJSONStatements) insertQueueJSON(
+	ctx context.Context, txn *sql.Tx, json string,
+) (int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.insertJSONStmt)
+	var lastid int64
+	if err := stmt.QueryRowContext(ctx, json).Scan(&lastid); err != nil {
+		return 0, err
+	}
+	return lastid, nil
+}
+
+func (s *queueJSONStatements) deleteQueueJSON(
+	ctx context.Context, txn *sql.Tx, nids []int64,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteJSONStmt)
+	_, err := stmt.ExecContext(ctx, pq.Int64Array(nids))
+	return err
+}
+
+func (s *queueJSONStatements) selectQueueJSON(
+	ctx context.Context, txn *sql.Tx, jsonNIDs []int64,
+) (map[int64][]byte, error) {
+	blobs := map[int64][]byte{}
+	stmt := sqlutil.TxStmt(txn, s.selectJSONStmt)
+	rows, err := stmt.QueryContext(ctx, pq.Int64Array(jsonNIDs))
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectJSON: rows.close() failed")
+	for rows.Next() {
+		var nid int64
+		var blob []byte
+		if err = rows.Scan(&nid, &blob); err != nil {
+			return nil, err
+		}
+		blobs[nid] = blob
+	}
+	return blobs, err
+}
diff --git a/federationsender/storage/postgres/queue_pdus_table.go b/federationsender/storage/postgres/queue_pdus_table.go
new file mode 100644
index 000000000..ef7a9f41e
--- /dev/null
+++ b/federationsender/storage/postgres/queue_pdus_table.go
@@ -0,0 +1,169 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const queuePDUsSchema = `
+CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
+    -- The transaction ID that was generated before persisting the event.
+	transaction_id TEXT NOT NULL,
+    -- The destination server that we will send the event to.
+	server_name TEXT NOT NULL,
+	-- The JSON NID from the federationsender_queue_pdus_json table.
+	json_nid BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
+    ON federationsender_queue_pdus (json_nid, server_name);
+`
+
+const insertQueuePDUSQL = "" +
+	"INSERT INTO federationsender_queue_pdus (transaction_id, server_name, json_nid)" +
+	" VALUES ($1, $2, $3)"
+
+const deleteQueueTransactionPDUsSQL = "" +
+	"DELETE FROM federationsender_queue_pdus WHERE server_name = $1 AND transaction_id = $2"
+
+const selectQueueNextTransactionIDSQL = "" +
+	"SELECT transaction_id FROM federationsender_queue_pdus" +
+	" WHERE server_name = $1" +
+	" ORDER BY transaction_id ASC" +
+	" LIMIT 1"
+
+const selectQueuePDUsByTransactionSQL = "" +
+	"SELECT json_nid FROM federationsender_queue_pdus" +
+	" WHERE server_name = $1 AND transaction_id = $2" +
+	" LIMIT $3"
+
+const selectQueueReferenceJSONCountSQL = "" +
+	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
+	" WHERE json_nid = $1"
+
+type queuePDUsStatements struct {
+	insertQueuePDUStmt                *sql.Stmt
+	deleteQueueTransactionPDUsStmt    *sql.Stmt
+	selectQueueNextTransactionIDStmt  *sql.Stmt
+	selectQueuePDUsByTransactionStmt  *sql.Stmt
+	selectQueueReferenceJSONCountStmt *sql.Stmt
+}
+
+func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(queuePDUsSchema)
+	if err != nil {
+		return
+	}
+	if s.insertQueuePDUStmt, err = db.Prepare(insertQueuePDUSQL); err != nil {
+		return
+	}
+	if s.deleteQueueTransactionPDUsStmt, err = db.Prepare(deleteQueueTransactionPDUsSQL); err != nil {
+		return
+	}
+	if s.selectQueueNextTransactionIDStmt, err = db.Prepare(selectQueueNextTransactionIDSQL); err != nil {
+		return
+	}
+	if s.selectQueuePDUsByTransactionStmt, err = db.Prepare(selectQueuePDUsByTransactionSQL); err != nil {
+		return
+	}
+	if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueueReferenceJSONCountSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *queuePDUsStatements) insertQueuePDU(
+	ctx context.Context,
+	txn *sql.Tx,
+	transactionID gomatrixserverlib.TransactionID,
+	serverName gomatrixserverlib.ServerName,
+	nid int64,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.insertQueuePDUStmt)
+	_, err := stmt.ExecContext(
+		ctx,
+		transactionID, // the transaction ID that we initially attempted
+		serverName,    // destination server name
+		nid,           // JSON blob NID
+	)
+	return err
+}
+
+func (s *queuePDUsStatements) deleteQueueTransaction(
+	ctx context.Context, txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	transactionID gomatrixserverlib.TransactionID,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteQueueTransactionPDUsStmt)
+	_, err := stmt.ExecContext(ctx, serverName, transactionID)
+	return err
+}
+
+func (s *queuePDUsStatements) selectQueueNextTransactionID(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) (gomatrixserverlib.TransactionID, error) {
+	var transactionID gomatrixserverlib.TransactionID
+	stmt := sqlutil.TxStmt(txn, s.selectQueueNextTransactionIDStmt)
+	err := stmt.QueryRowContext(ctx, serverName).Scan(&transactionID)
+	if err == sql.ErrNoRows {
+		return "", nil
+	}
+	return transactionID, err
+}
+
+func (s *queuePDUsStatements) selectQueueReferenceJSONCount(
+	ctx context.Context, txn *sql.Tx, jsonNID int64,
+) (int64, error) {
+	var count int64
+	stmt := sqlutil.TxStmt(txn, s.selectQueueReferenceJSONCountStmt)
+	err := stmt.QueryRowContext(ctx, jsonNID).Scan(&count)
+	if err == sql.ErrNoRows {
+		// It's acceptable for there to be no rows referencing a given
+		// JSON NID but it's not an error condition. Just return as if
+		// there's a zero count.
+		return 0, nil
+	}
+	return count, err
+}
+
+func (s *queuePDUsStatements) selectQueuePDUs(
+	ctx context.Context, txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	transactionID gomatrixserverlib.TransactionID,
+	limit int,
+) ([]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsByTransactionStmt)
+	rows, err := stmt.QueryContext(ctx, serverName, transactionID, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "queueFromStmt: rows.close() failed")
+	var result []int64
+	for rows.Next() {
+		var nid int64
+		if err = rows.Scan(&nid); err != nil {
+			return nil, err
+		}
+		result = append(result, nid)
+	}
+
+	return result, rows.Err()
+}
diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go
index 8fd4c11a3..18d1532a4 100644
--- a/federationsender/storage/postgres/storage.go
+++ b/federationsender/storage/postgres/storage.go
@@ -18,15 +18,20 @@ package postgres
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
+	"fmt"
 
 	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // Database stores information needed by the federation sender
 type Database struct {
 	joinedHostsStatements
 	roomStatements
+	queuePDUsStatements
+	queueJSONStatements
 	sqlutil.PartitionOffsetStatements
 	db *sql.DB
 }
@@ -55,6 +60,14 @@ func (d *Database) prepare() error {
 		return err
 	}
 
+	if err = d.queuePDUsStatements.prepare(d.db); err != nil {
+		return err
+	}
+
+	if err = d.queueJSONStatements.prepare(d.db); err != nil {
+		return err
+	}
+
 	return d.PartitionOffsetStatements.Prepare(d.db, "federationsender")
 }
 
@@ -120,3 +133,125 @@ func (d *Database) GetJoinedHosts(
 ) ([]types.JoinedHost, error) {
 	return d.selectJoinedHosts(ctx, roomID)
 }
+
+// StoreJSON adds a JSON blob into the queue JSON table and returns
+// a NID. The NID will then be used when inserting the per-destination
+// metadata entries.
+func (d *Database) StoreJSON(
+	ctx context.Context, js string,
+) (int64, error) {
+	nid, err := d.insertQueueJSON(ctx, nil, js)
+	if err != nil {
+		return 0, fmt.Errorf("d.insertQueueJSON: %w", err)
+	}
+	return nid, nil
+}
+
+// AssociatePDUWithDestination creates an association that the
+// destination queues will use to determine which JSON blobs to send
+// to which servers.
+func (d *Database) AssociatePDUWithDestination(
+	ctx context.Context,
+	transactionID gomatrixserverlib.TransactionID,
+	serverName gomatrixserverlib.ServerName,
+	nids []int64,
+) error {
+	return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
+		for _, nid := range nids {
+			if err := d.insertQueuePDU(
+				ctx,           // context
+				txn,           // SQL transaction
+				transactionID, // transaction ID
+				serverName,    // destination server name
+				nid,           // NID from the federationsender_queue_json table
+			); err != nil {
+				return fmt.Errorf("d.insertQueueRetryStmt.ExecContext: %w", err)
+			}
+		}
+		return nil
+	})
+}
+
+// GetNextTransactionPDUs retrieves events from the database for
+// the next pending transaction, up to the limit specified.
+func (d *Database) GetNextTransactionPDUs(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	limit int,
+) (
+	transactionID gomatrixserverlib.TransactionID,
+	events []*gomatrixserverlib.HeaderedEvent,
+	err error,
+) {
+	err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
+		transactionID, err = d.selectQueueNextTransactionID(ctx, txn, serverName)
+		if err != nil {
+			return fmt.Errorf("d.selectQueueNextTransactionID: %w", err)
+		}
+
+		if transactionID == "" {
+			return nil
+		}
+
+		nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, limit)
+		if err != nil {
+			return fmt.Errorf("d.selectQueuePDUs: %w", err)
+		}
+
+		blobs, err := d.selectQueueJSON(ctx, txn, nids)
+		if err != nil {
+			return fmt.Errorf("d.selectJSON: %w", err)
+		}
+
+		for _, blob := range blobs {
+			var event gomatrixserverlib.HeaderedEvent
+			if err := json.Unmarshal(blob, &event); err != nil {
+				return fmt.Errorf("json.Unmarshal: %w", err)
+			}
+			events = append(events, &event)
+		}
+
+		return nil
+	})
+	return
+}
+
+// CleanTransactionPDUs cleans up all associated events for a
+// given transaction. This is done when the transaction was sent
+// successfully.
+func (d *Database) CleanTransactionPDUs(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	transactionID gomatrixserverlib.TransactionID,
+) error {
+	return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
+		nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, 50)
+		if err != nil {
+			return fmt.Errorf("d.selectQueuePDUs: %w", err)
+		}
+
+		if err = d.deleteQueueTransaction(ctx, txn, serverName, transactionID); err != nil {
+			return fmt.Errorf("d.deleteQueueTransaction: %w", err)
+		}
+
+		var count int64
+		var deleteNIDs []int64
+		for _, nid := range nids {
+			count, err = d.selectQueueReferenceJSONCount(ctx, txn, nid)
+			if err != nil {
+				return fmt.Errorf("d.selectQueueReferenceJSONCount: %w", err)
+			}
+			if count == 0 {
+				deleteNIDs = append(deleteNIDs, nid)
+			}
+		}
+
+		if len(deleteNIDs) > 0 {
+			if err = d.deleteQueueJSON(ctx, txn, deleteNIDs); err != nil {
+				return fmt.Errorf("d.deleteQueueJSON: %w", err)
+			}
+		}
+
+		return nil
+	})
+}
diff --git a/federationsender/storage/sqlite3/queue_json_table.go b/federationsender/storage/sqlite3/queue_json_table.go
new file mode 100644
index 000000000..01b7160db
--- /dev/null
+++ b/federationsender/storage/sqlite3/queue_json_table.go
@@ -0,0 +1,132 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+const queueJSONSchema = `
+-- The queue_retry_json table contains event contents that
+-- we failed to send. 
+CREATE TABLE IF NOT EXISTS federationsender_queue_json (
+	-- The JSON NID. This allows the federationsender_queue_retry table to
+	-- cross-reference to find the JSON blob.
+	json_nid INTEGER PRIMARY KEY AUTOINCREMENT,
+	-- The JSON body. Text so that we preserve UTF-8.
+	json_body TEXT NOT NULL
+);
+`
+
+const insertJSONSQL = "" +
+	"INSERT INTO federationsender_queue_json (json_body)" +
+	" VALUES ($1)"
+
+const deleteJSONSQL = "" +
+	"DELETE FROM federationsender_queue_json WHERE json_nid IN ($1)"
+
+const selectJSONSQL = "" +
+	"SELECT json_nid, json_body FROM federationsender_queue_json" +
+	" WHERE json_nid IN ($1)"
+
+type queueJSONStatements struct {
+	insertJSONStmt *sql.Stmt
+	//deleteJSONStmt *sql.Stmt - prepared at runtime due to variadic
+	//selectJSONStmt *sql.Stmt - prepared at runtime due to variadic
+}
+
+func (s *queueJSONStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(queueJSONSchema)
+	if err != nil {
+		return
+	}
+	if s.insertJSONStmt, err = db.Prepare(insertJSONSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *queueJSONStatements) insertQueueJSON(
+	ctx context.Context, txn *sql.Tx, json string,
+) (int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.insertJSONStmt)
+	res, err := stmt.ExecContext(ctx, json)
+	if err != nil {
+		return 0, fmt.Errorf("stmt.QueryContext: %w", err)
+	}
+	lastid, err := res.LastInsertId()
+	if err != nil {
+		return 0, fmt.Errorf("res.LastInsertId: %w", err)
+	}
+	return lastid, nil
+}
+
+func (s *queueJSONStatements) deleteQueueJSON(
+	ctx context.Context, txn *sql.Tx, nids []int64,
+) error {
+	deleteSQL := strings.Replace(deleteJSONSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1)
+	deleteStmt, err := txn.Prepare(deleteSQL)
+	if err != nil {
+		return fmt.Errorf("s.deleteQueueJSON s.db.Prepare: %w", err)
+	}
+
+	iNIDs := make([]interface{}, len(nids))
+	for k, v := range nids {
+		iNIDs[k] = v
+	}
+
+	stmt := sqlutil.TxStmt(txn, deleteStmt)
+	_, err = stmt.ExecContext(ctx, iNIDs...)
+	return err
+}
+
+func (s *queueJSONStatements) selectQueueJSON(
+	ctx context.Context, txn *sql.Tx, jsonNIDs []int64,
+) (map[int64][]byte, error) {
+	selectSQL := strings.Replace(selectJSONSQL, "($1)", sqlutil.QueryVariadic(len(jsonNIDs)), 1)
+	selectStmt, err := txn.Prepare(selectSQL)
+	if err != nil {
+		return nil, fmt.Errorf("s.selectQueueJSON s.db.Prepare: %w", err)
+	}
+
+	iNIDs := make([]interface{}, len(jsonNIDs))
+	for k, v := range jsonNIDs {
+		iNIDs[k] = v
+	}
+
+	blobs := map[int64][]byte{}
+	stmt := sqlutil.TxStmt(txn, selectStmt)
+	rows, err := stmt.QueryContext(ctx, iNIDs...)
+	if err != nil {
+		return nil, fmt.Errorf("s.selectQueueJSON stmt.QueryContext: %w", err)
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectJSON: rows.close() failed")
+	for rows.Next() {
+		var nid int64
+		var blob []byte
+		if err = rows.Scan(&nid, &blob); err != nil {
+			return nil, fmt.Errorf("s.selectQueueJSON rows.Scan: %w", err)
+		}
+		blobs[nid] = blob
+	}
+	return blobs, err
+}
diff --git a/federationsender/storage/sqlite3/queue_pdus_table.go b/federationsender/storage/sqlite3/queue_pdus_table.go
new file mode 100644
index 000000000..dc08fd707
--- /dev/null
+++ b/federationsender/storage/sqlite3/queue_pdus_table.go
@@ -0,0 +1,167 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const queuePDUsSchema = `
+CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
+    -- The transaction ID that was generated before persisting the event.
+	transaction_id TEXT NOT NULL,
+    -- The domain part of the user ID the m.room.member event is for.
+	server_name TEXT NOT NULL,
+	-- The JSON NID from the federationsender_queue_pdus_json table.
+	json_nid BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
+    ON federationsender_queue_pdus (json_nid, server_name);
+`
+
+const insertQueuePDUSQL = "" +
+	"INSERT INTO federationsender_queue_pdus (transaction_id, server_name, json_nid)" +
+	" VALUES ($1, $2, $3)"
+
+const deleteQueueTransactionPDUsSQL = "" +
+	"DELETE FROM federationsender_queue_pdus WHERE server_name = $1 AND transaction_id = $2"
+
+const selectQueueNextTransactionIDSQL = "" +
+	"SELECT transaction_id FROM federationsender_queue_pdus" +
+	" WHERE server_name = $1" +
+	" ORDER BY transaction_id ASC" +
+	" LIMIT 1"
+
+const selectQueuePDUsByTransactionSQL = "" +
+	"SELECT json_nid FROM federationsender_queue_pdus" +
+	" WHERE server_name = $1 AND transaction_id = $2" +
+	" LIMIT $3"
+
+const selectQueueReferenceJSONCountSQL = "" +
+	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
+	" WHERE json_nid = $1"
+
+type queuePDUsStatements struct {
+	insertQueuePDUStmt                *sql.Stmt
+	deleteQueueTransactionPDUsStmt    *sql.Stmt
+	selectQueueNextTransactionIDStmt  *sql.Stmt
+	selectQueuePDUsByTransactionStmt  *sql.Stmt
+	selectQueueReferenceJSONCountStmt *sql.Stmt
+}
+
+func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(queuePDUsSchema)
+	if err != nil {
+		return
+	}
+	if s.insertQueuePDUStmt, err = db.Prepare(insertQueuePDUSQL); err != nil {
+		return
+	}
+	if s.deleteQueueTransactionPDUsStmt, err = db.Prepare(deleteQueueTransactionPDUsSQL); err != nil {
+		return
+	}
+	if s.selectQueueNextTransactionIDStmt, err = db.Prepare(selectQueueNextTransactionIDSQL); err != nil {
+		return
+	}
+	if s.selectQueuePDUsByTransactionStmt, err = db.Prepare(selectQueuePDUsByTransactionSQL); err != nil {
+		return
+	}
+	if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueueReferenceJSONCountSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *queuePDUsStatements) insertQueuePDU(
+	ctx context.Context,
+	txn *sql.Tx,
+	transactionID gomatrixserverlib.TransactionID,
+	serverName gomatrixserverlib.ServerName,
+	nid int64,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.insertQueuePDUStmt)
+	_, err := stmt.ExecContext(
+		ctx,
+		transactionID, // the transaction ID that we initially attempted
+		serverName,    // destination server name
+		nid,           // JSON blob NID
+	)
+	return err
+}
+
+func (s *queuePDUsStatements) deleteQueueTransaction(
+	ctx context.Context, txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	transactionID gomatrixserverlib.TransactionID,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteQueueTransactionPDUsStmt)
+	_, err := stmt.ExecContext(ctx, serverName, transactionID)
+	return err
+}
+
+func (s *queuePDUsStatements) selectQueueNextTransactionID(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) (gomatrixserverlib.TransactionID, error) {
+	var transactionID gomatrixserverlib.TransactionID
+	stmt := sqlutil.TxStmt(txn, s.selectQueueNextTransactionIDStmt)
+	err := stmt.QueryRowContext(ctx, serverName).Scan(&transactionID)
+	if err == sql.ErrNoRows {
+		return "", nil
+	}
+	return transactionID, err
+}
+
+func (s *queuePDUsStatements) selectQueueReferenceJSONCount(
+	ctx context.Context, txn *sql.Tx, jsonNID int64,
+) (int64, error) {
+	var count int64
+	stmt := sqlutil.TxStmt(txn, s.selectQueueReferenceJSONCountStmt)
+	err := stmt.QueryRowContext(ctx, jsonNID).Scan(&count)
+	if err == sql.ErrNoRows {
+		return -1, nil
+	}
+	return count, err
+}
+
+func (s *queuePDUsStatements) selectQueuePDUs(
+	ctx context.Context, txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	transactionID gomatrixserverlib.TransactionID,
+	limit int,
+) ([]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsByTransactionStmt)
+	rows, err := stmt.QueryContext(ctx, serverName, transactionID, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "queueFromStmt: rows.close() failed")
+	var result []int64
+	for rows.Next() {
+		var nid int64
+		if err = rows.Scan(&nid); err != nil {
+			return nil, err
+		}
+		result = append(result, nid)
+	}
+
+	return result, rows.Err()
+}
diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go
index ac303f646..7629ecd21 100644
--- a/federationsender/storage/sqlite3/storage.go
+++ b/federationsender/storage/sqlite3/storage.go
@@ -18,17 +18,22 @@ package sqlite3
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
+	"fmt"
 
 	_ "github.com/mattn/go-sqlite3"
 
 	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // Database stores information needed by the federation sender
 type Database struct {
 	joinedHostsStatements
 	roomStatements
+	queuePDUsStatements
+	queueJSONStatements
 	sqlutil.PartitionOffsetStatements
 	db *sql.DB
 }
@@ -61,6 +66,14 @@ func (d *Database) prepare() error {
 		return err
 	}
 
+	if err = d.queuePDUsStatements.prepare(d.db); err != nil {
+		return err
+	}
+
+	if err = d.queueJSONStatements.prepare(d.db); err != nil {
+		return err
+	}
+
 	return d.PartitionOffsetStatements.Prepare(d.db, "federationsender")
 }
 
@@ -126,3 +139,125 @@ func (d *Database) GetJoinedHosts(
 ) ([]types.JoinedHost, error) {
 	return d.selectJoinedHosts(ctx, roomID)
 }
+
+// StoreJSON adds a JSON blob into the queue JSON table and returns
+// a NID. The NID will then be used when inserting the per-destination
+// metadata entries.
+func (d *Database) StoreJSON(
+	ctx context.Context, js string,
+) (int64, error) {
+	nid, err := d.insertQueueJSON(ctx, nil, js)
+	if err != nil {
+		return 0, fmt.Errorf("d.insertQueueJSON: %w", err)
+	}
+	return nid, nil
+}
+
+// AssociatePDUWithDestination creates an association that the
+// destination queues will use to determine which JSON blobs to send
+// to which servers.
+func (d *Database) AssociatePDUWithDestination(
+	ctx context.Context,
+	transactionID gomatrixserverlib.TransactionID,
+	serverName gomatrixserverlib.ServerName,
+	nids []int64,
+) error {
+	return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
+		for _, nid := range nids {
+			if err := d.insertQueuePDU(
+				ctx,           // context
+				txn,           // SQL transaction
+				transactionID, // transaction ID
+				serverName,    // destination server name
+				nid,           // NID from the federationsender_queue_json table
+			); err != nil {
+				return fmt.Errorf("d.insertQueueRetryStmt.ExecContext: %w", err)
+			}
+		}
+		return nil
+	})
+}
+
+// GetNextTransactionPDUs retrieves events from the database for
+// the next pending transaction, up to the limit specified.
+func (d *Database) GetNextTransactionPDUs(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	limit int,
+) (
+	transactionID gomatrixserverlib.TransactionID,
+	events []*gomatrixserverlib.HeaderedEvent,
+	err error,
+) {
+	err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
+		transactionID, err = d.selectQueueNextTransactionID(ctx, txn, serverName)
+		if err != nil {
+			return fmt.Errorf("d.selectQueueNextTransactionID: %w", err)
+		}
+
+		if transactionID == "" {
+			return nil
+		}
+
+		nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, limit)
+		if err != nil {
+			return fmt.Errorf("d.selectQueuePDUs: %w", err)
+		}
+
+		blobs, err := d.selectQueueJSON(ctx, txn, nids)
+		if err != nil {
+			return fmt.Errorf("d.selectJSON: %w", err)
+		}
+
+		for _, blob := range blobs {
+			var event gomatrixserverlib.HeaderedEvent
+			if err := json.Unmarshal(blob, &event); err != nil {
+				return fmt.Errorf("json.Unmarshal: %w", err)
+			}
+			events = append(events, &event)
+		}
+
+		return nil
+	})
+	return
+}
+
+// CleanTransactionPDUs cleans up all associated events for a
+// given transaction. This is done when the transaction was sent
+// successfully.
+func (d *Database) CleanTransactionPDUs(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	transactionID gomatrixserverlib.TransactionID,
+) error {
+	return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
+		nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, 50)
+		if err != nil {
+			return fmt.Errorf("d.selectQueuePDUs: %w", err)
+		}
+
+		if err = d.deleteQueueTransaction(ctx, txn, serverName, transactionID); err != nil {
+			return fmt.Errorf("d.deleteQueueTransaction: %w", err)
+		}
+
+		var count int64
+		var deleteNIDs []int64
+		for _, nid := range nids {
+			count, err = d.selectQueueReferenceJSONCount(ctx, txn, nid)
+			if err != nil {
+				return fmt.Errorf("d.selectQueueReferenceJSONCount: %w", err)
+			}
+			if count == 0 {
+				deleteNIDs = append(deleteNIDs, nid)
+			}
+		}
+
+		if len(deleteNIDs) > 0 {
+			if err = d.deleteQueueJSON(ctx, txn, deleteNIDs); err != nil {
+				return fmt.Errorf("d.deleteQueueJSON: %w", err)
+			}
+		}
+
+		return nil
+	})
+}

From 8e7947926cd00a05f90a9d0bfdc29dcd2dc9ab7a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 1 Jul 2020 13:35:58 +0100
Subject: [PATCH 18/23] Fix Yggdrasil gobind build, set display name at
 registration

---
 build/gobind/monolith.go                    | 11 +++++++++++
 cmd/dendrite-demo-yggdrasil/yggconn/node.go |  4 ++++
 userapi/internal/api.go                     |  5 +++++
 3 files changed, 20 insertions(+)

diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go
index 750babad8..62ca17447 100644
--- a/build/gobind/monolith.go
+++ b/build/gobind/monolith.go
@@ -11,6 +11,7 @@ import (
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
+	"github.com/matrix-org/dendrite/currentstateserver"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
@@ -25,6 +26,7 @@ import (
 )
 
 type DendriteMonolith struct {
+	YggdrasilNode    *yggconn.Node
 	StorageDirectory string
 	listener         net.Listener
 }
@@ -33,6 +35,10 @@ func (m *DendriteMonolith) BaseURL() string {
 	return fmt.Sprintf("http://%s", m.listener.Addr().String())
 }
 
+func (m *DendriteMonolith) PeerCount() int {
+	return m.YggdrasilNode.PeerCount()
+}
+
 func (m *DendriteMonolith) Start() {
 	logger := logrus.Logger{
 		Out: BindLogger{},
@@ -49,6 +55,7 @@ func (m *DendriteMonolith) Start() {
 	if err != nil {
 		panic(err)
 	}
+	m.YggdrasilNode = ygg
 
 	cfg := &config.Dendrite{}
 	cfg.SetDefaults()
@@ -69,6 +76,7 @@ func (m *DendriteMonolith) Start() {
 	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory))
 	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory))
 	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory))
+	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s/dendrite-currentstate.db", m.StorageDirectory))
 	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
@@ -108,6 +116,8 @@ func (m *DendriteMonolith) Start() {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
 
+	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+
 	monolith := setup.Monolith{
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
@@ -123,6 +133,7 @@ func (m *DendriteMonolith) Start() {
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
 		UserAPI:             userAPI,
+		StateAPI:            stateAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 
 		PublicRoomsDB: publicRoomsDB,
diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
index c335f2eac..73e9cd5b3 100644
--- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go
+++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
@@ -174,3 +174,7 @@ func (n *Node) SigningPrivateKey() ed25519.PrivateKey {
 	privBytes, _ := hex.DecodeString(n.config.SigningPrivateKey)
 	return ed25519.PrivateKey(privBytes)
 }
+
+func (n *Node) PeerCount() int {
+	return len(n.core.GetSwitchPeers())
+}
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index b081eca49..1d10d1d8b 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -85,6 +85,11 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P
 		}
 		return nil
 	}
+
+	if err = a.AccountDB.SetDisplayName(ctx, req.Localpart, req.Localpart); err != nil {
+		return err
+	}
+
 	res.AccountCreated = true
 	res.Account = acc
 	return nil

From 55bc82c439057f379361871c863aa9611d70fce2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 1 Jul 2020 13:47:16 +0100
Subject: [PATCH 19/23] Update Yggdrasil demo peer count

---
 cmd/dendrite-demo-yggdrasil/yggconn/node.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
index 73e9cd5b3..18d207a9e 100644
--- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go
+++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
@@ -176,5 +176,5 @@ func (n *Node) SigningPrivateKey() ed25519.PrivateKey {
 }
 
 func (n *Node) PeerCount() int {
-	return len(n.core.GetSwitchPeers())
+	return len(n.core.GetPeers()) - 1
 }

From 4c1e6597c0ea82f5390b73f35036db58e65542cc Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Thu, 2 Jul 2020 15:41:18 +0100
Subject: [PATCH 20/23] Replace publicroomsapi with a combination of
 clientapi/roomserver/currentstateserver (#1174)

* Use content_value instead of membership

* Fix build

* Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver

- All public rooms paths are now handled by clientapi
- Requests to (un)publish rooms are sent to the roomserver via `PerformPublish`
  which are stored in a new `published_table.go`
- Requests for public rooms are handled in clientapi by:
    * Fetch all room IDs which are published using `QueryPublishedRooms` on the roomserver.
    * Apply pagination parameters to the slice.
    * Do a `QueryBulkStateContent` request to the currentstateserver to pull out
      required state event *content* (not entire events).
    * Aggregate and return the chunk.

Mostly but not fully implemented (DB queries on currentstateserver are missing)

* Fix pq query

* Make postgres work

* Make sqlite work

* Fix tests

* Unbreak pagination tests

* Linting
---
 clientapi/routing/createroom.go               |  13 +
 clientapi/routing/directory.go                |  90 ++++-
 clientapi/routing/directory_public.go         | 373 ++++++++++++++++++
 clientapi/routing/directory_public_test.go    |  48 +++
 clientapi/routing/membership.go               |  33 ++
 clientapi/routing/routing.go                  |  31 +-
 cmd/dendrite-federation-api-server/main.go    |   2 +-
 cmd/dendritejs/main.go                        |   1 +
 currentstateserver/api/api.go                 |  26 ++
 currentstateserver/internal/api.go            |  20 +
 currentstateserver/inthttp/client.go          |  17 +-
 currentstateserver/inthttp/server.go          |  13 +
 currentstateserver/storage/interface.go       |   4 +
 .../postgres/current_room_state_table.go      |  92 ++++-
 currentstateserver/storage/shared/storage.go  |  26 +-
 .../sqlite3/current_room_state_table.go       | 109 ++++-
 .../storage/tables/interface.go               |  65 ++-
 federationapi/federationapi.go                |   4 +-
 federationapi/federationapi_test.go           |   2 +-
 federationapi/routing/publicrooms.go          | 178 +++++++++
 federationapi/routing/routing.go              |   8 +
 federationapi/routing/send_test.go            |  15 +
 go.mod                                        |   2 +-
 go.sum                                        |   2 +
 internal/setup/monolith.go                    |  12 +-
 roomserver/api/api.go                         |  12 +
 roomserver/api/api_trace.go                   |  19 +
 roomserver/api/perform.go                     |  10 +
 roomserver/api/query.go                       |  10 +
 roomserver/internal/perform_publish.go        |  20 +
 roomserver/internal/query.go                  |  13 +
 roomserver/inthttp/client.go                  |  31 ++
 roomserver/inthttp/server.go                  |  25 ++
 roomserver/storage/interface.go               |   4 +
 .../storage/postgres/published_table.go       | 101 +++++
 roomserver/storage/postgres/storage.go        |   5 +
 roomserver/storage/shared/storage.go          |   9 +
 roomserver/storage/sqlite3/published_table.go | 100 +++++
 roomserver/storage/sqlite3/storage.go         |   5 +
 roomserver/storage/tables/interface.go        |   6 +
 sytest-whitelist                              |   4 +
 41 files changed, 1481 insertions(+), 79 deletions(-)
 create mode 100644 clientapi/routing/directory_public.go
 create mode 100644 clientapi/routing/directory_public_test.go
 create mode 100644 federationapi/routing/publicrooms.go
 create mode 100644 roomserver/internal/perform_publish.go
 create mode 100644 roomserver/storage/postgres/published_table.go
 create mode 100644 roomserver/storage/sqlite3/published_table.go

diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index 42e1895ce..b6a5d1221 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -410,6 +410,19 @@ func createRoom(
 		}
 	}
 
+	if r.Visibility == "public" {
+		// expose this room in the published room list
+		var pubRes roomserverAPI.PerformPublishResponse
+		rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
+			RoomID:     roomID,
+			Visibility: "public",
+		}, &pubRes)
+		if pubRes.Error != nil {
+			// treat as non-fatal since the room is already made by this point
+			util.GetLogger(req.Context()).WithError(pubRes.Error).Error("failed to visibility:public")
+		}
+	}
+
 	response := createRoomResponse{
 		RoomID:    roomID,
 		RoomAlias: roomAlias,
diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go
index 0dc4d5605..0f78f4a24 100644
--- a/clientapi/routing/directory.go
+++ b/clientapi/routing/directory.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,10 +20,12 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -232,3 +234,89 @@ func RemoveLocalAlias(
 		JSON: struct{}{},
 	}
 }
+
+type roomVisibility struct {
+	Visibility string `json:"visibility"`
+}
+
+// GetVisibility implements GET /directory/list/room/{roomID}
+func GetVisibility(
+	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
+	roomID string,
+) util.JSONResponse {
+	var res roomserverAPI.QueryPublishedRoomsResponse
+	err := rsAPI.QueryPublishedRooms(req.Context(), &roomserverAPI.QueryPublishedRoomsRequest{
+		RoomID: roomID,
+	}, &res)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("QueryPublishedRooms failed")
+		return jsonerror.InternalServerError()
+	}
+
+	var v roomVisibility
+	if len(res.RoomIDs) == 1 {
+		v.Visibility = gomatrixserverlib.Public
+	} else {
+		v.Visibility = "private"
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: v,
+	}
+}
+
+// SetVisibility implements PUT /directory/list/room/{roomID}
+// TODO: Allow admin users to edit the room visibility
+func SetVisibility(
+	req *http.Request, stateAPI currentstateAPI.CurrentStateInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, dev *userapi.Device,
+	roomID string,
+) util.JSONResponse {
+	resErr := checkMemberInRoom(req.Context(), stateAPI, dev.UserID, roomID)
+	if resErr != nil {
+		return *resErr
+	}
+
+	queryEventsReq := roomserverAPI.QueryLatestEventsAndStateRequest{
+		RoomID: roomID,
+		StateToFetch: []gomatrixserverlib.StateKeyTuple{{
+			EventType: gomatrixserverlib.MRoomPowerLevels,
+			StateKey:  "",
+		}},
+	}
+	var queryEventsRes roomserverAPI.QueryLatestEventsAndStateResponse
+	err := rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
+	if err != nil || len(queryEventsRes.StateEvents) == 0 {
+		util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
+		return jsonerror.InternalServerError()
+	}
+
+	// NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event
+	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
+	if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),
+		}
+	}
+
+	var v roomVisibility
+	if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil {
+		return *reqErr
+	}
+
+	var publishRes roomserverAPI.PerformPublishResponse
+	rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
+		RoomID:     roomID,
+		Visibility: v.Visibility,
+	}, &publishRes)
+	if publishRes.Error != nil {
+		util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
+		return publishRes.Error.JSONResponse()
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
diff --git a/clientapi/routing/directory_public.go b/clientapi/routing/directory_public.go
new file mode 100644
index 000000000..6d0db579b
--- /dev/null
+++ b/clientapi/routing/directory_public.go
@@ -0,0 +1,373 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"context"
+	"math/rand"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
+	"github.com/matrix-org/dendrite/publicroomsapi/types"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type PublicRoomReq struct {
+	Since  string `json:"since,omitempty"`
+	Limit  int16  `json:"limit,omitempty"`
+	Filter filter `json:"filter,omitempty"`
+}
+
+type filter struct {
+	SearchTerms string `json:"generic_search_term,omitempty"`
+}
+
+// GetPostPublicRooms implements GET and POST /publicRooms
+func GetPostPublicRooms(
+	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI,
+) util.JSONResponse {
+	var request PublicRoomReq
+	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
+		return *fillErr
+	}
+	response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: response,
+	}
+}
+
+// GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied.
+func GetPostPublicRoomsWithExternal(
+	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI,
+	fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider,
+) util.JSONResponse {
+	var request PublicRoomReq
+	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
+		return *fillErr
+	}
+	response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+
+	if request.Since != "" {
+		// TODO: handle pagination tokens sensibly rather than ignoring them.
+		// ignore paginated requests since we don't handle them yet over federation.
+		// Only the initial request will contain federated rooms.
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: response,
+		}
+	}
+
+	// If we have already hit the limit on the number of rooms, bail.
+	var limit int
+	if request.Limit > 0 {
+		limit = int(request.Limit) - len(response.Chunk)
+		if limit <= 0 {
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: response,
+			}
+		}
+	}
+
+	// downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16
+	fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit))
+	response.Chunk = append(response.Chunk, fedRooms...)
+
+	// de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers
+	// are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit)
+	var publicRooms []gomatrixserverlib.PublicRoom
+	haveRoomIDs := make(map[string]bool)
+	rand.Shuffle(len(response.Chunk), func(i, j int) {
+		response.Chunk[i], response.Chunk[j] = response.Chunk[j], response.Chunk[i]
+	})
+	for _, r := range response.Chunk {
+		if haveRoomIDs[r.RoomID] {
+			continue
+		}
+		haveRoomIDs[r.RoomID] = true
+		publicRooms = append(publicRooms, r)
+	}
+	// sort by member count
+	sort.SliceStable(publicRooms, func(i, j int) bool {
+		return publicRooms[i].JoinedMembersCount > publicRooms[j].JoinedMembersCount
+	})
+
+	response.Chunk = publicRooms
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: response,
+	}
+}
+
+// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
+// Returns a list of public rooms up to the limit specified.
+func bulkFetchPublicRoomsFromServers(
+	ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string, limit int16,
+) (publicRooms []gomatrixserverlib.PublicRoom) {
+	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
+	// goroutines send rooms to this channel
+	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
+	// signalling channel to tell goroutines to stop sending rooms and quit
+	done := make(chan bool)
+	// signalling to say when we can close the room channel
+	var wg sync.WaitGroup
+	wg.Add(len(homeservers))
+	// concurrently query for public rooms
+	for _, hs := range homeservers {
+		go func(homeserverDomain string) {
+			defer wg.Done()
+			util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
+			fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "")
+			if err != nil {
+				util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn(
+					"bulkFetchPublicRoomsFromServers: failed to query hs",
+				)
+				return
+			}
+			for _, room := range fres.Chunk {
+				// atomically send a room or stop
+				select {
+				case roomCh <- room:
+				case <-done:
+					util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms")
+					return
+				}
+			}
+		}(hs)
+	}
+
+	// Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request.
+	// This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be
+	// closed.
+	go func() {
+		wg.Wait()
+		util.GetLogger(ctx).Info("Cleaning up resources")
+		close(roomCh)
+	}()
+
+	// fan-in results with timeout. We stop when we reach the limit.
+FanIn:
+	for len(publicRooms) < int(limit) || limit == 0 {
+		// add a room or timeout
+		select {
+		case room, ok := <-roomCh:
+			if !ok {
+				util.GetLogger(ctx).Info("All homeservers have been queried, returning results.")
+				break FanIn
+			}
+			publicRooms = append(publicRooms, room)
+		case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got.
+			util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early")
+			break FanIn
+		case <-ctx.Done(): // the client hung up on us, let's stop.
+			util.GetLogger(ctx).Info("Client hung up, returning early")
+			break FanIn
+		}
+	}
+	// tell goroutines to stop
+	close(done)
+
+	return publicRooms
+}
+
+func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) {
+
+	var response gomatrixserverlib.RespPublicRooms
+	var limit int16
+	var offset int64
+	limit = request.Limit
+	if limit == 0 {
+		limit = 50
+	}
+	offset, err := strconv.ParseInt(request.Since, 10, 64)
+	// ParseInt returns 0 and an error when trying to parse an empty string
+	// In that case, we want to assign 0 so we ignore the error
+	if err != nil && len(request.Since) > 0 {
+		util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed")
+		return nil, err
+	}
+
+	var queryRes roomserverAPI.QueryPublishedRoomsResponse
+	err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
+		return nil, err
+	}
+	response.TotalRoomCountEstimate = len(queryRes.RoomIDs)
+
+	roomIDs, prev, next := sliceInto(queryRes.RoomIDs, offset, limit)
+	if prev >= 0 {
+		response.PrevBatch = "T" + strconv.Itoa(prev)
+	}
+	if next >= 0 {
+		response.NextBatch = "T" + strconv.Itoa(next)
+	}
+	response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI)
+	return &response, err
+}
+
+// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request
+// on /publicRooms by parsing the incoming HTTP request
+// Filter is only filled for POST requests
+func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse {
+	if httpReq.Method != "GET" && httpReq.Method != "POST" {
+		return &util.JSONResponse{
+			Code: http.StatusMethodNotAllowed,
+			JSON: jsonerror.NotFound("Bad method"),
+		}
+	}
+	if httpReq.Method == "GET" {
+		limit, err := strconv.Atoi(httpReq.FormValue("limit"))
+		// Atoi returns 0 and an error when trying to parse an empty string
+		// In that case, we want to assign 0 so we ignore the error
+		if err != nil && len(httpReq.FormValue("limit")) > 0 {
+			util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
+			return &util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.BadJSON("limit param is not a number"),
+			}
+		}
+		request.Limit = int16(limit)
+		request.Since = httpReq.FormValue("since")
+	} else {
+		resErr := httputil.UnmarshalJSONRequest(httpReq, request)
+		if resErr != nil {
+			return resErr
+		}
+	}
+
+	// strip the 'T' which is only required because when sytest does pagination tests it stops
+	// iterating when !prev_batch which then fails if prev_batch==0, so add arbitrary text to
+	// make it truthy not falsey.
+	request.Since = strings.TrimPrefix(request.Since, "T")
+	return nil
+}
+
+// due to lots of switches
+// nolint:gocyclo
+func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI.CurrentStateInternalAPI) ([]gomatrixserverlib.PublicRoom, error) {
+	avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
+	nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
+	canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
+	topicTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.topic", StateKey: ""}
+	guestTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.guest_access", StateKey: ""}
+	visibilityTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""}
+	joinRuleTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}
+
+	var stateRes currentstateAPI.QueryBulkStateContentResponse
+	err := stateAPI.QueryBulkStateContent(ctx, &currentstateAPI.QueryBulkStateContentRequest{
+		RoomIDs:        roomIDs,
+		AllowWildcards: true,
+		StateTuples: []gomatrixserverlib.StateKeyTuple{
+			nameTuple, canonicalTuple, topicTuple, guestTuple, visibilityTuple, joinRuleTuple, avatarTuple,
+			{EventType: gomatrixserverlib.MRoomMember, StateKey: "*"},
+		},
+	}, &stateRes)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed")
+		return nil, err
+	}
+	chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs))
+	i := 0
+	for roomID, data := range stateRes.Rooms {
+		pub := gomatrixserverlib.PublicRoom{
+			RoomID: roomID,
+		}
+		joinCount := 0
+		var joinRule, guestAccess string
+		for tuple, contentVal := range data {
+			if tuple.EventType == gomatrixserverlib.MRoomMember && contentVal == "join" {
+				joinCount++
+				continue
+			}
+			switch tuple {
+			case avatarTuple:
+				pub.AvatarURL = contentVal
+			case nameTuple:
+				pub.Name = contentVal
+			case topicTuple:
+				pub.Topic = contentVal
+			case canonicalTuple:
+				pub.CanonicalAlias = contentVal
+			case visibilityTuple:
+				pub.WorldReadable = contentVal == "world_readable"
+			// need both of these to determine whether guests can join
+			case joinRuleTuple:
+				joinRule = contentVal
+			case guestTuple:
+				guestAccess = contentVal
+			}
+		}
+		if joinRule == gomatrixserverlib.Public && guestAccess == "can_join" {
+			pub.GuestCanJoin = true
+		}
+		pub.JoinedMembersCount = joinCount
+		chunk[i] = pub
+		i++
+	}
+	return chunk, nil
+}
+
+// sliceInto returns a subslice of `slice` which honours the since/limit values given.
+//
+//    0  1  2  3  4  5  6   index
+//   [A, B, C, D, E, F, G]  slice
+//
+//   limit=3          => A,B,C (prev='', next='3')
+//   limit=3&since=3  => D,E,F (prev='0', next='6')
+//   limit=3&since=6  => G     (prev='3', next='')
+//
+//  A value of '-1' for prev/next indicates no position.
+func sliceInto(slice []string, since int64, limit int16) (subset []string, prev, next int) {
+	prev = -1
+	next = -1
+
+	if since > 0 {
+		prev = int(since) - int(limit)
+	}
+	nextIndex := int(since) + int(limit)
+	if len(slice) > nextIndex { // there are more rooms ahead of us
+		next = nextIndex
+	}
+
+	// apply sanity caps
+	if since < 0 {
+		since = 0
+	}
+	if nextIndex > len(slice) {
+		nextIndex = len(slice)
+	}
+
+	subset = slice[since:nextIndex]
+	return
+}
diff --git a/clientapi/routing/directory_public_test.go b/clientapi/routing/directory_public_test.go
new file mode 100644
index 000000000..f2a1d5515
--- /dev/null
+++ b/clientapi/routing/directory_public_test.go
@@ -0,0 +1,48 @@
+package routing
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestSliceInto(t *testing.T) {
+	slice := []string{"a", "b", "c", "d", "e", "f", "g"}
+	limit := int16(3)
+	testCases := []struct {
+		since      int64
+		wantPrev   int
+		wantNext   int
+		wantSubset []string
+	}{
+		{
+			since:      0,
+			wantPrev:   -1,
+			wantNext:   3,
+			wantSubset: slice[0:3],
+		},
+		{
+			since:      3,
+			wantPrev:   0,
+			wantNext:   6,
+			wantSubset: slice[3:6],
+		},
+		{
+			since:      6,
+			wantPrev:   3,
+			wantNext:   -1,
+			wantSubset: slice[6:7],
+		},
+	}
+	for _, tc := range testCases {
+		subset, prev, next := sliceInto(slice, tc.since, limit)
+		if !reflect.DeepEqual(subset, tc.wantSubset) {
+			t.Errorf("returned subset is wrong, got %v want %v", subset, tc.wantSubset)
+		}
+		if prev != tc.wantPrev {
+			t.Errorf("returned prev is wrong, got %d want %d", prev, tc.wantPrev)
+		}
+		if next != tc.wantNext {
+			t.Errorf("returned next is wrong, got %d want %d", next, tc.wantNext)
+		}
+	}
+}
diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go
index 1f316384b..c2145159a 100644
--- a/clientapi/routing/membership.go
+++ b/clientapi/routing/membership.go
@@ -25,6 +25,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -358,3 +359,35 @@ func checkAndProcessThreepid(
 	}
 	return
 }
+
+func checkMemberInRoom(ctx context.Context, stateAPI currentstateAPI.CurrentStateInternalAPI, userID, roomID string) *util.JSONResponse {
+	tuple := gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomMember,
+		StateKey:  userID,
+	}
+	var membershipRes currentstateAPI.QueryCurrentStateResponse
+	err := stateAPI.QueryCurrentState(ctx, &currentstateAPI.QueryCurrentStateRequest{
+		RoomID:      roomID,
+		StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
+	}, &membershipRes)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("QueryCurrentState: could not query membership for user")
+		e := jsonerror.InternalServerError()
+		return &e
+	}
+	ev, ok := membershipRes.StateEvents[tuple]
+	if !ok {
+		return &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("user does not belong to room"),
+		}
+	}
+	membership, err := ev.Membership()
+	if err != nil || membership != "join" {
+		return &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("user does not belong to room"),
+		}
+	}
+	return nil
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index deaa7b329..57bb921d9 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,6 +31,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -290,6 +291,34 @@ func Setup(
 			return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
+	r0mux.Handle("/directory/list/room/{roomID}",
+		httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetVisibility(req, rsAPI, vars["roomID"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+	// TODO: Add AS support
+	r0mux.Handle("/directory/list/room/{roomID}",
+		httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SetVisibility(req, stateAPI, rsAPI, device, vars["roomID"])
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+	r0mux.Handle("/publicRooms",
+		httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse {
+			/* TODO:
+			if extRoomsProvider != nil {
+				return GetPostPublicRoomsWithExternal(req, stateAPI, fedClient, extRoomsProvider)
+			} */
+			return GetPostPublicRooms(req, rsAPI, stateAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/logout",
 		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go
index e3bf5edc8..1bde56368 100644
--- a/cmd/dendrite-federation-api-server/main.go
+++ b/cmd/dendrite-federation-api-server/main.go
@@ -33,7 +33,7 @@ func main() {
 
 	federationapi.AddPublicRoutes(
 		base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing,
-		rsAPI, fsAPI, base.EDUServerClient(),
+		rsAPI, fsAPI, base.EDUServerClient(), base.CurrentStateAPIClient(),
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))
diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go
index 11f339b0f..36ce9f65b 100644
--- a/cmd/dendritejs/main.go
+++ b/cmd/dendritejs/main.go
@@ -173,6 +173,7 @@ func main() {
 	cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db"
 	cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db"
 	cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db"
+	cfg.Database.CurrentState = "file:/idb/dendritejs_currentstate.db"
 	cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event"
 	cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event"
 	cfg.Kafka.Topics.OutputClientData = "output_client_data"
diff --git a/currentstateserver/api/api.go b/currentstateserver/api/api.go
index b16306ab0..729a66baf 100644
--- a/currentstateserver/api/api.go
+++ b/currentstateserver/api/api.go
@@ -29,6 +29,8 @@ type CurrentStateInternalAPI interface {
 	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
 	// QueryRoomsForUser retrieves a list of room IDs matching the given query.
 	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
+	// QueryBulkStateContent does a bulk query for state event content in the given rooms.
+	QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error
 }
 
 type QueryRoomsForUserRequest struct {
@@ -41,6 +43,30 @@ type QueryRoomsForUserResponse struct {
 	RoomIDs []string
 }
 
+type QueryBulkStateContentRequest struct {
+	// Returns state events in these rooms
+	RoomIDs []string
+	// If true, treats the '*' StateKey as "all state events of this type" rather than a literal value of '*'
+	AllowWildcards bool
+	// The state events to return. Only a small subset of tuples are allowed in this request as only certain events
+	// have their content fields extracted. Specifically, the tuple Type must be one of:
+	//   m.room.avatar
+	//   m.room.create
+	//   m.room.canonical_alias
+	//   m.room.guest_access
+	//   m.room.history_visibility
+	//   m.room.join_rules
+	//   m.room.member
+	//   m.room.name
+	//   m.room.topic
+	// Any other tuple type will result in the query failing.
+	StateTuples []gomatrixserverlib.StateKeyTuple
+}
+type QueryBulkStateContentResponse struct {
+	// map of room ID -> tuple -> content_value
+	Rooms map[string]map[gomatrixserverlib.StateKeyTuple]string
+}
+
 type QueryCurrentStateRequest struct {
 	RoomID      string
 	StateTuples []gomatrixserverlib.StateKeyTuple
diff --git a/currentstateserver/internal/api.go b/currentstateserver/internal/api.go
index 85fbf51ef..c28760477 100644
--- a/currentstateserver/internal/api.go
+++ b/currentstateserver/internal/api.go
@@ -48,3 +48,23 @@ func (a *CurrentStateInternalAPI) QueryRoomsForUser(ctx context.Context, req *ap
 	res.RoomIDs = roomIDs
 	return nil
 }
+
+func (a *CurrentStateInternalAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error {
+	events, err := a.DB.GetBulkStateContent(ctx, req.RoomIDs, req.StateTuples, req.AllowWildcards)
+	if err != nil {
+		return err
+	}
+	res.Rooms = make(map[string]map[gomatrixserverlib.StateKeyTuple]string)
+	for _, ev := range events {
+		if res.Rooms[ev.RoomID] == nil {
+			res.Rooms[ev.RoomID] = make(map[gomatrixserverlib.StateKeyTuple]string)
+		}
+		room := res.Rooms[ev.RoomID]
+		room[gomatrixserverlib.StateKeyTuple{
+			EventType: ev.EventType,
+			StateKey:  ev.StateKey,
+		}] = ev.ContentValue
+		res.Rooms[ev.RoomID] = room
+	}
+	return nil
+}
diff --git a/currentstateserver/inthttp/client.go b/currentstateserver/inthttp/client.go
index 6fd9907bd..b8c6a1198 100644
--- a/currentstateserver/inthttp/client.go
+++ b/currentstateserver/inthttp/client.go
@@ -26,8 +26,9 @@ import (
 
 // HTTP paths for the internal HTTP APIs
 const (
-	QueryCurrentStatePath = "/currentstateserver/queryCurrentState"
-	QueryRoomsForUserPath = "/currentstateserver/queryRoomsForUser"
+	QueryCurrentStatePath     = "/currentstateserver/queryCurrentState"
+	QueryRoomsForUserPath     = "/currentstateserver/queryRoomsForUser"
+	QueryBulkStateContentPath = "/currentstateserver/queryBulkStateContent"
 )
 
 // NewCurrentStateAPIClient creates a CurrentStateInternalAPI implemented by talking to a HTTP POST API.
@@ -73,3 +74,15 @@ func (h *httpCurrentStateInternalAPI) QueryRoomsForUser(
 	apiURL := h.apiURL + QueryRoomsForUserPath
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
+
+func (h *httpCurrentStateInternalAPI) QueryBulkStateContent(
+	ctx context.Context,
+	request *api.QueryBulkStateContentRequest,
+	response *api.QueryBulkStateContentResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBulkStateContent")
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryBulkStateContentPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
diff --git a/currentstateserver/inthttp/server.go b/currentstateserver/inthttp/server.go
index fa7ecb22e..dafb9f643 100644
--- a/currentstateserver/inthttp/server.go
+++ b/currentstateserver/inthttp/server.go
@@ -51,4 +51,17 @@ func AddRoutes(internalAPIMux *mux.Router, intAPI api.CurrentStateInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(QueryBulkStateContentPath,
+		httputil.MakeInternalAPI("queryBulkStateContent", func(req *http.Request) util.JSONResponse {
+			request := api.QueryBulkStateContentRequest{}
+			response := api.QueryBulkStateContentResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := intAPI.QueryBulkStateContent(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }
diff --git a/currentstateserver/storage/interface.go b/currentstateserver/storage/interface.go
index dbf223f33..04636bafb 100644
--- a/currentstateserver/storage/interface.go
+++ b/currentstateserver/storage/interface.go
@@ -17,6 +17,7 @@ package storage
 import (
 	"context"
 
+	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -31,4 +32,7 @@ type Database interface {
 	GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
 	// GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key).
 	GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error)
+	// GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match.
+	// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
+	GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
 }
diff --git a/currentstateserver/storage/postgres/current_room_state_table.go b/currentstateserver/storage/postgres/current_room_state_table.go
index 95621913b..bd2e075f0 100644
--- a/currentstateserver/storage/postgres/current_room_state_table.go
+++ b/currentstateserver/storage/postgres/current_room_state_table.go
@@ -18,7 +18,6 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
-	"strconv"
 
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
@@ -27,8 +26,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-var leaveEnum = strconv.Itoa(tables.MembershipToEnum["leave"])
-
 var currentRoomStateSchema = `
 -- Stores the current room state for every room.
 CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
@@ -44,29 +41,29 @@ CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
     state_key TEXT NOT NULL,
     -- The JSON for the event. Stored as TEXT because this should be valid UTF-8.
     headered_event_json TEXT NOT NULL,
-    -- The 'content.membership' enum value if this event is an m.room.member event.
-    membership SMALLINT NOT NULL DEFAULT 0,
+    -- A piece of extracted content e.g membership for m.room.member events
+    content_value TEXT NOT NULL DEFAULT '',
     -- Clobber based on 3-uple of room_id, type and state_key
     CONSTRAINT currentstate_current_room_state_unique UNIQUE (room_id, type, state_key)
 );
 -- for event deletion
 CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender);
 -- for querying membership states of users
-CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership)
-WHERE membership IS NOT NULL AND membership != ` + leaveEnum + `;
+CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, content_value)
+WHERE type='m.room.member' AND content_value IS NOT NULL AND content_value != 'leave';
 `
 
 const upsertRoomStateSQL = "" +
-	"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, membership)" +
+	"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" +
 	" VALUES ($1, $2, $3, $4, $5, $6, $7)" +
 	" ON CONFLICT ON CONSTRAINT currentstate_current_room_state_unique" +
-	" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, membership = $7"
+	" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7"
 
 const deleteRoomStateByEventIDSQL = "" +
 	"DELETE FROM currentstate_current_room_state WHERE event_id = $1"
 
 const selectRoomIDsWithMembershipSQL = "" +
-	"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
+	"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2"
 
 const selectStateEventSQL = "" +
 	"SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
@@ -74,12 +71,20 @@ const selectStateEventSQL = "" +
 const selectEventsWithEventIDsSQL = "" +
 	"SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id = ANY($1)"
 
+const selectBulkStateContentSQL = "" +
+	"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id = ANY($1) AND type = ANY($2) AND state_key = ANY($3)"
+
+const selectBulkStateContentWildSQL = "" +
+	"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id = ANY($1) AND type = ANY($2)"
+
 type currentRoomStateStatements struct {
 	upsertRoomStateStmt             *sql.Stmt
 	deleteRoomStateByEventIDStmt    *sql.Stmt
 	selectRoomIDsWithMembershipStmt *sql.Stmt
 	selectEventsWithEventIDsStmt    *sql.Stmt
 	selectStateEventStmt            *sql.Stmt
+	selectBulkStateContentStmt      *sql.Stmt
+	selectBulkStateContentWildStmt  *sql.Stmt
 }
 
 func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
@@ -103,6 +108,12 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro
 	if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil {
 		return nil, err
 	}
+	if s.selectBulkStateContentStmt, err = db.Prepare(selectBulkStateContentSQL); err != nil {
+		return nil, err
+	}
+	if s.selectBulkStateContentWildStmt, err = db.Prepare(selectBulkStateContentWildSQL); err != nil {
+		return nil, err
+	}
 	return s, nil
 }
 
@@ -111,10 +122,10 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
 	ctx context.Context,
 	txn *sql.Tx,
 	userID string,
-	membershipEnum int,
+	contentVal string,
 ) ([]string, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
-	rows, err := stmt.QueryContext(ctx, userID, membershipEnum)
+	rows, err := stmt.QueryContext(ctx, userID, contentVal)
 	if err != nil {
 		return nil, err
 	}
@@ -141,7 +152,7 @@ func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
 
 func (s *currentRoomStateStatements) UpsertRoomState(
 	ctx context.Context, txn *sql.Tx,
-	event gomatrixserverlib.HeaderedEvent, membershipEnum int,
+	event gomatrixserverlib.HeaderedEvent, contentVal string,
 ) error {
 	headeredJSON, err := json.Marshal(event)
 	if err != nil {
@@ -158,7 +169,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
 		event.Sender(),
 		*event.StateKey(),
 		headeredJSON,
-		membershipEnum,
+		contentVal,
 	)
 	return err
 }
@@ -206,3 +217,56 @@ func (s *currentRoomStateStatements) SelectStateEvent(
 	}
 	return &ev, err
 }
+
+func (s *currentRoomStateStatements) SelectBulkStateContent(
+	ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool,
+) ([]tables.StrippedEvent, error) {
+	hasWildcards := false
+	eventTypeSet := make(map[string]bool)
+	stateKeySet := make(map[string]bool)
+	var eventTypes []string
+	var stateKeys []string
+	for _, tuple := range tuples {
+		if !eventTypeSet[tuple.EventType] {
+			eventTypeSet[tuple.EventType] = true
+			eventTypes = append(eventTypes, tuple.EventType)
+		}
+		if !stateKeySet[tuple.StateKey] {
+			stateKeySet[tuple.StateKey] = true
+			stateKeys = append(stateKeys, tuple.StateKey)
+		}
+		if tuple.StateKey == "*" {
+			hasWildcards = true
+		}
+	}
+	var rows *sql.Rows
+	var err error
+	if hasWildcards && allowWildcards {
+		rows, err = s.selectBulkStateContentWildStmt.QueryContext(ctx, pq.StringArray(roomIDs), pq.StringArray(eventTypes))
+	} else {
+		rows, err = s.selectBulkStateContentStmt.QueryContext(
+			ctx, pq.StringArray(roomIDs), pq.StringArray(eventTypes), pq.StringArray(stateKeys),
+		)
+	}
+	if err != nil {
+		return nil, err
+	}
+	strippedEvents := []tables.StrippedEvent{}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectBulkStateContent: rows.close() failed")
+	for rows.Next() {
+		var roomID string
+		var eventType string
+		var stateKey string
+		var contentVal string
+		if err = rows.Scan(&roomID, &eventType, &stateKey, &contentVal); err != nil {
+			return nil, err
+		}
+		strippedEvents = append(strippedEvents, tables.StrippedEvent{
+			RoomID:       roomID,
+			ContentValue: contentVal,
+			EventType:    eventType,
+			StateKey:     stateKey,
+		})
+	}
+	return strippedEvents, rows.Err()
+}
diff --git a/currentstateserver/storage/shared/storage.go b/currentstateserver/storage/shared/storage.go
index d78b3e0ed..cd59ac129 100644
--- a/currentstateserver/storage/shared/storage.go
+++ b/currentstateserver/storage/shared/storage.go
@@ -17,7 +17,6 @@ package shared
 import (
 	"context"
 	"database/sql"
-	"fmt"
 
 	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -33,6 +32,10 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s
 	return d.CurrentRoomState.SelectStateEvent(ctx, roomID, evType, stateKey)
 }
 
+func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) {
+	return d.CurrentRoomState.SelectBulkStateContent(ctx, roomIDs, tuples, allowWildcards)
+}
+
 func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatrixserverlib.HeaderedEvent,
 	removeStateEventIDs []string) error {
 	return sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error {
@@ -48,20 +51,9 @@ func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatr
 				// ignore non state events
 				continue
 			}
-			var membershipEnum int
-			if event.Type() == "m.room.member" {
-				membership, err := event.Membership()
-				if err != nil {
-					return err
-				}
-				enum, ok := tables.MembershipToEnum[membership]
-				if !ok {
-					return fmt.Errorf("unknown membership: %s", membership)
-				}
-				membershipEnum = enum
-			}
+			contentVal := tables.ExtractContentValue(&event)
 
-			if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membershipEnum); err != nil {
+			if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, contentVal); err != nil {
 				return err
 			}
 		}
@@ -70,9 +62,5 @@ func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatr
 }
 
 func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) {
-	enum, ok := tables.MembershipToEnum[membership]
-	if !ok {
-		return nil, fmt.Errorf("unknown membership: %s", membership)
-	}
-	return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, enum)
+	return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership)
 }
diff --git a/currentstateserver/storage/sqlite3/current_room_state_table.go b/currentstateserver/storage/sqlite3/current_room_state_table.go
index 2e2b0e423..95185d9a8 100644
--- a/currentstateserver/storage/sqlite3/current_room_state_table.go
+++ b/currentstateserver/storage/sqlite3/current_room_state_table.go
@@ -35,39 +35,39 @@ CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
     sender TEXT NOT NULL,
     state_key TEXT NOT NULL,
     headered_event_json TEXT NOT NULL,
-    membership INTEGER NOT NULL DEFAULT 0,
+    content_value TEXT NOT NULL DEFAULT '',
     UNIQUE (room_id, type, state_key)
 );
 -- for event deletion
 CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender);
--- for querying membership states of users
--- CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave';
 `
 
 const upsertRoomStateSQL = "" +
-	"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, membership)" +
+	"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" +
 	" VALUES ($1, $2, $3, $4, $5, $6, $7)" +
 	" ON CONFLICT (event_id, room_id, type, sender)" +
-	" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, membership = $7"
+	" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7"
 
 const deleteRoomStateByEventIDSQL = "" +
 	"DELETE FROM currentstate_current_room_state WHERE event_id = $1"
 
 const selectRoomIDsWithMembershipSQL = "" +
-	"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
+	"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2"
 
 const selectStateEventSQL = "" +
 	"SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
 
 const selectEventsWithEventIDsSQL = "" +
-	// TODO: The session_id and transaction_id blanks are here because otherwise
-	// the rowsToStreamEvents expects there to be exactly five columns. We need to
-	// figure out if these really need to be in the DB, and if so, we need a
-	// better permanent fix for this. - neilalexander, 2 Jan 2020
-	"SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" +
-	" FROM currentstate_current_room_state WHERE event_id IN ($1)"
+	"SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id IN ($1)"
+
+const selectBulkStateContentSQL = "" +
+	"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id IN ($1) AND type IN ($2) AND state_key IN ($3)"
+
+const selectBulkStateContentWildSQL = "" +
+	"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id IN ($1) AND type IN ($2)"
 
 type currentRoomStateStatements struct {
+	db                              *sql.DB
 	upsertRoomStateStmt             *sql.Stmt
 	deleteRoomStateByEventIDStmt    *sql.Stmt
 	selectRoomIDsWithMembershipStmt *sql.Stmt
@@ -75,7 +75,9 @@ type currentRoomStateStatements struct {
 }
 
 func NewSqliteCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
-	s := &currentRoomStateStatements{}
+	s := &currentRoomStateStatements{
+		db: db,
+	}
 	_, err := db.Exec(currentRoomStateSchema)
 	if err != nil {
 		return nil, err
@@ -100,10 +102,10 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
 	ctx context.Context,
 	txn *sql.Tx,
 	userID string,
-	membershipEnum int,
+	membership string,
 ) ([]string, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
-	rows, err := stmt.QueryContext(ctx, userID, membershipEnum)
+	rows, err := stmt.QueryContext(ctx, userID, membership)
 	if err != nil {
 		return nil, err
 	}
@@ -130,7 +132,7 @@ func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
 
 func (s *currentRoomStateStatements) UpsertRoomState(
 	ctx context.Context, txn *sql.Tx,
-	event gomatrixserverlib.HeaderedEvent, membershipEnum int,
+	event gomatrixserverlib.HeaderedEvent, contentVal string,
 ) error {
 	headeredJSON, err := json.Marshal(event)
 	if err != nil {
@@ -147,7 +149,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
 		event.Sender(),
 		*event.StateKey(),
 		headeredJSON,
-		membershipEnum,
+		contentVal,
 	)
 	return err
 }
@@ -199,3 +201,76 @@ func (s *currentRoomStateStatements) SelectStateEvent(
 	}
 	return &ev, err
 }
+
+func (s *currentRoomStateStatements) SelectBulkStateContent(
+	ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool,
+) ([]tables.StrippedEvent, error) {
+	hasWildcards := false
+	eventTypeSet := make(map[string]bool)
+	stateKeySet := make(map[string]bool)
+	var eventTypes []string
+	var stateKeys []string
+	for _, tuple := range tuples {
+		if !eventTypeSet[tuple.EventType] {
+			eventTypeSet[tuple.EventType] = true
+			eventTypes = append(eventTypes, tuple.EventType)
+		}
+		if !stateKeySet[tuple.StateKey] {
+			stateKeySet[tuple.StateKey] = true
+			stateKeys = append(stateKeys, tuple.StateKey)
+		}
+		if tuple.StateKey == "*" {
+			hasWildcards = true
+		}
+	}
+
+	iRoomIDs := make([]interface{}, len(roomIDs))
+	for i, v := range roomIDs {
+		iRoomIDs[i] = v
+	}
+	iEventTypes := make([]interface{}, len(eventTypes))
+	for i, v := range eventTypes {
+		iEventTypes[i] = v
+	}
+	iStateKeys := make([]interface{}, len(stateKeys))
+	for i, v := range stateKeys {
+		iStateKeys[i] = v
+	}
+
+	var query string
+	var args []interface{}
+	if hasWildcards && allowWildcards {
+		query = strings.Replace(selectBulkStateContentWildSQL, "($1)", sqlutil.QueryVariadic(len(iRoomIDs)), 1)
+		query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(iEventTypes), len(iRoomIDs)), 1)
+		args = append(iRoomIDs, iEventTypes...)
+	} else {
+		query = strings.Replace(selectBulkStateContentSQL, "($1)", sqlutil.QueryVariadic(len(iRoomIDs)), 1)
+		query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(iEventTypes), len(iRoomIDs)), 1)
+		query = strings.Replace(query, "($3)", sqlutil.QueryVariadicOffset(len(iStateKeys), len(iEventTypes)+len(iRoomIDs)), 1)
+		args = append(iRoomIDs, iEventTypes...)
+		args = append(args, iStateKeys...)
+	}
+	rows, err := s.db.QueryContext(ctx, query, args...)
+
+	if err != nil {
+		return nil, err
+	}
+	strippedEvents := []tables.StrippedEvent{}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectBulkStateContent: rows.close() failed")
+	for rows.Next() {
+		var roomID string
+		var eventType string
+		var stateKey string
+		var contentVal string
+		if err = rows.Scan(&roomID, &eventType, &stateKey, &contentVal); err != nil {
+			return nil, err
+		}
+		strippedEvents = append(strippedEvents, tables.StrippedEvent{
+			RoomID:       roomID,
+			ContentValue: contentVal,
+			EventType:    eventType,
+			StateKey:     stateKey,
+		})
+	}
+	return strippedEvents, rows.Err()
+}
diff --git a/currentstateserver/storage/tables/interface.go b/currentstateserver/storage/tables/interface.go
index f2c8b14ed..8ba4e4eb9 100644
--- a/currentstateserver/storage/tables/interface.go
+++ b/currentstateserver/storage/tables/interface.go
@@ -19,26 +19,61 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/tidwall/gjson"
 )
 
-var MembershipToEnum = map[string]int{
-	gomatrixserverlib.Invite: 1,
-	gomatrixserverlib.Join:   2,
-	gomatrixserverlib.Leave:  3,
-	gomatrixserverlib.Ban:    4,
-}
-var EnumToMembership = map[int]string{
-	1: gomatrixserverlib.Invite,
-	2: gomatrixserverlib.Join,
-	3: gomatrixserverlib.Leave,
-	4: gomatrixserverlib.Ban,
-}
-
 type CurrentRoomState interface {
 	SelectStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
 	SelectEventsWithEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error)
-	UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, membershipEnum int) error
+	// UpsertRoomState stores the given event in the database, along with an extracted piece of content.
+	// The piece of content will vary depending on the event type, and table implementations may use this information to optimise
+	// lookups e.g membership lookups. The mapped value of `contentVal` is outlined in ExtractContentValue. An empty `contentVal`
+	// means there is nothing to store for this field.
+	UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, contentVal string) error
 	DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error
 	// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
-	SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membershipEnum int) ([]string, error)
+	SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error)
+	SelectBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]StrippedEvent, error)
+}
+
+// StrippedEvent represents a stripped event for returning extracted content values.
+type StrippedEvent struct {
+	RoomID       string
+	EventType    string
+	StateKey     string
+	ContentValue string
+}
+
+// ExtractContentValue from the given state event. For example, given an m.room.name event with:
+//    content: { name: "Foo" }
+// this returns "Foo".
+func ExtractContentValue(ev *gomatrixserverlib.HeaderedEvent) string {
+	content := ev.Content()
+	key := ""
+	switch ev.Type() {
+	case gomatrixserverlib.MRoomCreate:
+		key = "creator"
+	case gomatrixserverlib.MRoomCanonicalAlias:
+		key = "alias"
+	case gomatrixserverlib.MRoomHistoryVisibility:
+		key = "history_visibility"
+	case gomatrixserverlib.MRoomJoinRules:
+		key = "join_rule"
+	case gomatrixserverlib.MRoomMember:
+		key = "membership"
+	case gomatrixserverlib.MRoomName:
+		key = "name"
+	case "m.room.avatar":
+		key = "url"
+	case "m.room.topic":
+		key = "topic"
+	case "m.room.guest_access":
+		key = "guest_access"
+	}
+	result := gjson.GetBytes(content, key)
+	if !result.Exists() {
+		return ""
+	}
+	// this returns the empty string if this is not a string type
+	return result.Str
 }
diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index c0c000434..7d1994b25 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -16,6 +16,7 @@ package federationapi
 
 import (
 	"github.com/gorilla/mux"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
@@ -36,11 +37,12 @@ func AddPublicRoutes(
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI,
 ) {
 
 	routing.Setup(
 		router, cfg, rsAPI,
 		eduAPI, federationSenderAPI, keyRing,
-		federation, userAPI,
+		federation, userAPI, stateAPI,
 	)
 }
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index cc85c61bf..6bbe9d80e 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -31,7 +31,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	fsAPI := base.FederationSenderHTTPClient()
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil)
+	federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil, nil)
 	httputil.SetupHTTPAPI(
 		base.BaseMux,
 		base.PublicAPIMux,
diff --git a/federationapi/routing/publicrooms.go b/federationapi/routing/publicrooms.go
new file mode 100644
index 000000000..3807a5183
--- /dev/null
+++ b/federationapi/routing/publicrooms.go
@@ -0,0 +1,178 @@
+package routing
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type PublicRoomReq struct {
+	Since  string `json:"since,omitempty"`
+	Limit  int16  `json:"limit,omitempty"`
+	Filter filter `json:"filter,omitempty"`
+}
+
+type filter struct {
+	SearchTerms string `json:"generic_search_term,omitempty"`
+}
+
+// GetPostPublicRooms implements GET and POST /publicRooms
+func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI) util.JSONResponse {
+	var request PublicRoomReq
+	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
+		return *fillErr
+	}
+	if request.Limit == 0 {
+		request.Limit = 50
+	}
+	response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: response,
+	}
+}
+
+func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) {
+
+	var response gomatrixserverlib.RespPublicRooms
+	var limit int16
+	var offset int64
+	limit = request.Limit
+	offset, err := strconv.ParseInt(request.Since, 10, 64)
+	// ParseInt returns 0 and an error when trying to parse an empty string
+	// In that case, we want to assign 0 so we ignore the error
+	if err != nil && len(request.Since) > 0 {
+		util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed")
+		return nil, err
+	}
+
+	var queryRes roomserverAPI.QueryPublishedRoomsResponse
+	err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
+		return nil, err
+	}
+	response.TotalRoomCountEstimate = len(queryRes.RoomIDs)
+
+	if offset > 0 {
+		response.PrevBatch = strconv.Itoa(int(offset) - 1)
+	}
+	nextIndex := int(offset) + int(limit)
+	if response.TotalRoomCountEstimate > nextIndex {
+		response.NextBatch = strconv.Itoa(nextIndex)
+	}
+
+	if offset < 0 {
+		offset = 0
+	}
+	if nextIndex > len(queryRes.RoomIDs) {
+		nextIndex = len(queryRes.RoomIDs)
+	}
+	roomIDs := queryRes.RoomIDs[offset:nextIndex]
+	response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI)
+	return &response, err
+}
+
+// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request
+// on /publicRooms by parsing the incoming HTTP request
+// Filter is only filled for POST requests
+func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse {
+	if httpReq.Method == http.MethodGet {
+		limit, err := strconv.Atoi(httpReq.FormValue("limit"))
+		// Atoi returns 0 and an error when trying to parse an empty string
+		// In that case, we want to assign 0 so we ignore the error
+		if err != nil && len(httpReq.FormValue("limit")) > 0 {
+			util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
+			reqErr := jsonerror.InternalServerError()
+			return &reqErr
+		}
+		request.Limit = int16(limit)
+		request.Since = httpReq.FormValue("since")
+		return nil
+	} else if httpReq.Method == http.MethodPost {
+		return httputil.UnmarshalJSONRequest(httpReq, request)
+	}
+
+	return &util.JSONResponse{
+		Code: http.StatusMethodNotAllowed,
+		JSON: jsonerror.NotFound("Bad method"),
+	}
+}
+
+// due to lots of switches
+// nolint:gocyclo
+func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI.CurrentStateInternalAPI) ([]gomatrixserverlib.PublicRoom, error) {
+	avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
+	nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
+	canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
+	topicTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.topic", StateKey: ""}
+	guestTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.guest_access", StateKey: ""}
+	visibilityTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""}
+	joinRuleTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}
+
+	var stateRes currentstateAPI.QueryBulkStateContentResponse
+	err := stateAPI.QueryBulkStateContent(ctx, &currentstateAPI.QueryBulkStateContentRequest{
+		RoomIDs:        roomIDs,
+		AllowWildcards: true,
+		StateTuples: []gomatrixserverlib.StateKeyTuple{
+			nameTuple, canonicalTuple, topicTuple, guestTuple, visibilityTuple, joinRuleTuple, avatarTuple,
+			{EventType: gomatrixserverlib.MRoomMember, StateKey: "*"},
+		},
+	}, &stateRes)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed")
+		return nil, err
+	}
+	util.GetLogger(ctx).Infof("room IDs: %+v", roomIDs)
+	util.GetLogger(ctx).Infof("State res: %+v", stateRes.Rooms)
+	chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs))
+	i := 0
+	for roomID, data := range stateRes.Rooms {
+		pub := gomatrixserverlib.PublicRoom{
+			RoomID: roomID,
+		}
+		joinCount := 0
+		var joinRule, guestAccess string
+		for tuple, contentVal := range data {
+			if tuple.EventType == gomatrixserverlib.MRoomMember && contentVal == "join" {
+				joinCount++
+				continue
+			}
+			switch tuple {
+			case avatarTuple:
+				pub.AvatarURL = contentVal
+			case nameTuple:
+				pub.Name = contentVal
+			case topicTuple:
+				pub.Topic = contentVal
+			case canonicalTuple:
+				pub.CanonicalAlias = contentVal
+			case visibilityTuple:
+				pub.WorldReadable = contentVal == "world_readable"
+			// need both of these to determine whether guests can join
+			case joinRuleTuple:
+				joinRule = contentVal
+			case guestTuple:
+				guestAccess = contentVal
+			}
+		}
+		if joinRule == gomatrixserverlib.Public && guestAccess == "can_join" {
+			pub.GuestCanJoin = true
+		}
+		pub.JoinedMembersCount = joinCount
+		chunk[i] = pub
+		i++
+	}
+	return chunk, nil
+}
diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go
index 0afea7d04..cd97f2978 100644
--- a/federationapi/routing/routing.go
+++ b/federationapi/routing/routing.go
@@ -19,6 +19,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
@@ -52,6 +53,7 @@ func Setup(
 	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
 	userAPI userapi.UserInternalAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI,
 ) {
 	v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter()
 	v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter()
@@ -291,4 +293,10 @@ func Setup(
 			return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg)
 		},
 	)).Methods(http.MethodGet)
+
+	v1fedmux.Handle("/publicRooms",
+		httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse {
+			return GetPostPublicRooms(req, rsAPI, stateAPI)
+		}),
+	).Methods(http.MethodGet)
 }
diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go
index 3f5d5f4e0..bfbdaa5ff 100644
--- a/federationapi/routing/send_test.go
+++ b/federationapi/routing/send_test.go
@@ -111,6 +111,13 @@ func (t *testRoomserverAPI) PerformJoin(
 ) {
 }
 
+func (t *testRoomserverAPI) PerformPublish(
+	ctx context.Context,
+	req *api.PerformPublishRequest,
+	res *api.PerformPublishResponse,
+) {
+}
+
 func (t *testRoomserverAPI) PerformLeave(
 	ctx context.Context,
 	req *api.PerformLeaveRequest,
@@ -168,6 +175,14 @@ func (t *testRoomserverAPI) QueryMembershipForUser(
 	return fmt.Errorf("not implemented")
 }
 
+func (t *testRoomserverAPI) QueryPublishedRooms(
+	ctx context.Context,
+	request *api.QueryPublishedRoomsRequest,
+	response *api.QueryPublishedRoomsResponse,
+) error {
+	return fmt.Errorf("not implemented")
+}
+
 // Query a list of membership events for a room
 func (t *testRoomserverAPI) QueryMembershipsForRoom(
 	ctx context.Context,
diff --git a/go.mod b/go.mod
index 5f5a74a1b..4add4c3f5 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible
diff --git a/go.sum b/go.sum
index 24c8d74a0..52a556977 100644
--- a/go.sum
+++ b/go.sum
@@ -379,6 +379,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d h1:v1
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328 h1:rz6aiTpUyNPRcWZBWUGDkQjI7lfeLdhzy+x/Pw2jha8=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe h1:rCjG+azihYsO+EIdm//Zx5gQ7hzeJVraeSukLsW1Mic=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go
index 86275e28d..d4ae0915c 100644
--- a/internal/setup/monolith.go
+++ b/internal/setup/monolith.go
@@ -27,7 +27,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/mediaapi"
-	"github.com/matrix-org/dendrite/publicroomsapi"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/publicroomsapi/types"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
@@ -81,13 +80,14 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) {
 	federationapi.AddPublicRoutes(
 		publicMux, m.Config, m.UserAPI, m.FedClient,
 		m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI,
-		m.EDUInternalAPI,
+		m.EDUInternalAPI, m.StateAPI,
 	)
 	mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client)
-	publicroomsapi.AddPublicRoutes(
-		publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient,
-		m.ExtPublicRoomsProvider,
-	)
+	/*
+		publicroomsapi.AddPublicRoutes(
+			publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient,
+			m.ExtPublicRoomsProvider,
+		) */
 	syncapi.AddPublicRoutes(
 		publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, m.Config,
 	)
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 26ec8ca1d..0a5845dd6 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -36,6 +36,18 @@ type RoomserverInternalAPI interface {
 		res *PerformLeaveResponse,
 	) error
 
+	PerformPublish(
+		ctx context.Context,
+		req *PerformPublishRequest,
+		res *PerformPublishResponse,
+	)
+
+	QueryPublishedRooms(
+		ctx context.Context,
+		req *QueryPublishedRoomsRequest,
+		res *QueryPublishedRoomsResponse,
+	) error
+
 	// Query the latest events and state for a room from the room server.
 	QueryLatestEventsAndState(
 		ctx context.Context,
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index 8645b6f28..bdebc57b0 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -57,6 +57,25 @@ func (t *RoomserverInternalAPITrace) PerformLeave(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) PerformPublish(
+	ctx context.Context,
+	req *PerformPublishRequest,
+	res *PerformPublishResponse,
+) {
+	t.Impl.PerformPublish(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformPublish req=%+v res=%+v", js(req), js(res))
+}
+
+func (t *RoomserverInternalAPITrace) QueryPublishedRooms(
+	ctx context.Context,
+	req *QueryPublishedRoomsRequest,
+	res *QueryPublishedRoomsResponse,
+) error {
+	err := t.Impl.QueryPublishedRooms(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("QueryPublishedRooms req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
 func (t *RoomserverInternalAPITrace) QueryLatestEventsAndState(
 	ctx context.Context,
 	req *QueryLatestEventsAndStateRequest,
diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index 5d8d88a5a..9e8447339 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -136,3 +136,13 @@ type PerformBackfillResponse struct {
 	// Missing events, arbritrary order.
 	Events []gomatrixserverlib.HeaderedEvent `json:"events"`
 }
+
+type PerformPublishRequest struct {
+	RoomID     string
+	Visibility string
+}
+
+type PerformPublishResponse struct {
+	// If non-nil, the publish request failed. Contains more information why it failed.
+	Error *PerformError
+}
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index f0cb9374b..4e1d09c30 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -215,3 +215,13 @@ type QueryRoomVersionForRoomRequest struct {
 type QueryRoomVersionForRoomResponse struct {
 	RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
 }
+
+type QueryPublishedRoomsRequest struct {
+	// Optional. If specified, returns whether this room is published or not.
+	RoomID string
+}
+
+type QueryPublishedRoomsResponse struct {
+	// The list of published rooms.
+	RoomIDs []string
+}
diff --git a/roomserver/internal/perform_publish.go b/roomserver/internal/perform_publish.go
new file mode 100644
index 000000000..d7863620a
--- /dev/null
+++ b/roomserver/internal/perform_publish.go
@@ -0,0 +1,20 @@
+package internal
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+)
+
+func (r *RoomserverInternalAPI) PerformPublish(
+	ctx context.Context,
+	req *api.PerformPublishRequest,
+	res *api.PerformPublishResponse,
+) {
+	err := r.DB.PublishRoom(ctx, req.RoomID, req.Visibility == "public")
+	if err != nil {
+		res.Error = &api.PerformError{
+			Msg: err.Error(),
+		}
+	}
+}
diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go
index 19236bfbd..7fa3247a6 100644
--- a/roomserver/internal/query.go
+++ b/roomserver/internal/query.go
@@ -930,3 +930,16 @@ func (r *RoomserverInternalAPI) QueryRoomVersionForRoom(
 	r.Cache.StoreRoomVersion(request.RoomID, response.RoomVersion)
 	return nil
 }
+
+func (r *RoomserverInternalAPI) QueryPublishedRooms(
+	ctx context.Context,
+	req *api.QueryPublishedRoomsRequest,
+	res *api.QueryPublishedRoomsResponse,
+) error {
+	rooms, err := r.DB.GetPublishedRooms(ctx)
+	if err != nil {
+		return err
+	}
+	res.RoomIDs = rooms
+	return nil
+}
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index 8a2b1204c..ad24af4ad 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -29,6 +29,7 @@ const (
 	RoomserverPerformJoinPath     = "/roomserver/performJoin"
 	RoomserverPerformLeavePath    = "/roomserver/performLeave"
 	RoomserverPerformBackfillPath = "/roomserver/performBackfill"
+	RoomserverPerformPublishPath  = "/roomserver/performPublish"
 
 	// Query operations
 	RoomserverQueryLatestEventsAndStatePath    = "/roomserver/queryLatestEventsAndState"
@@ -41,6 +42,7 @@ const (
 	RoomserverQueryStateAndAuthChainPath       = "/roomserver/queryStateAndAuthChain"
 	RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities"
 	RoomserverQueryRoomVersionForRoomPath      = "/roomserver/queryRoomVersionForRoom"
+	RoomserverQueryPublishedRoomsPath          = "/roomserver/queryPublishedRooms"
 )
 
 type httpRoomserverInternalAPI struct {
@@ -194,6 +196,23 @@ func (h *httpRoomserverInternalAPI) PerformLeave(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
+func (h *httpRoomserverInternalAPI) PerformPublish(
+	ctx context.Context,
+	req *api.PerformPublishRequest,
+	res *api.PerformPublishResponse,
+) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPublish")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverPerformPublishPath
+	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
+		}
+	}
+}
+
 // QueryLatestEventsAndState implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState(
 	ctx context.Context,
@@ -233,6 +252,18 @@ func (h *httpRoomserverInternalAPI) QueryEventsByID(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
+func (h *httpRoomserverInternalAPI) QueryPublishedRooms(
+	ctx context.Context,
+	request *api.QueryPublishedRoomsRequest,
+	response *api.QueryPublishedRoomsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublishedRooms")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverQueryPublishedRoomsPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
 // QueryMembershipForUser implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryMembershipForUser(
 	ctx context.Context,
diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go
index 1c47e87e2..bb54abf9c 100644
--- a/roomserver/inthttp/server.go
+++ b/roomserver/inthttp/server.go
@@ -61,6 +61,31 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverPerformPublishPath,
+		httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse {
+			var request api.PerformPublishRequest
+			var response api.PerformPublishResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			r.PerformPublish(req.Context(), &request, &response)
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(
+		RoomserverQueryPublishedRoomsPath,
+		httputil.MakeInternalAPI("queryPublishedRooms", func(req *http.Request) util.JSONResponse {
+			var request api.QueryPublishedRoomsRequest
+			var response api.QueryPublishedRoomsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryPublishedRooms(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	internalAPIMux.Handle(
 		RoomserverQueryLatestEventsAndStatePath,
 		httputil.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse {
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 0c4e2e0b5..5c916f294 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -139,4 +139,8 @@ type Database interface {
 	EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error)
 	// Look up the room version for a given room.
 	GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error)
+	// Publish or unpublish a room from the room directory.
+	PublishRoom(ctx context.Context, roomID string, publish bool) error
+	// Returns a list of room IDs for rooms which are published.
+	GetPublishedRooms(ctx context.Context) ([]string, error)
 }
diff --git a/roomserver/storage/postgres/published_table.go b/roomserver/storage/postgres/published_table.go
new file mode 100644
index 000000000..23a9b067e
--- /dev/null
+++ b/roomserver/storage/postgres/published_table.go
@@ -0,0 +1,101 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+)
+
+const publishedSchema = `
+-- Stores which rooms are published in the room directory
+CREATE TABLE IF NOT EXISTS roomserver_published (
+    -- The room ID of the room
+    room_id TEXT NOT NULL PRIMARY KEY,
+    -- Whether it is published or not
+    published BOOLEAN NOT NULL DEFAULT false
+);
+`
+
+const upsertPublishedSQL = "" +
+	"INSERT INTO roomserver_published (room_id, published) VALUES ($1, $2) " +
+	"ON CONFLICT (room_id) DO UPDATE SET published=$2"
+
+const selectAllPublishedSQL = "" +
+	"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
+
+const selectPublishedSQL = "" +
+	"SELECT published FROM roomserver_published WHERE room_id = $1"
+
+type publishedStatements struct {
+	upsertPublishedStmt    *sql.Stmt
+	selectAllPublishedStmt *sql.Stmt
+	selectPublishedStmt    *sql.Stmt
+}
+
+func NewPostgresPublishedTable(db *sql.DB) (tables.Published, error) {
+	s := &publishedStatements{}
+	_, err := db.Exec(publishedSchema)
+	if err != nil {
+		return nil, err
+	}
+	return s, shared.StatementList{
+		{&s.upsertPublishedStmt, upsertPublishedSQL},
+		{&s.selectAllPublishedStmt, selectAllPublishedSQL},
+		{&s.selectPublishedStmt, selectPublishedSQL},
+	}.Prepare(db)
+}
+
+func (s *publishedStatements) UpsertRoomPublished(
+	ctx context.Context, roomID string, published bool,
+) (err error) {
+	_, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published)
+	return
+}
+
+func (s *publishedStatements) SelectPublishedFromRoomID(
+	ctx context.Context, roomID string,
+) (published bool, err error) {
+	err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published)
+	if err == sql.ErrNoRows {
+		return false, nil
+	}
+	return
+}
+
+func (s *publishedStatements) SelectAllPublishedRooms(
+	ctx context.Context, published bool,
+) ([]string, error) {
+	rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
+
+	var roomIDs []string
+	for rows.Next() {
+		var roomID string
+		if err = rows.Scan(&roomID); err != nil {
+			return nil, err
+		}
+
+		roomIDs = append(roomIDs, roomID)
+	}
+	return roomIDs, rows.Err()
+}
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index d76ee0a92..23d078e4a 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -87,6 +87,10 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database,
 	if err != nil {
 		return nil, err
 	}
+	published, err := NewPostgresPublishedTable(db)
+	if err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                  db,
 		EventTypesTable:     eventTypes,
@@ -101,6 +105,7 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database,
 		RoomAliasesTable:    roomAliases,
 		InvitesTable:        invites,
 		MembershipTable:     membership,
+		PublishedTable:      published,
 	}
 	return &d, nil
 }
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index e6d0e34e2..166822d0c 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -26,6 +26,7 @@ type Database struct {
 	PrevEventsTable     tables.PreviousEvents
 	InvitesTable        tables.Invites
 	MembershipTable     tables.Membership
+	PublishedTable      tables.Published
 }
 
 func (d *Database) EventTypeNIDs(
@@ -420,6 +421,14 @@ func (d *Database) StoreEvent(
 	}, nil
 }
 
+func (d *Database) PublishRoom(ctx context.Context, roomID string, publish bool) error {
+	return d.PublishedTable.UpsertRoomPublished(ctx, roomID, publish)
+}
+
+func (d *Database) GetPublishedRooms(ctx context.Context) ([]string, error) {
+	return d.PublishedTable.SelectAllPublishedRooms(ctx, true)
+}
+
 func (d *Database) assignRoomNID(
 	ctx context.Context, txn *sql.Tx,
 	roomID string, roomVersion gomatrixserverlib.RoomVersion,
diff --git a/roomserver/storage/sqlite3/published_table.go b/roomserver/storage/sqlite3/published_table.go
new file mode 100644
index 000000000..9995fff6d
--- /dev/null
+++ b/roomserver/storage/sqlite3/published_table.go
@@ -0,0 +1,100 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+)
+
+const publishedSchema = `
+-- Stores which rooms are published in the room directory
+CREATE TABLE IF NOT EXISTS roomserver_published (
+    -- The room ID of the room
+    room_id TEXT NOT NULL PRIMARY KEY,
+    -- Whether it is published or not
+    published BOOLEAN NOT NULL DEFAULT false
+);
+`
+
+const upsertPublishedSQL = "" +
+	"INSERT OR REPLACE INTO roomserver_published (room_id, published) VALUES ($1, $2)"
+
+const selectAllPublishedSQL = "" +
+	"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
+
+const selectPublishedSQL = "" +
+	"SELECT published FROM roomserver_published WHERE room_id = $1"
+
+type publishedStatements struct {
+	upsertPublishedStmt    *sql.Stmt
+	selectAllPublishedStmt *sql.Stmt
+	selectPublishedStmt    *sql.Stmt
+}
+
+func NewSqlitePublishedTable(db *sql.DB) (tables.Published, error) {
+	s := &publishedStatements{}
+	_, err := db.Exec(publishedSchema)
+	if err != nil {
+		return nil, err
+	}
+	return s, shared.StatementList{
+		{&s.upsertPublishedStmt, upsertPublishedSQL},
+		{&s.selectAllPublishedStmt, selectAllPublishedSQL},
+		{&s.selectPublishedStmt, selectPublishedSQL},
+	}.Prepare(db)
+}
+
+func (s *publishedStatements) UpsertRoomPublished(
+	ctx context.Context, roomID string, published bool,
+) (err error) {
+	_, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published)
+	return
+}
+
+func (s *publishedStatements) SelectPublishedFromRoomID(
+	ctx context.Context, roomID string,
+) (published bool, err error) {
+	err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published)
+	if err == sql.ErrNoRows {
+		return false, nil
+	}
+	return
+}
+
+func (s *publishedStatements) SelectAllPublishedRooms(
+	ctx context.Context, published bool,
+) ([]string, error) {
+	rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
+
+	var roomIDs []string
+	for rows.Next() {
+		var roomID string
+		if err = rows.Scan(&roomID); err != nil {
+			return nil, err
+		}
+
+		roomIDs = append(roomIDs, roomID)
+	}
+	return roomIDs, rows.Err()
+}
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index 8e9352192..767b13ce0 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -110,6 +110,10 @@ func Open(dataSourceName string) (*Database, error) {
 	if err != nil {
 		return nil, err
 	}
+	published, err := NewSqlitePublishedTable(d.db)
+	if err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		EventsTable:         d.events,
@@ -124,6 +128,7 @@ func Open(dataSourceName string) (*Database, error) {
 		RoomAliasesTable:    roomAliases,
 		InvitesTable:        d.invites,
 		MembershipTable:     d.membership,
+		PublishedTable:      published,
 	}
 	return &d, nil
 }
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 3aa8c538c..7499089ca 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -120,3 +120,9 @@ type Membership interface {
 	SelectMembershipsFromRoomAndMembership(ctx context.Context, roomNID types.RoomNID, membership MembershipState, localOnly bool) (eventNIDs []types.EventNID, err error)
 	UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID) error
 }
+
+type Published interface {
+	UpsertRoomPublished(ctx context.Context, roomID string, published bool) (err error)
+	SelectPublishedFromRoomID(ctx context.Context, roomID string) (published bool, err error)
+	SelectAllPublishedRooms(ctx context.Context, published bool) ([]string, error)
+}
diff --git a/sytest-whitelist b/sytest-whitelist
index d055e75a2..85517cf9a 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -181,7 +181,11 @@ Outbound federation can query profile data
 /event/ on joined room works
 /event/ does not allow access to events before the user joined
 Federation key API allows unsigned requests for keys
+GET /publicRooms lists rooms
+GET /publicRooms includes avatar URLs
 Can paginate public room list
+GET /publicRooms lists newly-created room
+Name/topic keys are correct
 GET /directory/room/:room_alias yields room ID
 PUT /directory/room/:room_alias creates alias
 Room aliases can contain Unicode

From 9c1f38621c4d787761092bc841e06ca424fbbf35 Mon Sep 17 00:00:00 2001
From: Kegsay <kegan@matrix.org>
Date: Thu, 2 Jul 2020 17:11:33 +0100
Subject: [PATCH 21/23] Remove publicroomsapi (#1176)

* Remove all of publicroomsapi

* Remove references to publicroomsapi

* Remove doc references to publicroomsapi
---
 build/docker/config/dendrite-config.yaml      |   4 +-
 build/docker/docker-compose.polylith.yml      |  10 +-
 build/docker/images-build.sh                  |   2 +-
 build/docker/images-pull.sh                   |   2 +-
 build/docker/images-push.sh                   |   2 +-
 build/docker/postgres/create_db.sh            |   2 +-
 build/gobind/monolith.go                      |   9 -
 .../types/types.go => clientapi/api/api.go    |   6 +-
 clientapi/clientapi.go                        |   4 +-
 clientapi/routing/directory_public.go         |   4 +-
 clientapi/routing/routing.go                  |  97 +++---
 cmd/client-api-proxy/main.go                  |  28 +-
 cmd/dendrite-client-api-server/main.go        |   2 +-
 cmd/dendrite-current-state-server/main.go     |  33 +++
 cmd/dendrite-demo-libp2p/main.go              |   6 +-
 .../storage/postgreswithdht/storage.go        | 164 ----------
 .../storage/postgreswithpubsub/storage.go     | 179 -----------
 cmd/dendrite-demo-libp2p/storage/storage.go   |  62 ----
 cmd/dendrite-demo-yggdrasil/main.go           |   9 -
 cmd/dendrite-monolith-server/main.go          |   8 -
 cmd/dendrite-public-rooms-api-server/main.go  |  41 ---
 cmd/dendritejs/main.go                        |   9 -
 dendrite-config.yaml                          |   2 -
 docs/INSTALL.md                               |  32 +-
 docs/WIRING-Current.md                        |  50 ++--
 go.mod                                        |   2 +
 go.sum                                        |   9 +
 internal/config/config.go                     |   5 -
 internal/setup/monolith.go                    |  17 +-
 internal/test/config.go                       |   6 +-
 internal/test/server.go                       |   1 -
 publicroomsapi/README.md                      |   5 -
 publicroomsapi/consumers/roomserver.go        |  99 -------
 publicroomsapi/directory/directory.go         | 120 --------
 publicroomsapi/directory/public_rooms.go      | 262 ----------------
 publicroomsapi/publicroomsapi.go              |  51 ----
 publicroomsapi/routing/routing.go             |  79 -----
 publicroomsapi/storage/interface.go           |  32 --
 publicroomsapi/storage/postgres/prepare.go    |  36 ---
 .../storage/postgres/public_rooms_table.go    | 280 ------------------
 publicroomsapi/storage/postgres/storage.go    | 259 ----------------
 publicroomsapi/storage/sqlite3/prepare.go     |  36 ---
 .../storage/sqlite3/public_rooms_table.go     | 273 -----------------
 publicroomsapi/storage/sqlite3/storage.go     | 265 -----------------
 publicroomsapi/storage/storage.go             |  45 ---
 publicroomsapi/storage/storage_wasm.go        |  39 ---
 46 files changed, 160 insertions(+), 2528 deletions(-)
 rename publicroomsapi/types/types.go => clientapi/api/api.go (81%)
 create mode 100644 cmd/dendrite-current-state-server/main.go
 delete mode 100644 cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go
 delete mode 100644 cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go
 delete mode 100644 cmd/dendrite-demo-libp2p/storage/storage.go
 delete mode 100644 cmd/dendrite-public-rooms-api-server/main.go
 delete mode 100644 publicroomsapi/README.md
 delete mode 100644 publicroomsapi/consumers/roomserver.go
 delete mode 100644 publicroomsapi/directory/directory.go
 delete mode 100644 publicroomsapi/directory/public_rooms.go
 delete mode 100644 publicroomsapi/publicroomsapi.go
 delete mode 100644 publicroomsapi/routing/routing.go
 delete mode 100644 publicroomsapi/storage/interface.go
 delete mode 100644 publicroomsapi/storage/postgres/prepare.go
 delete mode 100644 publicroomsapi/storage/postgres/public_rooms_table.go
 delete mode 100644 publicroomsapi/storage/postgres/storage.go
 delete mode 100644 publicroomsapi/storage/sqlite3/prepare.go
 delete mode 100644 publicroomsapi/storage/sqlite3/public_rooms_table.go
 delete mode 100644 publicroomsapi/storage/sqlite3/storage.go
 delete mode 100644 publicroomsapi/storage/storage.go
 delete mode 100644 publicroomsapi/storage/storage_wasm.go

diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml
index 53d9f7b02..c8302c0d9 100644
--- a/build/docker/config/dendrite-config.yaml
+++ b/build/docker/config/dendrite-config.yaml
@@ -98,7 +98,7 @@ database:
     room_server: "postgres://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable"
     server_key: "postgres://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable"
     federation_sender: "postgres://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable"
-    public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable"
+    current_state: "postgres://dendrite:itsasecret@postgres/dendrite_currentstate?sslmode=disable"
     appservice: "postgres://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable"
     # If using naffka you need to specify a naffka database
     #naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable"
@@ -113,7 +113,7 @@ listen:
     server_key_api: "server_key_api:7778"
     sync_api: "sync_api:7773"
     media_api: "media_api:7774"
-    public_rooms_api: "public_rooms_api:7775"
+    current_state_server: "current_state_server:7775"
     federation_sender: "federation_sender:7776"
     edu_server: "edu_server:7777"
     key_server: "key_server:7779"
diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml
index d424d43b1..62ca6763d 100644
--- a/build/docker/docker-compose.polylith.yml
+++ b/build/docker/docker-compose.polylith.yml
@@ -7,8 +7,7 @@ services:
       "--bind-address=:8008",
       "--client-api-server-url=http://client_api:7771",
       "--sync-api-server-url=http://sync_api:7773",
-      "--media-api-server-url=http://media_api:7774",
-      "--public-rooms-api-server-url=http://public_rooms_api:7775"
+      "--media-api-server-url=http://media_api:7774"
     ]
     volumes:
       - ./config:/etc/dendrite
@@ -18,7 +17,6 @@ services:
       - sync_api
       - client_api
       - media_api
-      - public_rooms_api
     ports:
       - "8008:8008"
 
@@ -45,9 +43,9 @@ services:
     networks:
       - internal
 
-  public_rooms_api:
-    hostname: public_rooms_api
-    image: matrixdotorg/dendrite:publicroomsapi
+  current_state_server:
+    hostname: current_state_server
+    image: matrixdotorg/dendrite:currentstateserver
     command: [
       "--config=dendrite.yaml"
     ]
diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh
index 9ee5a09de..443f30920 100755
--- a/build/docker/images-build.sh
+++ b/build/docker/images-build.sh
@@ -15,7 +15,7 @@ docker build -t matrixdotorg/dendrite:federationsender  --build-arg component=de
 docker build -t matrixdotorg/dendrite:federationproxy   --build-arg component=federation-api-proxy              -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:keyserver         --build-arg component=dendrite-key-server               -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:mediaapi          --build-arg component=dendrite-media-api-server         -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:publicroomsapi    --build-arg component=dendrite-public-rooms-api-server  -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:currentstateserver    --build-arg component=dendrite-current-state-server  -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:roomserver        --build-arg component=dendrite-room-server              -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:syncapi           --build-arg component=dendrite-sync-api-server          -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:serverkeyapi      --build-arg component=dendrite-server-key-api-server    -f build/docker/Dockerfile.component .
diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh
index da08a7325..b4a4b2fce 100755
--- a/build/docker/images-pull.sh
+++ b/build/docker/images-pull.sh
@@ -11,7 +11,7 @@ docker pull matrixdotorg/dendrite:federationsender
 docker pull matrixdotorg/dendrite:federationproxy
 docker pull matrixdotorg/dendrite:keyserver
 docker pull matrixdotorg/dendrite:mediaapi
-docker pull matrixdotorg/dendrite:publicroomsapi
+docker pull matrixdotorg/dendrite:currentstateserver
 docker pull matrixdotorg/dendrite:roomserver
 docker pull matrixdotorg/dendrite:syncapi
 docker pull matrixdotorg/dendrite:userapi
diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh
index 1ac60b921..ec1e860f0 100755
--- a/build/docker/images-push.sh
+++ b/build/docker/images-push.sh
@@ -11,7 +11,7 @@ docker push matrixdotorg/dendrite:federationsender
 docker push matrixdotorg/dendrite:federationproxy
 docker push matrixdotorg/dendrite:keyserver
 docker push matrixdotorg/dendrite:mediaapi
-docker push matrixdotorg/dendrite:publicroomsapi
+docker push matrixdotorg/dendrite:currentstateserver
 docker push matrixdotorg/dendrite:roomserver
 docker push matrixdotorg/dendrite:syncapi
 docker push matrixdotorg/dendrite:serverkeyapi
diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh
index 8ed11db1e..a884b5ccd 100644
--- a/build/docker/postgres/create_db.sh
+++ b/build/docker/postgres/create_db.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
 
-for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do
+for db in account device mediaapi syncapi roomserver serverkey federationsender currentstate appservice naffka; do
     createdb -U dendrite -O dendrite dendrite_$db
 done
diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go
index 62ca17447..ea9c62f20 100644
--- a/build/gobind/monolith.go
+++ b/build/gobind/monolith.go
@@ -18,7 +18,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -75,7 +74,6 @@ func (m *DendriteMonolith) Start() {
 	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory))
 	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory))
 	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory))
-	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory))
 	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s/dendrite-currentstate.db", m.StorageDirectory))
 	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory))
 	if err = cfg.Derive(); err != nil {
@@ -111,11 +109,6 @@ func (m *DendriteMonolith) Start() {
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
 	rsAPI.SetFederationSenderAPI(fsAPI)
 
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to public rooms db")
-	}
-
 	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
 
 	monolith := setup.Monolith{
@@ -135,8 +128,6 @@ func (m *DendriteMonolith) Start() {
 		UserAPI:             userAPI,
 		StateAPI:            stateAPI,
 		//ServerKeyAPI:        serverKeyAPI,
-
-		PublicRoomsDB: publicRoomsDB,
 	}
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
 
diff --git a/publicroomsapi/types/types.go b/clientapi/api/api.go
similarity index 81%
rename from publicroomsapi/types/types.go
rename to clientapi/api/api.go
index 11cb0d204..dae462c08 100644
--- a/publicroomsapi/types/types.go
+++ b/clientapi/api/api.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,10 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package types
+package api
 
-// ExternalPublicRoomsProvider provides a list of homeservers who should be queried
-// periodically for a list of public rooms on their server.
 type ExternalPublicRoomsProvider interface {
 	// The list of homeserver domains to query. These servers will receive a request
 	// via this API: https://matrix.org/docs/spec/server_server/latest#public-room-directory
diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index 8ea84249a..bbce6dccf 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -18,6 +18,7 @@ import (
 	"github.com/Shopify/sarama"
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
@@ -47,6 +48,7 @@ func AddPublicRoutes(
 	transactionsCache *transactions.Cache,
 	fsAPI federationSenderAPI.FederationSenderInternalAPI,
 	userAPI userapi.UserInternalAPI,
+	extRoomsProvider api.ExternalPublicRoomsProvider,
 ) {
 	syncProducer := &producers.SyncAPIProducer{
 		Producer: producer,
@@ -56,6 +58,6 @@ func AddPublicRoutes(
 	routing.Setup(
 		router, cfg, eduInputAPI, rsAPI, asAPI,
 		accountsDB, deviceDB, userAPI, federation,
-		syncProducer, transactionsCache, fsAPI, stateAPI,
+		syncProducer, transactionsCache, fsAPI, stateAPI, extRoomsProvider,
 	)
 }
diff --git a/clientapi/routing/directory_public.go b/clientapi/routing/directory_public.go
index 6d0db579b..64600cb49 100644
--- a/clientapi/routing/directory_public.go
+++ b/clientapi/routing/directory_public.go
@@ -24,10 +24,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -64,7 +64,7 @@ func GetPostPublicRooms(
 // GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied.
 func GetPostPublicRoomsWithExternal(
 	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI,
-	fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider,
+	fedClient *gomatrixserverlib.FederationClient, extRoomsProvider api.ExternalPublicRoomsProvider,
 ) util.JSONResponse {
 	var request PublicRoomReq
 	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 57bb921d9..85c5c0d9a 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
@@ -30,7 +31,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/userapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/dendrite/userapi/storage/devices"
@@ -55,12 +55,13 @@ func Setup(
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	accountDB accounts.Database,
 	deviceDB devices.Database,
-	userAPI api.UserInternalAPI,
+	userAPI userapi.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
 	federationSender federationSenderAPI.FederationSenderInternalAPI,
 	stateAPI currentstateAPI.CurrentStateInternalAPI,
+	extRoomsProvider api.ExternalPublicRoomsProvider,
 ) {
 
 	publicAPIMux.Handle("/client/versions",
@@ -84,12 +85,12 @@ func Setup(
 	unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter()
 
 	r0mux.Handle("/createRoom",
-		httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
-		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -100,12 +101,12 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/joined_rooms",
-		httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetJoinedRooms(req, device, stateAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/join",
-		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -116,7 +117,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/leave",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -127,7 +128,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/ban",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -136,7 +137,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/invite",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -145,7 +146,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/kick",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -154,7 +155,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/unban",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -163,7 +164,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
-		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -172,7 +173,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
-		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -183,7 +184,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/event/{eventID}",
-		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -192,7 +193,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
@@ -200,7 +201,7 @@ func Setup(
 		return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
@@ -214,7 +215,7 @@ func Setup(
 		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], eventType, "", eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
@@ -224,7 +225,7 @@ func Setup(
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
-		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -240,7 +241,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
-		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -273,7 +274,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -283,7 +284,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -321,19 +322,19 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/logout",
-		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return Logout(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/logout/all",
-		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return LogoutAll(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
-		httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -343,7 +344,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/sendToDevice/{eventType}/{txnID}",
-		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -357,7 +358,7 @@ func Setup(
 	// rather than r0. It's an exact duplicate of the above handler.
 	// TODO: Remove this if/when sytest is fixed!
 	unstableMux.Handle("/sendToDevice/{eventType}/{txnID}",
-		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -368,7 +369,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/account/whoami",
-		httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return Whoami(req, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -430,7 +431,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
-		httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -452,7 +453,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/displayname",
-		httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -464,19 +465,19 @@ func Setup(
 	// PUT requests, so we need to allow this method
 
 	r0mux.Handle("/account/3pid",
-		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetAssociated3PIDs(req, accountDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/account/3pid",
-		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return CheckAndSave3PIDAssociation(req, accountDB, device, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	unstableMux.Handle("/account/3pid/delete",
-		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return Forget3PID(req, accountDB)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -499,7 +500,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/voip/turnServer",
-		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return RequestTurnServer(req, device, cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -525,7 +526,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
-		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -535,7 +536,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
-		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -545,7 +546,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
-		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -555,7 +556,7 @@ func Setup(
 	).Methods(http.MethodGet)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
-		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -565,7 +566,7 @@ func Setup(
 	).Methods(http.MethodGet)
 
 	r0mux.Handle("/rooms/{roomID}/members",
-		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -575,7 +576,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/joined_members",
-		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -592,13 +593,13 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/devices",
-		httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetDevicesByLocalpart(req, deviceDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -608,7 +609,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -618,7 +619,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -628,7 +629,7 @@ func Setup(
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/delete_devices",
-		httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return DeleteDevices(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -653,7 +654,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags",
-		httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -663,7 +664,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
-		httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -673,7 +674,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
-		httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -683,7 +684,7 @@ func Setup(
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/capabilities",
-		httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetCapabilities(req, rsAPI)
 		}),
 	).Methods(http.MethodGet)
diff --git a/cmd/client-api-proxy/main.go b/cmd/client-api-proxy/main.go
index ebc0629f8..742ec3e31 100644
--- a/cmd/client-api-proxy/main.go
+++ b/cmd/client-api-proxy/main.go
@@ -48,13 +48,12 @@ Arguments:
 `
 
 var (
-	syncServerURL     = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'")
-	clientAPIURL      = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'")
-	mediaAPIURL       = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'")
-	publicRoomsAPIURL = flag.String("public-rooms-api-server-url", "", "The base URL of the listening 'dendrite-public-rooms-api-server' process. E.g. 'http://localhost:7775'")
-	bindAddress       = flag.String("bind-address", ":8008", "The listening port for the proxy.")
-	certFile          = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
-	keyFile           = flag.String("tls-key", "", "The PEM private key to use for TLS")
+	syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'")
+	clientAPIURL  = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'")
+	mediaAPIURL   = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'")
+	bindAddress   = flag.String("bind-address", ":8008", "The listening port for the proxy.")
+	certFile      = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
+	keyFile       = flag.String("tls-key", "", "The PEM private key to use for TLS")
 )
 
 func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
@@ -121,13 +120,6 @@ func main() {
 		fmt.Fprintln(os.Stderr, "no --media-api-server-url specified.")
 		os.Exit(1)
 	}
-
-	if *publicRoomsAPIURL == "" {
-		flag.Usage()
-		fmt.Fprintln(os.Stderr, "no --public-rooms-api-server-url specified.")
-		os.Exit(1)
-	}
-
 	syncProxy, err := makeProxy(*syncServerURL)
 	if err != nil {
 		panic(err)
@@ -140,14 +132,8 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	publicRoomsProxy, err := makeProxy(*publicRoomsAPIURL)
-	if err != nil {
-		panic(err)
-	}
 
 	http.Handle("/_matrix/client/r0/sync", syncProxy)
-	http.Handle("/_matrix/client/r0/directory/list/", publicRoomsProxy)
-	http.Handle("/_matrix/client/r0/publicRooms", publicRoomsProxy)
 	http.Handle("/_matrix/media/v1/", mediaProxy)
 	http.Handle("/", clientProxy)
 
@@ -159,8 +145,6 @@ func main() {
 
 	fmt.Println("Proxying requests to:")
 	fmt.Println("  /_matrix/client/r0/sync            => ", *syncServerURL+"/api/_matrix/client/r0/sync")
-	fmt.Println("  /_matrix/client/r0/directory/list  => ", *publicRoomsAPIURL+"/_matrix/client/r0/directory/list")
-	fmt.Println("  /_matrix/client/r0/publicRooms     => ", *publicRoomsAPIURL+"/_matrix/media/client/r0/publicRooms")
 	fmt.Println("  /_matrix/media/v1                  => ", *mediaAPIURL+"/api/_matrix/media/v1")
 	fmt.Println("  /*                                 => ", *clientAPIURL+"/api/*")
 	fmt.Println("Listening on ", *bindAddress)
diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go
index f46dae502..58c029fed 100644
--- a/cmd/dendrite-client-api-server/main.go
+++ b/cmd/dendrite-client-api-server/main.go
@@ -39,7 +39,7 @@ func main() {
 
 	clientapi.AddPublicRoutes(
 		base.PublicAPIMux, base.Cfg, base.KafkaProducer, deviceDB, accountDB, federation,
-		rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI,
+		rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI, nil,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))
diff --git a/cmd/dendrite-current-state-server/main.go b/cmd/dendrite-current-state-server/main.go
new file mode 100644
index 000000000..0d4eae7b5
--- /dev/null
+++ b/cmd/dendrite-current-state-server/main.go
@@ -0,0 +1,33 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/matrix-org/dendrite/currentstateserver"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+func main() {
+	cfg := setup.ParseFlags(false)
+	base := setup.NewBaseDendrite(cfg, "CurrentStateServer", true)
+	defer base.Close() // nolint: errcheck
+
+	stateAPI := currentstateserver.NewInternalAPI(cfg, base.KafkaConsumer)
+
+	currentstateserver.AddInternalRoutes(base.InternalAPIMux, stateAPI)
+
+	base.SetupAndServeHTTP(string(base.Cfg.Bind.CurrentState), string(base.Cfg.Listen.CurrentState))
+
+}
diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go
index 4bb7a96c2..988f4aa7f 100644
--- a/cmd/dendrite-demo-libp2p/main.go
+++ b/cmd/dendrite-demo-libp2p/main.go
@@ -29,7 +29,6 @@ import (
 	p2phttp "github.com/libp2p/go-libp2p-http"
 	p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery"
 	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage"
 	"github.com/matrix-org/dendrite/currentstateserver"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/federationsender"
@@ -130,7 +129,6 @@ func main() {
 	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
 	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
 	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicrooms.db", *instanceName))
 	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
 	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
@@ -164,10 +162,12 @@ func main() {
 		&base.Base, federation, rsAPI, keyRing,
 	)
 	rsAPI.SetFederationSenderAPI(fsAPI)
+	/* TODO:
 	publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
+	*/
 	stateAPI := currentstateserver.NewInternalAPI(base.Base.Cfg, base.Base.KafkaConsumer)
 
 	monolith := setup.Monolith{
@@ -187,8 +187,6 @@ func main() {
 		ServerKeyAPI:        serverKeyAPI,
 		StateAPI:            stateAPI,
 		UserAPI:             userAPI,
-
-		PublicRoomsDB: publicRoomsDB,
 	}
 	monolith.AddAllPublicRoutes(base.Base.PublicAPIMux)
 
diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go
deleted file mode 100644
index d2cb36a8b..000000000
--- a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go
+++ /dev/null
@@ -1,164 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgreswithdht
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	dht "github.com/libp2p/go-libp2p-kad-dht"
-)
-
-const DHTInterval = time.Second * 10
-
-// PublicRoomsServerDatabase represents a public rooms server database.
-type PublicRoomsServerDatabase struct {
-	dht *dht.IpfsDHT
-	postgres.PublicRoomsServerDatabase
-	ourRoomsContext  context.Context                         // our current value in the DHT
-	ourRoomsCancel   context.CancelFunc                      // cancel when we want to expire our value
-	foundRooms       map[string]gomatrixserverlib.PublicRoom // additional rooms we have learned about from the DHT
-	foundRoomsMutex  sync.RWMutex                            // protects foundRooms
-	maintenanceTimer *time.Timer                             //
-	roomsAdvertised  atomic.Value                            // stores int
-	roomsDiscovered  atomic.Value                            // stores int
-}
-
-// NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName)
-	if err != nil {
-		return nil, err
-	}
-	provider := PublicRoomsServerDatabase{
-		dht:                       dht,
-		PublicRoomsServerDatabase: *pg,
-	}
-	go provider.ResetDHTMaintenance()
-	provider.roomsAdvertised.Store(0)
-	provider.roomsDiscovered.Store(0)
-	return &provider, nil
-}
-
-func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) {
-	return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID)
-}
-
-func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error {
-	d.ResetDHTMaintenance()
-	return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID)
-}
-
-func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
-	count, err := d.PublicRoomsServerDatabase.CountPublicRooms(ctx)
-	if err != nil {
-		return 0, err
-	}
-	d.foundRoomsMutex.RLock()
-	defer d.foundRoomsMutex.RUnlock()
-	return count + int64(len(d.foundRooms)), nil
-}
-
-func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) {
-	realfilter := filter
-	if realfilter == "__local__" {
-		realfilter = ""
-	}
-	rooms, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, realfilter)
-	if err != nil {
-		return []gomatrixserverlib.PublicRoom{}, err
-	}
-	if filter != "__local__" {
-		d.foundRoomsMutex.RLock()
-		defer d.foundRoomsMutex.RUnlock()
-		for _, room := range d.foundRooms {
-			rooms = append(rooms, room)
-		}
-	}
-	return rooms, nil
-}
-
-func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error {
-	return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove)
-}
-
-func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error {
-	return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event)
-}
-
-func (d *PublicRoomsServerDatabase) ResetDHTMaintenance() {
-	if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() {
-		<-d.maintenanceTimer.C
-	}
-	d.Interval()
-}
-
-func (d *PublicRoomsServerDatabase) Interval() {
-	if err := d.AdvertiseRoomsIntoDHT(); err != nil {
-		//	fmt.Println("Failed to advertise room in DHT:", err)
-	}
-	if err := d.FindRoomsInDHT(); err != nil {
-		//	fmt.Println("Failed to find rooms in DHT:", err)
-	}
-	fmt.Println("Found", d.roomsDiscovered.Load(), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)")
-	d.maintenanceTimer = time.AfterFunc(DHTInterval, d.Interval)
-}
-
-func (d *PublicRoomsServerDatabase) AdvertiseRoomsIntoDHT() error {
-	dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second)
-	_ = dbCancel
-	ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__")
-	if err != nil {
-		return err
-	}
-	if j, err := json.Marshal(ourRooms); err == nil {
-		d.roomsAdvertised.Store(len(ourRooms))
-		d.ourRoomsContext, d.ourRoomsCancel = context.WithCancel(context.Background())
-		if err := d.dht.PutValue(d.ourRoomsContext, "/matrix/publicRooms", j); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (d *PublicRoomsServerDatabase) FindRoomsInDHT() error {
-	d.foundRoomsMutex.Lock()
-	searchCtx, searchCancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer searchCancel()
-	defer d.foundRoomsMutex.Unlock()
-	results, err := d.dht.GetValues(searchCtx, "/matrix/publicRooms", 1024)
-	if err != nil {
-		return err
-	}
-	d.foundRooms = make(map[string]gomatrixserverlib.PublicRoom)
-	for _, result := range results {
-		var received []gomatrixserverlib.PublicRoom
-		if err := json.Unmarshal(result.Val, &received); err != nil {
-			return err
-		}
-		for _, room := range received {
-			d.foundRooms[room.RoomID] = room
-		}
-	}
-	d.roomsDiscovered.Store(len(d.foundRooms))
-	return nil
-}
diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go
deleted file mode 100644
index cf642eb38..000000000
--- a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go
+++ /dev/null
@@ -1,179 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgreswithpubsub
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-)
-
-const MaintenanceInterval = time.Second * 10
-
-type discoveredRoom struct {
-	time time.Time
-	room gomatrixserverlib.PublicRoom
-}
-
-// PublicRoomsServerDatabase represents a public rooms server database.
-type PublicRoomsServerDatabase struct {
-	postgres.PublicRoomsServerDatabase                           //
-	pubsub                             *pubsub.PubSub            //
-	subscription                       *pubsub.Subscription      //
-	foundRooms                         map[string]discoveredRoom // additional rooms we have learned about from the DHT
-	foundRoomsMutex                    sync.RWMutex              // protects foundRooms
-	maintenanceTimer                   *time.Timer               //
-	roomsAdvertised                    atomic.Value              // stores int
-}
-
-// NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName)
-	if err != nil {
-		return nil, err
-	}
-	provider := PublicRoomsServerDatabase{
-		pubsub:                    pubsub,
-		PublicRoomsServerDatabase: *pg,
-		foundRooms:                make(map[string]discoveredRoom),
-	}
-	if topic, err := pubsub.Join("/matrix/publicRooms"); err != nil {
-		return nil, err
-	} else if sub, err := topic.Subscribe(); err == nil {
-		provider.subscription = sub
-		go provider.MaintenanceTimer()
-		go provider.FindRooms()
-		provider.roomsAdvertised.Store(0)
-		return &provider, nil
-	} else {
-		return nil, err
-	}
-}
-
-func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) {
-	return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID)
-}
-
-func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error {
-	d.MaintenanceTimer()
-	return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID)
-}
-
-func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
-	d.foundRoomsMutex.RLock()
-	defer d.foundRoomsMutex.RUnlock()
-	return int64(len(d.foundRooms)), nil
-}
-
-func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) {
-	var rooms []gomatrixserverlib.PublicRoom
-	if filter == "__local__" {
-		if r, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, ""); err == nil {
-			rooms = append(rooms, r...)
-		} else {
-			return []gomatrixserverlib.PublicRoom{}, err
-		}
-	} else {
-		d.foundRoomsMutex.RLock()
-		defer d.foundRoomsMutex.RUnlock()
-		for _, room := range d.foundRooms {
-			rooms = append(rooms, room.room)
-		}
-	}
-	return rooms, nil
-}
-
-func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error {
-	return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove)
-}
-
-func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error {
-	return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event)
-}
-
-func (d *PublicRoomsServerDatabase) MaintenanceTimer() {
-	if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() {
-		<-d.maintenanceTimer.C
-	}
-	d.Interval()
-}
-
-func (d *PublicRoomsServerDatabase) Interval() {
-	d.foundRoomsMutex.Lock()
-	for k, v := range d.foundRooms {
-		if time.Since(v.time) > time.Minute {
-			delete(d.foundRooms, k)
-		}
-	}
-	d.foundRoomsMutex.Unlock()
-	if err := d.AdvertiseRooms(); err != nil {
-		fmt.Println("Failed to advertise room in DHT:", err)
-	}
-	d.foundRoomsMutex.RLock()
-	defer d.foundRoomsMutex.RUnlock()
-	fmt.Println("Found", len(d.foundRooms), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)")
-	d.maintenanceTimer = time.AfterFunc(MaintenanceInterval, d.Interval)
-}
-
-func (d *PublicRoomsServerDatabase) AdvertiseRooms() error {
-	dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second)
-	_ = dbCancel
-	ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__")
-	if err != nil {
-		return err
-	}
-	advertised := 0
-	for _, room := range ourRooms {
-		if j, err := json.Marshal(room); err == nil {
-			if topic, err := d.pubsub.Join("/matrix/publicRooms"); err != nil {
-				fmt.Println("Failed to subscribe to topic:", err)
-			} else if err := topic.Publish(context.TODO(), j); err != nil {
-				fmt.Println("Failed to publish public room:", err)
-			} else {
-				advertised++
-			}
-		}
-	}
-
-	d.roomsAdvertised.Store(advertised)
-	return nil
-}
-
-func (d *PublicRoomsServerDatabase) FindRooms() {
-	for {
-		msg, err := d.subscription.Next(context.Background())
-		if err != nil {
-			continue
-		}
-		received := discoveredRoom{
-			time: time.Now(),
-		}
-		if err := json.Unmarshal(msg.Data, &received.room); err != nil {
-			fmt.Println("Unmarshal error:", err)
-			continue
-		}
-		d.foundRoomsMutex.Lock()
-		d.foundRooms[received.room.RoomID] = received
-		d.foundRoomsMutex.Unlock()
-	}
-}
diff --git a/cmd/dendrite-demo-libp2p/storage/storage.go b/cmd/dendrite-demo-libp2p/storage/storage.go
deleted file mode 100644
index 2d8dc1817..000000000
--- a/cmd/dendrite-demo-libp2p/storage/storage.go
+++ /dev/null
@@ -1,62 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package storage
-
-import (
-	"net/url"
-
-	dht "github.com/libp2p/go-libp2p-kad-dht"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithdht"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-const schemePostgres = "postgres"
-const schemeFile = "file"
-
-// NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (storage.Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
-	}
-	switch uri.Scheme {
-	case schemePostgres:
-		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
-	case schemeFile:
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
-	default:
-		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
-	}
-}
-
-// NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (storage.Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
-	}
-	switch uri.Scheme {
-	case schemePostgres:
-		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
-	case schemeFile:
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
-	default:
-		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
-	}
-}
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index cef34c7ed..7476d686b 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -34,7 +34,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -74,7 +73,6 @@ func main() {
 	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
 	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
 	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicrooms.db", *instanceName))
 	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
 	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
@@ -110,11 +108,6 @@ func main() {
 
 	rsComponent.SetFederationSenderAPI(fsAPI)
 
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to public rooms db")
-	}
-
 	embed.Embed(base.BaseMux, *instancePort, "Yggdrasil Demo")
 
 	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
@@ -136,8 +129,6 @@ func main() {
 		UserAPI:             userAPI,
 		StateAPI:            stateAPI,
 		//ServerKeyAPI:        serverKeyAPI,
-
-		PublicRoomsDB: publicRoomsDB,
 	}
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
 
diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go
index 905eda2ba..9ac6941b2 100644
--- a/cmd/dendrite-monolith-server/main.go
+++ b/cmd/dendrite-monolith-server/main.go
@@ -27,7 +27,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/serverkeyapi"
@@ -118,11 +117,6 @@ func main() {
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
 	rsImpl.SetFederationSenderAPI(fsAPI)
 
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to public rooms db")
-	}
-
 	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
 
 	monolith := setup.Monolith{
@@ -142,8 +136,6 @@ func main() {
 		ServerKeyAPI:        serverKeyAPI,
 		StateAPI:            stateAPI,
 		UserAPI:             userAPI,
-
-		PublicRoomsDB: publicRoomsDB,
 	}
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
 
diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go
deleted file mode 100644
index 23866b757..000000000
--- a/cmd/dendrite-public-rooms-api-server/main.go
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/publicroomsapi"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
-	"github.com/sirupsen/logrus"
-)
-
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "PublicRoomsAPI", true)
-	defer base.Close() // nolint: errcheck
-
-	userAPI := base.UserAPIClient()
-
-	rsAPI := base.RoomserverHTTPClient()
-
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to public rooms db")
-	}
-	publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, userAPI, publicRoomsDB, rsAPI, nil, nil)
-
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI))
-
-}
diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go
index 36ce9f65b..6e2bdafec 100644
--- a/cmd/dendritejs/main.go
+++ b/cmd/dendritejs/main.go
@@ -29,7 +29,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/userapi"
 	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
@@ -169,7 +168,6 @@ func main() {
 	cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db"
 	cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db"
 	cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db"
-	cfg.Database.PublicRoomsAPI = "file:/idb/dendritejs_publicrooms.db"
 	cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db"
 	cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db"
 	cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db"
@@ -215,11 +213,6 @@ func main() {
 	rsAPI.SetFederationSenderAPI(fedSenderAPI)
 	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI)
 
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to public rooms db")
-	}
-
 	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
 
 	monolith := setup.Monolith{
@@ -239,8 +232,6 @@ func main() {
 		StateAPI:            stateAPI,
 		UserAPI:             userAPI,
 		//ServerKeyAPI:        serverKeyAPI,
-
-		PublicRoomsDB:          publicRoomsDB,
 		ExtPublicRoomsProvider: p2pPublicRoomProvider,
 	}
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
diff --git a/dendrite-config.yaml b/dendrite-config.yaml
index 70c8f7958..2b95c102b 100644
--- a/dendrite-config.yaml
+++ b/dendrite-config.yaml
@@ -120,7 +120,6 @@ database:
     server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable"
     federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable"
     appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable"
-    public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable"
     current_state: "postgres://dendrite:itsasecret@localhost/dendrite_currentstate?sslmode=disable"
     max_open_conns: 100
     max_idle_conns: 2
@@ -137,7 +136,6 @@ listen:
     federation_api: "localhost:7772"
     sync_api: "localhost:7773"
     media_api: "localhost:7774"
-    public_rooms_api: "localhost:7775"
     federation_sender: "localhost:7776"
     appservice_api: "localhost:7777"
     edu_server: "localhost:7778"
diff --git a/docs/INSTALL.md b/docs/INSTALL.md
index b4c81a42b..c97351809 100644
--- a/docs/INSTALL.md
+++ b/docs/INSTALL.md
@@ -108,7 +108,7 @@ Assuming that Postgres 9.5 (or later) is installed:
 * Create the component databases:
 
   ```bash
-  for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do
+  for i in account device mediaapi syncapi roomserver serverkey federationsender currentstate appservice naffka; do
       sudo -u postgres createdb -O dendrite dendrite_$i
   done
   ```
@@ -176,17 +176,17 @@ The following contains scripts which will run all the required processes in orde
                       |            |            :7774
                       |            |
                       |            |
-                      |            |   /directory +----------------------------------+
-                      |            |   +--------->| dendrite-public-rooms-api-server |<========++
-                      |            |   |          +----------------------------------+         ||
-                      |            |   |        :7775    |                                     ||
-                      |            |   |    +<-----------+                                     ||
-                      |            |   |    |                                                  ||
-                      |            |   | /sync    +--------------------------+                 ||
+                      |            |   
+                      |            |   
+                      |            |   
+                      |            |  
+                      |            |  
+                      |            |  
+                      |            |     /sync    +--------------------------+                 
                       |            |   +--------->| dendrite-sync-api-server |<================++
-                      |            |   |    |     +--------------------------+                 ||
-                      |            |   |    |   :7773    |         ^^                          ||
-Matrix      +------------------+   |   |    |            |         ||    client_data           ||
+                      |            |   |          +--------------------------+                 ||
+                      |            |   |        :7773    |         ^^                          ||
+Matrix      +------------------+   |   |                 |         ||    client_data           ||
 Clients --->| client-api-proxy |-------+    +<-----------+         ++=============++           ||
             +------------------+   |   |    |                                     ||           ||
           :8008                    |   | CS API   +----------------------------+  ||           ||
@@ -232,7 +232,6 @@ your client at `http://localhost:8008`.
 --client-api-server-url "http://localhost:7771" \
 --sync-api-server-url "http://localhost:7773" \
 --media-api-server-url "http://localhost:7774" \
---public-rooms-api-server-url "http://localhost:7775" \
 ```
 
 ### Federation proxy
@@ -282,15 +281,6 @@ order to upload and retrieve media.
 ./bin/dendrite-media-api-server --config dendrite.yaml
 ```
 
-### Public room server
-
-This implements `/directory` requests. Clients talk to this via the proxy
-in order to retrieve room directory listings.
-
-```bash
-./bin/dendrite-public-rooms-api-server --config dendrite.yaml
-```
-
 ### Federation API server
 
 This implements federation requests. Servers talk to this via the proxy in
diff --git a/docs/WIRING-Current.md b/docs/WIRING-Current.md
index ec539d4e9..b74f341e5 100644
--- a/docs/WIRING-Current.md
+++ b/docs/WIRING-Current.md
@@ -9,23 +9,22 @@ a request/response model like HTTP or RPC. Therefore, components can expose "int
 Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests.
 
 ```
-   Tier 1            Sync        PublicRooms      FederationAPI      ClientAPI    MediaAPI
-Public Facing        | .-----1------`              | | |    | |        | | | |
-                     2 | .-------3-----------------` | |    | `--------|-|-|-|--11--------------------.
-                     | | | .--------4----------------------------------` | | |                        |
-                     | | | |         .---5-----------` |    |            | | |                        |
-                     | | | |         |  .---6----------------------------` | |                        |
-                     | | | |         |  |              |  .-----7----------` |                        |
-                     | | | |         |  |              8  | |                10                       |
-                     | | | |         |  |              |  | `---9----.       |                        |
-                     V V V V         V  V              V  V          V       V                        V
+   Tier 1            Sync                         FederationAPI      ClientAPI    MediaAPI
+Public Facing        |                             | | |    | |        | | | |
+                     2   .-------3-----------------` | |    | `--------|-|-|-|--11--------------------.
+                     |   | .--------4----------------------------------` | | |                        |
+                     |   | |         .---5-----------` |    |            | | |                        |
+                     |   | |         |  .---6----------------------------` | |                        |
+                     |   | |         |  |              |  .-----7----------` |                        |
+                     |   | |         |  |              8  | |                10                       |
+                     |   | |         |  |              |  | `---9----.       |                        |
+                     V   V V         V  V              V  V          V       V                        V
    Tier 2           Roomserver     EDUServer         FedSender       AppService    KeyServer   ServerKeyAPI
 Internal only               |                               `------------------------12----------^   ^
                             `------------------------------------------------------------13----------`
 
  Client ---> Server
 ```
-- 1 (PublicRooms -> Roomserver): Calculating current auth for changing visibility
 - 2 (Sync -> Roomserver): When making backfill requests
 - 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests
 - 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests
@@ -46,20 +45,20 @@ In addition to this, all public facing components (Tier 1) talk to the `UserAPI`
 ```
                        .----1--------------------------------------------.
                        V                                                 |
-   Tier 1            Sync        PublicRooms      FederationAPI      ClientAPI    MediaAPI
-Public Facing        ^   ^           ^                                 ^  
-                     |   |           |                                 |
-                     2   |           |                                 |
+   Tier 1            Sync                         FederationAPI      ClientAPI    MediaAPI
+Public Facing        ^   ^                                             ^  
+                     |   |                                             |
+                     2   |                                             |
                      |   `-3------------.                              |
-                     |               |  |                              |
-                     |               |  |                              |
-                     | .------4------`  |                              |
-                     | | .--------5-----|------------------------------`             
-                     | | |              |           
+                     |                  |                              |
+                     |                  |                              |
+                     |                  |                              |
+                     |   .--------4-----|------------------------------`             
+                     |   |              |           
    Tier 2           Roomserver     EDUServer         FedSender       AppService    KeyServer   ServerKeyAPI
 Internal only              |          |                ^ ^
-                           |          `-----6----------` |
-                           `--------------------7--------`
+                           |          `-----5----------` |
+                           `--------------------6--------`
 
 
 Producer ----> Consumer
@@ -67,7 +66,6 @@ Producer ----> Consumer
 - 1 (ClientAPI -> Sync): For tracking account data
 - 2 (Roomserver -> Sync): For all data to send to clients
 - 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients
-- 4 (Roomserver -> PublicRooms): For tracking the current room name/topic/joined count/etc.
-- 5 (Roomserver -> ClientAPI): For tracking memberships for profile updates.
-- 6 (EDUServer -> FedSender): For sending EDUs over federation
-- 7 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts.
+- 4 (Roomserver -> ClientAPI): For tracking memberships for profile updates.
+- 5 (EDUServer -> FedSender): For sending EDUs over federation
+- 6 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts.
diff --git a/go.mod b/go.mod
index 4add4c3f5..b278ec4b3 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,8 @@ module github.com/matrix-org/dendrite
 require (
 	github.com/Shopify/sarama v1.26.1
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/gologme/log v1.2.0
 	github.com/gorilla/mux v1.7.3
 	github.com/hashicorp/golang-lru v0.5.4
diff --git a/go.sum b/go.sum
index 52a556977..a98ae851c 100644
--- a/go.sum
+++ b/go.sum
@@ -59,6 +59,10 @@ github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhY
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
+github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
 github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
@@ -189,6 +193,7 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
@@ -497,6 +502,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8=
 github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -504,10 +510,12 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
@@ -650,6 +658,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/p
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/internal/config/config.go b/internal/config/config.go
index 8275fc478..777bd6a39 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -186,9 +186,6 @@ type Dendrite struct {
 		// The AppServices database stores information used by the AppService component.
 		// It is only accessed by the AppService component.
 		AppService DataSource `yaml:"appservice"`
-		// The PublicRoomsAPI database stores information used to compute the public
-		// room directory. It is only accessed by the PublicRoomsAPI server.
-		PublicRoomsAPI DataSource `yaml:"public_rooms_api"`
 		// The Naffka database is used internally by the naffka library, if used.
 		Naffka DataSource `yaml:"naffka,omitempty"`
 		// Maximum open connections to the DB (0 = use default, negative means unlimited)
@@ -233,7 +230,6 @@ type Dendrite struct {
 		UserAPI          Address `yaml:"user_api"`
 		RoomServer       Address `yaml:"room_server"`
 		FederationSender Address `yaml:"federation_sender"`
-		PublicRoomsAPI   Address `yaml:"public_rooms_api"`
 		EDUServer        Address `yaml:"edu_server"`
 		KeyServer        Address `yaml:"key_server"`
 	} `yaml:"bind"`
@@ -250,7 +246,6 @@ type Dendrite struct {
 		UserAPI          Address `yaml:"user_api"`
 		RoomServer       Address `yaml:"room_server"`
 		FederationSender Address `yaml:"federation_sender"`
-		PublicRoomsAPI   Address `yaml:"public_rooms_api"`
 		EDUServer        Address `yaml:"edu_server"`
 		KeyServer        Address `yaml:"key_server"`
 	} `yaml:"listen"`
diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go
index d4ae0915c..ca90986e9 100644
--- a/internal/setup/monolith.go
+++ b/internal/setup/monolith.go
@@ -19,6 +19,7 @@ import (
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/clientapi/api"
 	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/federationapi"
@@ -27,8 +28,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/mediaapi"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api"
 	"github.com/matrix-org/dendrite/syncapi"
@@ -58,13 +57,8 @@ type Monolith struct {
 	UserAPI             userapi.UserInternalAPI
 	StateAPI            currentstateAPI.CurrentStateInternalAPI
 
-	// TODO: can we remove this? It's weird that we are required the database
-	// yet every other component can do that on its own. libp2p-demo uses a custom
-	// database though annoyingly.
-	PublicRoomsDB storage.Database
-
 	// Optional
-	ExtPublicRoomsProvider types.ExternalPublicRoomsProvider
+	ExtPublicRoomsProvider api.ExternalPublicRoomsProvider
 }
 
 // AddAllPublicRoutes attaches all public paths to the given router
@@ -73,7 +67,7 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) {
 		publicMux, m.Config, m.KafkaProducer, m.DeviceDB, m.AccountDB,
 		m.FedClient, m.RoomserverAPI,
 		m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(),
-		m.FederationSenderAPI, m.UserAPI,
+		m.FederationSenderAPI, m.UserAPI, m.ExtPublicRoomsProvider,
 	)
 
 	keyserver.AddPublicRoutes(publicMux, m.Config, m.UserAPI)
@@ -83,11 +77,6 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) {
 		m.EDUInternalAPI, m.StateAPI,
 	)
 	mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client)
-	/*
-		publicroomsapi.AddPublicRoutes(
-			publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient,
-			m.ExtPublicRoomsProvider,
-		) */
 	syncapi.AddPublicRoutes(
 		publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, m.Config,
 	)
diff --git a/internal/test/config.go b/internal/test/config.go
index 951f65a12..bbcc9bed2 100644
--- a/internal/test/config.go
+++ b/internal/test/config.go
@@ -96,7 +96,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
 	cfg.Database.RoomServer = config.DataSource(database)
 	cfg.Database.ServerKey = config.DataSource(database)
 	cfg.Database.SyncAPI = config.DataSource(database)
-	cfg.Database.PublicRoomsAPI = config.DataSource(database)
+	cfg.Database.CurrentState = config.DataSource(database)
 
 	cfg.Listen.ClientAPI = assignAddress()
 	cfg.Listen.AppServiceAPI = assignAddress()
@@ -104,7 +104,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
 	cfg.Listen.MediaAPI = assignAddress()
 	cfg.Listen.RoomServer = assignAddress()
 	cfg.Listen.SyncAPI = assignAddress()
-	cfg.Listen.PublicRoomsAPI = assignAddress()
+	cfg.Listen.CurrentState = assignAddress()
 	cfg.Listen.EDUServer = assignAddress()
 
 	// Bind to the same address as the listen address
@@ -115,7 +115,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
 	cfg.Bind.MediaAPI = cfg.Listen.MediaAPI
 	cfg.Bind.RoomServer = cfg.Listen.RoomServer
 	cfg.Bind.SyncAPI = cfg.Listen.SyncAPI
-	cfg.Bind.PublicRoomsAPI = cfg.Listen.PublicRoomsAPI
+	cfg.Bind.CurrentState = cfg.Listen.CurrentState
 	cfg.Bind.EDUServer = cfg.Listen.EDUServer
 
 	return &cfg, port, nil
diff --git a/internal/test/server.go b/internal/test/server.go
index c3348d533..2d4117f4c 100644
--- a/internal/test/server.go
+++ b/internal/test/server.go
@@ -99,7 +99,6 @@ func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
 		"--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI),
 		"--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI),
 		"--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI),
-		"--public-rooms-api-server-url", "http://" + string(cfg.Listen.PublicRoomsAPI),
 		"--tls-cert", "server.crt",
 		"--tls-key", "server.key",
 	}
diff --git a/publicroomsapi/README.md b/publicroomsapi/README.md
deleted file mode 100644
index 594fe29c5..000000000
--- a/publicroomsapi/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Public rooms API
-
-This server is responsible for serving requests hitting `/publicRooms` and `/directory/list/room/{roomID}` as per:
-
-https://matrix.org/docs/spec/client_server/r0.2.0.html#listing-rooms
diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go
deleted file mode 100644
index b9686d56d..000000000
--- a/publicroomsapi/consumers/roomserver.go
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package consumers
-
-import (
-	"context"
-	"encoding/json"
-
-	"github.com/Shopify/sarama"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	log "github.com/sirupsen/logrus"
-)
-
-// OutputRoomEventConsumer consumes events that originated in the room server.
-type OutputRoomEventConsumer struct {
-	rsAPI      api.RoomserverInternalAPI
-	rsConsumer *internal.ContinualConsumer
-	db         storage.Database
-}
-
-// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
-func NewOutputRoomEventConsumer(
-	cfg *config.Dendrite,
-	kafkaConsumer sarama.Consumer,
-	store storage.Database,
-	rsAPI api.RoomserverInternalAPI,
-) *OutputRoomEventConsumer {
-	consumer := internal.ContinualConsumer{
-		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
-		Consumer:       kafkaConsumer,
-		PartitionStore: store,
-	}
-	s := &OutputRoomEventConsumer{
-		rsConsumer: &consumer,
-		db:         store,
-		rsAPI:      rsAPI,
-	}
-	consumer.ProcessMessage = s.onMessage
-
-	return s
-}
-
-// Start consuming from room servers
-func (s *OutputRoomEventConsumer) Start() error {
-	return s.rsConsumer.Start()
-}
-
-// onMessage is called when the sync server receives a new event from the room server output log.
-func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
-	// Parse out the event JSON
-	var output api.OutputEvent
-	if err := json.Unmarshal(msg.Value, &output); err != nil {
-		// If the message was invalid, log it and move on to the next message in the stream
-		log.WithError(err).Errorf("roomserver output log: message parse failure")
-		return nil
-	}
-
-	if output.Type != api.OutputTypeNewRoomEvent {
-		log.WithField("type", output.Type).Debug(
-			"roomserver output log: ignoring unknown output type",
-		)
-		return nil
-	}
-
-	var remQueryRes api.QueryEventsByIDResponse
-	if len(output.NewRoomEvent.RemovesStateEventIDs) > 0 {
-		remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs}
-		if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil {
-			log.Warn(err)
-			return err
-		}
-	}
-
-	var addQueryEvents, remQueryEvents []gomatrixserverlib.Event
-	for _, headeredEvent := range output.NewRoomEvent.AddsState() {
-		addQueryEvents = append(addQueryEvents, headeredEvent.Event)
-	}
-	for _, headeredEvent := range remQueryRes.Events {
-		remQueryEvents = append(remQueryEvents, headeredEvent.Event)
-	}
-
-	return s.db.UpdateRoomFromEvents(context.TODO(), addQueryEvents, remQueryEvents)
-}
diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go
deleted file mode 100644
index 8b68279aa..000000000
--- a/publicroomsapi/directory/directory.go
+++ /dev/null
@@ -1,120 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package directory
-
-import (
-	"net/http"
-
-	"github.com/matrix-org/dendrite/roomserver/api"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-
-	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/matrix-org/util"
-)
-
-type roomVisibility struct {
-	Visibility string `json:"visibility"`
-}
-
-// GetVisibility implements GET /directory/list/room/{roomID}
-func GetVisibility(
-	req *http.Request, publicRoomsDatabase storage.Database,
-	roomID string,
-) util.JSONResponse {
-	isPublic, err := publicRoomsDatabase.GetRoomVisibility(req.Context(), roomID)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.GetRoomVisibility failed")
-		return jsonerror.InternalServerError()
-	}
-
-	var v roomVisibility
-	if isPublic {
-		v.Visibility = gomatrixserverlib.Public
-	} else {
-		v.Visibility = "private"
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: v,
-	}
-}
-
-// SetVisibility implements PUT /directory/list/room/{roomID}
-// TODO: Allow admin users to edit the room visibility
-func SetVisibility(
-	req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *userapi.Device,
-	roomID string,
-) util.JSONResponse {
-	queryMembershipReq := api.QueryMembershipForUserRequest{
-		RoomID: roomID,
-		UserID: dev.UserID,
-	}
-	var queryMembershipRes api.QueryMembershipForUserResponse
-	err := rsAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user")
-		return jsonerror.InternalServerError()
-	}
-	// Check if user id is in room
-	if !queryMembershipRes.IsInRoom {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("user does not belong to room"),
-		}
-	}
-	queryEventsReq := api.QueryLatestEventsAndStateRequest{
-		RoomID: roomID,
-		StateToFetch: []gomatrixserverlib.StateKeyTuple{{
-			EventType: gomatrixserverlib.MRoomPowerLevels,
-			StateKey:  "",
-		}},
-	}
-	var queryEventsRes api.QueryLatestEventsAndStateResponse
-	err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
-	if err != nil || len(queryEventsRes.StateEvents) == 0 {
-		util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
-		return jsonerror.InternalServerError()
-	}
-
-	// NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event
-	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
-	if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),
-		}
-	}
-
-	var v roomVisibility
-	if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil {
-		return *reqErr
-	}
-
-	isPublic := v.Visibility == gomatrixserverlib.Public
-	if err := publicRoomsDatabase.SetRoomVisibility(req.Context(), isPublic, roomID); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.SetRoomVisibility failed")
-		return jsonerror.InternalServerError()
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: struct{}{},
-	}
-}
diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go
deleted file mode 100644
index df9df8ff9..000000000
--- a/publicroomsapi/directory/public_rooms.go
+++ /dev/null
@@ -1,262 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package directory
-
-import (
-	"context"
-	"math/rand"
-	"net/http"
-	"sort"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-type PublicRoomReq struct {
-	Since  string `json:"since,omitempty"`
-	Limit  int16  `json:"limit,omitempty"`
-	Filter filter `json:"filter,omitempty"`
-}
-
-type filter struct {
-	SearchTerms string `json:"generic_search_term,omitempty"`
-}
-
-// GetPostPublicRooms implements GET and POST /publicRooms
-func GetPostPublicRooms(
-	req *http.Request, publicRoomDatabase storage.Database,
-) util.JSONResponse {
-	var request PublicRoomReq
-	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
-		return *fillErr
-	}
-	response, err := publicRooms(req.Context(), request, publicRoomDatabase)
-	if err != nil {
-		return jsonerror.InternalServerError()
-	}
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: response,
-	}
-}
-
-// GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied.
-func GetPostPublicRoomsWithExternal(
-	req *http.Request, publicRoomDatabase storage.Database, fedClient *gomatrixserverlib.FederationClient,
-	extRoomsProvider types.ExternalPublicRoomsProvider,
-) util.JSONResponse {
-	var request PublicRoomReq
-	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
-		return *fillErr
-	}
-	response, err := publicRooms(req.Context(), request, publicRoomDatabase)
-	if err != nil {
-		return jsonerror.InternalServerError()
-	}
-
-	if request.Since != "" {
-		// TODO: handle pagination tokens sensibly rather than ignoring them.
-		// ignore paginated requests since we don't handle them yet over federation.
-		// Only the initial request will contain federated rooms.
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: response,
-		}
-	}
-
-	// If we have already hit the limit on the number of rooms, bail.
-	var limit int
-	if request.Limit > 0 {
-		limit = int(request.Limit) - len(response.Chunk)
-		if limit <= 0 {
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: response,
-			}
-		}
-	}
-
-	// downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16
-	fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit))
-	response.Chunk = append(response.Chunk, fedRooms...)
-
-	// de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers
-	// are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit)
-	var publicRooms []gomatrixserverlib.PublicRoom
-	haveRoomIDs := make(map[string]bool)
-	rand.Shuffle(len(response.Chunk), func(i, j int) {
-		response.Chunk[i], response.Chunk[j] = response.Chunk[j], response.Chunk[i]
-	})
-	for _, r := range response.Chunk {
-		if haveRoomIDs[r.RoomID] {
-			continue
-		}
-		haveRoomIDs[r.RoomID] = true
-		publicRooms = append(publicRooms, r)
-	}
-	// sort by member count
-	sort.SliceStable(publicRooms, func(i, j int) bool {
-		return publicRooms[i].JoinedMembersCount > publicRooms[j].JoinedMembersCount
-	})
-
-	response.Chunk = publicRooms
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: response,
-	}
-}
-
-// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
-// Returns a list of public rooms up to the limit specified.
-func bulkFetchPublicRoomsFromServers(
-	ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string, limit int16,
-) (publicRooms []gomatrixserverlib.PublicRoom) {
-	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
-	// goroutines send rooms to this channel
-	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
-	// signalling channel to tell goroutines to stop sending rooms and quit
-	done := make(chan bool)
-	// signalling to say when we can close the room channel
-	var wg sync.WaitGroup
-	wg.Add(len(homeservers))
-	// concurrently query for public rooms
-	for _, hs := range homeservers {
-		go func(homeserverDomain string) {
-			defer wg.Done()
-			util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
-			fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "")
-			if err != nil {
-				util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn(
-					"bulkFetchPublicRoomsFromServers: failed to query hs",
-				)
-				return
-			}
-			for _, room := range fres.Chunk {
-				// atomically send a room or stop
-				select {
-				case roomCh <- room:
-				case <-done:
-					util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms")
-					return
-				}
-			}
-		}(hs)
-	}
-
-	// Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request.
-	// This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be
-	// closed.
-	go func() {
-		wg.Wait()
-		util.GetLogger(ctx).Info("Cleaning up resources")
-		close(roomCh)
-	}()
-
-	// fan-in results with timeout. We stop when we reach the limit.
-FanIn:
-	for len(publicRooms) < int(limit) || limit == 0 {
-		// add a room or timeout
-		select {
-		case room, ok := <-roomCh:
-			if !ok {
-				util.GetLogger(ctx).Info("All homeservers have been queried, returning results.")
-				break FanIn
-			}
-			publicRooms = append(publicRooms, room)
-		case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got.
-			util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early")
-			break FanIn
-		case <-ctx.Done(): // the client hung up on us, let's stop.
-			util.GetLogger(ctx).Info("Client hung up, returning early")
-			break FanIn
-		}
-	}
-	// tell goroutines to stop
-	close(done)
-
-	return publicRooms
-}
-
-func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase storage.Database) (*gomatrixserverlib.RespPublicRooms, error) {
-	var response gomatrixserverlib.RespPublicRooms
-	var limit int16
-	var offset int64
-	limit = request.Limit
-	offset, err := strconv.ParseInt(request.Since, 10, 64)
-	// ParseInt returns 0 and an error when trying to parse an empty string
-	// In that case, we want to assign 0 so we ignore the error
-	if err != nil && len(request.Since) > 0 {
-		util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed")
-		return nil, err
-	}
-
-	est, err := publicRoomDatabase.CountPublicRooms(ctx)
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed")
-		return nil, err
-	}
-	response.TotalRoomCountEstimate = int(est)
-
-	if offset > 0 {
-		response.PrevBatch = strconv.Itoa(int(offset) - 1)
-	}
-	nextIndex := int(offset) + int(limit)
-	if response.TotalRoomCountEstimate > nextIndex {
-		response.NextBatch = strconv.Itoa(nextIndex)
-	}
-
-	if response.Chunk, err = publicRoomDatabase.GetPublicRooms(
-		ctx, offset, limit, request.Filter.SearchTerms,
-	); err != nil {
-		util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed")
-		return nil, err
-	}
-
-	return &response, nil
-}
-
-// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request
-// on /publicRooms by parsing the incoming HTTP request
-// Filter is only filled for POST requests
-func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse {
-	if httpReq.Method == http.MethodGet {
-		limit, err := strconv.Atoi(httpReq.FormValue("limit"))
-		// Atoi returns 0 and an error when trying to parse an empty string
-		// In that case, we want to assign 0 so we ignore the error
-		if err != nil && len(httpReq.FormValue("limit")) > 0 {
-			util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
-			reqErr := jsonerror.InternalServerError()
-			return &reqErr
-		}
-		request.Limit = int16(limit)
-		request.Since = httpReq.FormValue("since")
-		return nil
-	} else if httpReq.Method == http.MethodPost {
-		return httputil.UnmarshalJSONRequest(httpReq, request)
-	}
-
-	return &util.JSONResponse{
-		Code: http.StatusMethodNotAllowed,
-		JSON: jsonerror.NotFound("Bad method"),
-	}
-}
diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go
deleted file mode 100644
index b9baa1056..000000000
--- a/publicroomsapi/publicroomsapi.go
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package publicroomsapi
-
-import (
-	"github.com/Shopify/sarama"
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/publicroomsapi/consumers"
-	"github.com/matrix-org/dendrite/publicroomsapi/routing"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
-)
-
-// AddPublicRoutes sets up and registers HTTP handlers for the PublicRoomsAPI
-// component.
-func AddPublicRoutes(
-	router *mux.Router,
-	cfg *config.Dendrite,
-	consumer sarama.Consumer,
-	userAPI userapi.UserInternalAPI,
-	publicRoomsDB storage.Database,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
-	fedClient *gomatrixserverlib.FederationClient,
-	extRoomsProvider types.ExternalPublicRoomsProvider,
-) {
-	rsConsumer := consumers.NewOutputRoomEventConsumer(
-		cfg, consumer, publicRoomsDB, rsAPI,
-	)
-	if err := rsConsumer.Start(); err != nil {
-		logrus.WithError(err).Panic("failed to start public rooms server consumer")
-	}
-
-	routing.Setup(router, userAPI, publicRoomsDB, rsAPI, fedClient, extRoomsProvider)
-}
diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go
deleted file mode 100644
index 9c82d3508..000000000
--- a/publicroomsapi/routing/routing.go
+++ /dev/null
@@ -1,79 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package routing
-
-import (
-	"net/http"
-
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/publicroomsapi/directory"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage"
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-const pathPrefixR0 = "/client/r0"
-
-// Setup configures the given mux with publicroomsapi server listeners
-//
-// Due to Setup being used to call many other functions, a gocyclo nolint is
-// applied:
-// nolint: gocyclo
-func Setup(
-	publicAPIMux *mux.Router, userAPI userapi.UserInternalAPI, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI,
-	fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider,
-) {
-	r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter()
-
-	r0mux.Handle("/directory/list/room/{roomID}",
-		httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return directory.GetVisibility(req, publicRoomsDB, vars["roomID"])
-		}),
-	).Methods(http.MethodGet, http.MethodOptions)
-	// TODO: Add AS support
-	r0mux.Handle("/directory/list/room/{roomID}",
-		httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return directory.SetVisibility(req, publicRoomsDB, rsAPI, device, vars["roomID"])
-		}),
-	).Methods(http.MethodPut, http.MethodOptions)
-	r0mux.Handle("/publicRooms",
-		httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse {
-			if extRoomsProvider != nil {
-				return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider)
-			}
-			return directory.GetPostPublicRooms(req, publicRoomsDB)
-		}),
-	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
-
-	// Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is.
-	publicAPIMux.Handle("/federation/v1/publicRooms",
-		httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse {
-			return directory.GetPostPublicRooms(req, publicRoomsDB)
-		}),
-	).Methods(http.MethodGet)
-}
diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go
deleted file mode 100644
index 0ca6f455c..000000000
--- a/publicroomsapi/storage/interface.go
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package storage
-
-import (
-	"context"
-
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-type Database interface {
-	internal.PartitionStorer
-	GetRoomVisibility(ctx context.Context, roomID string) (bool, error)
-	SetRoomVisibility(ctx context.Context, visible bool, roomID string) error
-	CountPublicRooms(ctx context.Context) (int64, error)
-	GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error)
-	UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error
-	UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error
-}
diff --git a/publicroomsapi/storage/postgres/prepare.go b/publicroomsapi/storage/postgres/prepare.go
deleted file mode 100644
index 70b6e5161..000000000
--- a/publicroomsapi/storage/postgres/prepare.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"database/sql"
-)
-
-// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
-type statementList []struct {
-	statement **sql.Stmt
-	sql       string
-}
-
-// prepare the SQL for each statement in the list and assign the result to the prepared statement.
-func (s statementList) prepare(db *sql.DB) (err error) {
-	for _, statement := range s {
-		if *statement.statement, err = db.Prepare(statement.sql); err != nil {
-			return
-		}
-	}
-	return
-}
diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go
deleted file mode 100644
index 39e355368..000000000
--- a/publicroomsapi/storage/postgres/public_rooms_table.go
+++ /dev/null
@@ -1,280 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/lib/pq"
-)
-
-var editableAttributes = []string{
-	"aliases",
-	"canonical_alias",
-	"name",
-	"topic",
-	"world_readable",
-	"guest_can_join",
-	"avatar_url",
-	"visibility",
-}
-
-const publicRoomsSchema = `
--- Stores all of the rooms with data needed to create the server's room directory
-CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms(
-	-- The room's ID
-	room_id TEXT NOT NULL PRIMARY KEY,
-	-- Number of joined members in the room
-	joined_members INTEGER NOT NULL DEFAULT 0,
-	-- Aliases of the room (empty array if none)
-	aliases TEXT[] NOT NULL DEFAULT '{}'::TEXT[],
-	-- Canonical alias of the room (empty string if none)
-	canonical_alias TEXT NOT NULL DEFAULT '',
-	-- Name of the room (empty string if none)
-	name TEXT NOT NULL DEFAULT '',
-	-- Topic of the room (empty string if none)
-	topic TEXT NOT NULL DEFAULT '',
-	-- Is the room world readable?
-	world_readable BOOLEAN NOT NULL DEFAULT false,
-	-- Can guest join the room?
-	guest_can_join BOOLEAN NOT NULL DEFAULT false,
-	-- URL of the room avatar (empty string if none)
-	avatar_url TEXT NOT NULL DEFAULT '',
-	-- Visibility of the room: true means the room is publicly visible, false
-	-- means the room is private
-	visibility BOOLEAN NOT NULL DEFAULT false
-);
-`
-
-const countPublicRoomsSQL = "" +
-	"SELECT COUNT(*) FROM publicroomsapi_public_rooms" +
-	" WHERE visibility = true"
-
-const selectPublicRoomsSQL = "" +
-	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
-	" FROM publicroomsapi_public_rooms WHERE visibility = true" +
-	" ORDER BY joined_members DESC" +
-	" OFFSET $1"
-
-const selectPublicRoomsWithLimitSQL = "" +
-	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
-	" FROM publicroomsapi_public_rooms WHERE visibility = true" +
-	" ORDER BY joined_members DESC" +
-	" OFFSET $1 LIMIT $2"
-
-const selectPublicRoomsWithFilterSQL = "" +
-	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
-	" FROM publicroomsapi_public_rooms" +
-	" WHERE visibility = true" +
-	" AND (LOWER(name) LIKE LOWER($1)" +
-	" OR LOWER(topic) LIKE LOWER($1)" +
-	" OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" +
-	" ORDER BY joined_members DESC" +
-	" OFFSET $2"
-
-const selectPublicRoomsWithLimitAndFilterSQL = "" +
-	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
-	" FROM publicroomsapi_public_rooms" +
-	" WHERE visibility = true" +
-	" AND (LOWER(name) LIKE LOWER($1)" +
-	" OR LOWER(topic) LIKE LOWER($1)" +
-	" OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" +
-	" ORDER BY joined_members DESC" +
-	" OFFSET $2 LIMIT $3"
-
-const selectRoomVisibilitySQL = "" +
-	"SELECT visibility FROM publicroomsapi_public_rooms" +
-	" WHERE room_id = $1"
-
-const insertNewRoomSQL = "" +
-	"INSERT INTO publicroomsapi_public_rooms(room_id)" +
-	" VALUES ($1)"
-
-const incrementJoinedMembersInRoomSQL = "" +
-	"UPDATE publicroomsapi_public_rooms" +
-	" SET joined_members = joined_members + 1" +
-	" WHERE room_id = $1"
-
-const decrementJoinedMembersInRoomSQL = "" +
-	"UPDATE publicroomsapi_public_rooms" +
-	" SET joined_members = joined_members - 1" +
-	" WHERE room_id = $1"
-
-const updateRoomAttributeSQL = "" +
-	"UPDATE publicroomsapi_public_rooms" +
-	" SET %s = $1" +
-	" WHERE room_id = $2"
-
-type publicRoomsStatements struct {
-	countPublicRoomsStmt                    *sql.Stmt
-	selectPublicRoomsStmt                   *sql.Stmt
-	selectPublicRoomsWithLimitStmt          *sql.Stmt
-	selectPublicRoomsWithFilterStmt         *sql.Stmt
-	selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt
-	selectRoomVisibilityStmt                *sql.Stmt
-	insertNewRoomStmt                       *sql.Stmt
-	incrementJoinedMembersInRoomStmt        *sql.Stmt
-	decrementJoinedMembersInRoomStmt        *sql.Stmt
-	updateRoomAttributeStmts                map[string]*sql.Stmt
-}
-
-func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(publicRoomsSchema)
-	if err != nil {
-		return
-	}
-
-	stmts := statementList{
-		{&s.countPublicRoomsStmt, countPublicRoomsSQL},
-		{&s.selectPublicRoomsStmt, selectPublicRoomsSQL},
-		{&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL},
-		{&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL},
-		{&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL},
-		{&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL},
-		{&s.insertNewRoomStmt, insertNewRoomSQL},
-		{&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL},
-		{&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL},
-	}
-
-	if err = stmts.prepare(db); err != nil {
-		return
-	}
-
-	s.updateRoomAttributeStmts = make(map[string]*sql.Stmt)
-	for _, editable := range editableAttributes {
-		stmt := fmt.Sprintf(updateRoomAttributeSQL, editable)
-		if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil {
-			return
-		}
-	}
-
-	return
-}
-
-func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) {
-	err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb)
-	return
-}
-
-func (s *publicRoomsStatements) selectPublicRooms(
-	ctx context.Context, offset int64, limit int16, filter string,
-) ([]gomatrixserverlib.PublicRoom, error) {
-	var rows *sql.Rows
-	var err error
-
-	if len(filter) > 0 {
-		pattern := "%" + filter + "%"
-		if limit == 0 {
-			rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext(
-				ctx, pattern, offset,
-			)
-		} else {
-			rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext(
-				ctx, pattern, offset, limit,
-			)
-		}
-	} else {
-		if limit == 0 {
-			rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset)
-		} else {
-			rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext(
-				ctx, offset, limit,
-			)
-		}
-	}
-
-	if err != nil {
-		return []gomatrixserverlib.PublicRoom{}, nil
-	}
-	defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed")
-
-	rooms := []gomatrixserverlib.PublicRoom{}
-	for rows.Next() {
-		var r gomatrixserverlib.PublicRoom
-		var aliases pq.StringArray
-
-		err = rows.Scan(
-			&r.RoomID, &r.JoinedMembersCount, &aliases, &r.CanonicalAlias,
-			&r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL,
-		)
-		if err != nil {
-			return rooms, err
-		}
-
-		r.Aliases = aliases
-
-		rooms = append(rooms, r)
-	}
-
-	return rooms, rows.Err()
-}
-
-func (s *publicRoomsStatements) selectRoomVisibility(
-	ctx context.Context, roomID string,
-) (v bool, err error) {
-	err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v)
-	return
-}
-
-func (s *publicRoomsStatements) insertNewRoom(
-	ctx context.Context, roomID string,
-) error {
-	_, err := s.insertNewRoomStmt.ExecContext(ctx, roomID)
-	return err
-}
-
-func (s *publicRoomsStatements) incrementJoinedMembersInRoom(
-	ctx context.Context, roomID string,
-) error {
-	_, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
-	return err
-}
-
-func (s *publicRoomsStatements) decrementJoinedMembersInRoom(
-	ctx context.Context, roomID string,
-) error {
-	_, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
-	return err
-}
-
-func (s *publicRoomsStatements) updateRoomAttribute(
-	ctx context.Context, attrName string, attrValue attributeValue, roomID string,
-) error {
-	stmt, isEditable := s.updateRoomAttributeStmts[attrName]
-
-	if !isEditable {
-		return errors.New("Cannot edit " + attrName)
-	}
-
-	var value interface{}
-	switch v := attrValue.(type) {
-	case []string:
-		value = pq.StringArray(v)
-	case bool, string:
-		value = attrValue
-	default:
-		return errors.New("Unsupported attribute type, must be bool, string or []string")
-	}
-
-	_, err := stmt.ExecContext(ctx, value, roomID)
-	return err
-}
diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go
deleted file mode 100644
index 36c6aec64..000000000
--- a/publicroomsapi/storage/postgres/storage.go
+++ /dev/null
@@ -1,259 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-
-	"github.com/matrix-org/dendrite/internal/eventutil"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// PublicRoomsServerDatabase represents a public rooms server database.
-type PublicRoomsServerDatabase struct {
-	db *sql.DB
-	sqlutil.PartitionOffsetStatements
-	statements      publicRoomsStatements
-	localServerName gomatrixserverlib.ServerName
-}
-
-type attributeValue interface{}
-
-// NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
-	var db *sql.DB
-	var err error
-	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
-		return nil, err
-	}
-	storage := PublicRoomsServerDatabase{
-		db:              db,
-		localServerName: localServerName,
-	}
-	if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil {
-		return nil, err
-	}
-	if err = storage.statements.prepare(db); err != nil {
-		return nil, err
-	}
-	return &storage, nil
-}
-
-// GetRoomVisibility returns the room visibility as a boolean: true if the room
-// is publicly visible, false if not.
-// Returns an error if the retrieval failed.
-func (d *PublicRoomsServerDatabase) GetRoomVisibility(
-	ctx context.Context, roomID string,
-) (bool, error) {
-	return d.statements.selectRoomVisibility(ctx, roomID)
-}
-
-// SetRoomVisibility updates the visibility attribute of a room. This attribute
-// must be set to true if the room is publicly visible, false if not.
-// Returns an error if the update failed.
-func (d *PublicRoomsServerDatabase) SetRoomVisibility(
-	ctx context.Context, visible bool, roomID string,
-) error {
-	return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID)
-}
-
-// CountPublicRooms returns the number of room set as publicly visible on the server.
-// Returns an error if the retrieval failed.
-func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
-	return d.statements.countPublicRooms(ctx)
-}
-
-// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number
-// of joined members. This array can be limited by a given number of elements, and offset by a given value.
-// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all
-// the rooms set as publicly visible on the server.
-// Returns an error if the retrieval failed.
-func (d *PublicRoomsServerDatabase) GetPublicRooms(
-	ctx context.Context, offset int64, limit int16, filter string,
-) ([]gomatrixserverlib.PublicRoom, error) {
-	return d.statements.selectPublicRooms(ctx, offset, limit, filter)
-}
-
-// UpdateRoomFromEvents iterate over a slice of state events and call
-// UpdateRoomFromEvent on each of them to update the database representation of
-// the rooms updated by each event.
-// The slice of events to remove is used to update the number of joined members
-// for the room in the database.
-// If the update triggered by one of the events failed, aborts the process and
-// returns an error.
-func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(
-	ctx context.Context,
-	eventsToAdd []gomatrixserverlib.Event,
-	eventsToRemove []gomatrixserverlib.Event,
-) error {
-	for _, event := range eventsToAdd {
-		if err := d.UpdateRoomFromEvent(ctx, event); err != nil {
-			return err
-		}
-	}
-
-	for _, event := range eventsToRemove {
-		if event.Type() == "m.room.member" {
-			if err := d.updateNumJoinedUsers(ctx, event, true); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by
-// checking the event's type to know which attribute to change and using the event's content
-// to define the new value of the attribute.
-// If the event doesn't match with any property used to compute the public room directory,
-// does nothing.
-// If something went wrong during the process, returns an error.
-func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(
-	ctx context.Context, event gomatrixserverlib.Event,
-) error {
-	// Process the event according to its type
-	switch event.Type() {
-	case "m.room.create":
-		return d.statements.insertNewRoom(ctx, event.RoomID())
-	case "m.room.member":
-		return d.updateNumJoinedUsers(ctx, event, false)
-	case "m.room.aliases":
-		return d.updateRoomAliases(ctx, event)
-	case "m.room.canonical_alias":
-		var content eventutil.CanonicalAliasContent
-		field := &(content.Alias)
-		attrName := "canonical_alias"
-		return d.updateStringAttribute(ctx, attrName, event, &content, field)
-	case "m.room.name":
-		var content eventutil.NameContent
-		field := &(content.Name)
-		attrName := "name"
-		return d.updateStringAttribute(ctx, attrName, event, &content, field)
-	case "m.room.topic":
-		var content eventutil.TopicContent
-		field := &(content.Topic)
-		attrName := "topic"
-		return d.updateStringAttribute(ctx, attrName, event, &content, field)
-	case "m.room.avatar":
-		var content eventutil.AvatarContent
-		field := &(content.URL)
-		attrName := "avatar_url"
-		return d.updateStringAttribute(ctx, attrName, event, &content, field)
-	case "m.room.history_visibility":
-		var content eventutil.HistoryVisibilityContent
-		field := &(content.HistoryVisibility)
-		attrName := "world_readable"
-		strForTrue := "world_readable"
-		return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
-	case "m.room.guest_access":
-		var content eventutil.GuestAccessContent
-		field := &(content.GuestAccess)
-		attrName := "guest_can_join"
-		strForTrue := "can_join"
-		return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
-	}
-
-	// If the event type didn't match, return with no error
-	return nil
-}
-
-// updateNumJoinedUsers updates the number of joined user in the database representation
-// of a room using a given "m.room.member" Matrix event.
-// If the membership property of the event isn't "join", ignores it and returs nil.
-// If the remove parameter is set to false, increments the joined members counter in the
-// database, if set to truem decrements it.
-// Returns an error if the update failed.
-func (d *PublicRoomsServerDatabase) updateNumJoinedUsers(
-	ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool,
-) error {
-	membership, err := membershipEvent.Membership()
-	if err != nil {
-		return err
-	}
-
-	if membership != gomatrixserverlib.Join {
-		return nil
-	}
-
-	if remove {
-		return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
-	}
-	return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
-}
-
-// updateStringAttribute updates a given string attribute in the database
-// representation of a room using a given string data field from content of the
-// Matrix event triggering the update.
-// Returns an error if decoding the Matrix event's content or updating the attribute
-// failed.
-func (d *PublicRoomsServerDatabase) updateStringAttribute(
-	ctx context.Context, attrName string, event gomatrixserverlib.Event,
-	content interface{}, field *string,
-) error {
-	if err := json.Unmarshal(event.Content(), content); err != nil {
-		return err
-	}
-
-	return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID())
-}
-
-// updateBooleanAttribute updates a given boolean attribute in the database
-// representation of a room using a given string data field from content of the
-// Matrix event triggering the update.
-// The attribute is set to true if the field matches a given string, false if not.
-// Returns an error if decoding the Matrix event's content or updating the attribute
-// failed.
-func (d *PublicRoomsServerDatabase) updateBooleanAttribute(
-	ctx context.Context, attrName string, event gomatrixserverlib.Event,
-	content interface{}, field *string, strForTrue string,
-) error {
-	if err := json.Unmarshal(event.Content(), content); err != nil {
-		return err
-	}
-
-	var attrValue bool
-	if *field == strForTrue {
-		attrValue = true
-	} else {
-		attrValue = false
-	}
-
-	return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID())
-}
-
-// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of
-// a given room with it.
-// Returns an error if decoding the Matrix event or updating the list failed.
-func (d *PublicRoomsServerDatabase) updateRoomAliases(
-	ctx context.Context, aliasesEvent gomatrixserverlib.Event,
-) error {
-	if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) {
-		return nil // only store our own aliases
-	}
-	var content eventutil.AliasesContent
-	if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil {
-		return err
-	}
-
-	return d.statements.updateRoomAttribute(
-		ctx, "aliases", content.Aliases, aliasesEvent.RoomID(),
-	)
-}
diff --git a/publicroomsapi/storage/sqlite3/prepare.go b/publicroomsapi/storage/sqlite3/prepare.go
deleted file mode 100644
index 482dfa2b9..000000000
--- a/publicroomsapi/storage/sqlite3/prepare.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"database/sql"
-)
-
-// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
-type statementList []struct {
-	statement **sql.Stmt
-	sql       string
-}
-
-// prepare the SQL for each statement in the list and assign the result to the prepared statement.
-func (s statementList) prepare(db *sql.DB) (err error) {
-	for _, statement := range s {
-		if *statement.statement, err = db.Prepare(statement.sql); err != nil {
-			return
-		}
-	}
-	return
-}
diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go
deleted file mode 100644
index 7b332e175..000000000
--- a/publicroomsapi/storage/sqlite3/public_rooms_table.go
+++ /dev/null
@@ -1,273 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"errors"
-	"fmt"
-
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-var editableAttributes = []string{
-	"aliases",
-	"canonical_alias",
-	"name",
-	"topic",
-	"world_readable",
-	"guest_can_join",
-	"avatar_url",
-	"visibility",
-}
-
-const publicRoomsSchema = `
--- Stores all of the rooms with data needed to create the server's room directory
-CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms(
-	room_id TEXT NOT NULL PRIMARY KEY,
-	joined_members INTEGER NOT NULL DEFAULT 0,
-	aliases TEXT NOT NULL DEFAULT '',
-	canonical_alias TEXT NOT NULL DEFAULT '',
-	name TEXT NOT NULL DEFAULT '',
-	topic TEXT NOT NULL DEFAULT '',
-	world_readable BOOLEAN NOT NULL DEFAULT false,
-	guest_can_join BOOLEAN NOT NULL DEFAULT false,
-	avatar_url TEXT NOT NULL DEFAULT '',
-	visibility BOOLEAN NOT NULL DEFAULT false
-);
-`
-
-const countPublicRoomsSQL = "" +
-	"SELECT COUNT(*) FROM publicroomsapi_public_rooms" +
-	" WHERE visibility = true"
-
-const selectPublicRoomsSQL = "" +
-	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
-	" FROM publicroomsapi_public_rooms WHERE visibility = true" +
-	" ORDER BY joined_members DESC" +
-	" LIMIT 30 OFFSET $1"
-
-const selectPublicRoomsWithLimitSQL = "" +
-	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
-	" FROM publicroomsapi_public_rooms WHERE visibility = true" +
-	" ORDER BY joined_members DESC" +
-	" LIMIT $1 OFFSET $2"
-
-const selectPublicRoomsWithFilterSQL = "" +
-	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
-	" FROM publicroomsapi_public_rooms" +
-	" WHERE visibility = true" +
-	" AND (LOWER(name) LIKE LOWER($1)" +
-	" OR LOWER(topic) LIKE LOWER($1)" +
-	" OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases?
-	" ORDER BY joined_members DESC" +
-	" LIMIT 30 OFFSET $2"
-
-const selectPublicRoomsWithLimitAndFilterSQL = "" +
-	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
-	" FROM publicroomsapi_public_rooms" +
-	" WHERE visibility = true" +
-	" AND (LOWER(name) LIKE LOWER($1)" +
-	" OR LOWER(topic) LIKE LOWER($1)" +
-	" OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases?
-	" ORDER BY joined_members DESC" +
-	" LIMIT $3 OFFSET $2"
-
-const selectRoomVisibilitySQL = "" +
-	"SELECT visibility FROM publicroomsapi_public_rooms" +
-	" WHERE room_id = $1"
-
-const insertNewRoomSQL = "" +
-	"INSERT INTO publicroomsapi_public_rooms(room_id)" +
-	" VALUES ($1)"
-
-const incrementJoinedMembersInRoomSQL = "" +
-	"UPDATE publicroomsapi_public_rooms" +
-	" SET joined_members = joined_members + 1" +
-	" WHERE room_id = $1"
-
-const decrementJoinedMembersInRoomSQL = "" +
-	"UPDATE publicroomsapi_public_rooms" +
-	" SET joined_members = joined_members - 1" +
-	" WHERE room_id = $1"
-
-const updateRoomAttributeSQL = "" +
-	"UPDATE publicroomsapi_public_rooms" +
-	" SET %s = $1" +
-	" WHERE room_id = $2"
-
-type publicRoomsStatements struct {
-	countPublicRoomsStmt                    *sql.Stmt
-	selectPublicRoomsStmt                   *sql.Stmt
-	selectPublicRoomsWithLimitStmt          *sql.Stmt
-	selectPublicRoomsWithFilterStmt         *sql.Stmt
-	selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt
-	selectRoomVisibilityStmt                *sql.Stmt
-	insertNewRoomStmt                       *sql.Stmt
-	incrementJoinedMembersInRoomStmt        *sql.Stmt
-	decrementJoinedMembersInRoomStmt        *sql.Stmt
-	updateRoomAttributeStmts                map[string]*sql.Stmt
-}
-
-func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(publicRoomsSchema)
-	if err != nil {
-		return
-	}
-
-	stmts := statementList{
-		{&s.countPublicRoomsStmt, countPublicRoomsSQL},
-		{&s.selectPublicRoomsStmt, selectPublicRoomsSQL},
-		{&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL},
-		{&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL},
-		{&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL},
-		{&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL},
-		{&s.insertNewRoomStmt, insertNewRoomSQL},
-		{&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL},
-		{&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL},
-	}
-
-	if err = stmts.prepare(db); err != nil {
-		return
-	}
-
-	s.updateRoomAttributeStmts = make(map[string]*sql.Stmt)
-	for _, editable := range editableAttributes {
-		stmt := fmt.Sprintf(updateRoomAttributeSQL, editable)
-		if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil {
-			return
-		}
-	}
-
-	return
-}
-
-func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) {
-	err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb)
-	return
-}
-
-func (s *publicRoomsStatements) selectPublicRooms(
-	ctx context.Context, offset int64, limit int16, filter string,
-) ([]gomatrixserverlib.PublicRoom, error) {
-	var rows *sql.Rows
-	var err error
-
-	if len(filter) > 0 {
-		pattern := "%" + filter + "%"
-		if limit == 0 {
-			rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext(
-				ctx, pattern, offset,
-			)
-		} else {
-			rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext(
-				ctx, pattern, limit, offset,
-			)
-		}
-	} else {
-		if limit == 0 {
-			rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset)
-		} else {
-			rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext(
-				ctx, limit, offset,
-			)
-		}
-	}
-
-	if err != nil {
-		return []gomatrixserverlib.PublicRoom{}, nil
-	}
-	defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows")
-
-	rooms := []gomatrixserverlib.PublicRoom{}
-	for rows.Next() {
-		var r gomatrixserverlib.PublicRoom
-		var aliasesJSON string
-
-		err = rows.Scan(
-			&r.RoomID, &r.JoinedMembersCount, &aliasesJSON, &r.CanonicalAlias,
-			&r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL,
-		)
-		if err != nil {
-			return rooms, err
-		}
-
-		if len(aliasesJSON) > 0 {
-			if err := json.Unmarshal([]byte(aliasesJSON), &r.Aliases); err != nil {
-				return rooms, err
-			}
-		}
-
-		rooms = append(rooms, r)
-	}
-
-	return rooms, nil
-}
-
-func (s *publicRoomsStatements) selectRoomVisibility(
-	ctx context.Context, roomID string,
-) (v bool, err error) {
-	err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v)
-	return
-}
-
-func (s *publicRoomsStatements) insertNewRoom(
-	ctx context.Context, roomID string,
-) error {
-	_, err := s.insertNewRoomStmt.ExecContext(ctx, roomID)
-	return err
-}
-
-func (s *publicRoomsStatements) incrementJoinedMembersInRoom(
-	ctx context.Context, roomID string,
-) error {
-	_, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
-	return err
-}
-
-func (s *publicRoomsStatements) decrementJoinedMembersInRoom(
-	ctx context.Context, roomID string,
-) error {
-	_, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
-	return err
-}
-
-func (s *publicRoomsStatements) updateRoomAttribute(
-	ctx context.Context, attrName string, attrValue attributeValue, roomID string,
-) error {
-	stmt, isEditable := s.updateRoomAttributeStmts[attrName]
-
-	if !isEditable {
-		return errors.New("Cannot edit " + attrName)
-	}
-
-	var value interface{}
-	switch v := attrValue.(type) {
-	case []string:
-		b, _ := json.Marshal(v)
-		value = string(b)
-	case bool, string:
-		value = attrValue
-	default:
-		return errors.New("Unsupported attribute type, must be bool, string or []string")
-	}
-
-	_, err := stmt.ExecContext(ctx, value, roomID)
-	return err
-}
diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go
deleted file mode 100644
index 5c685d131..000000000
--- a/publicroomsapi/storage/sqlite3/storage.go
+++ /dev/null
@@ -1,265 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-
-	_ "github.com/mattn/go-sqlite3"
-
-	"github.com/matrix-org/dendrite/internal/eventutil"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// PublicRoomsServerDatabase represents a public rooms server database.
-type PublicRoomsServerDatabase struct {
-	db *sql.DB
-	sqlutil.PartitionOffsetStatements
-	statements      publicRoomsStatements
-	localServerName gomatrixserverlib.ServerName
-}
-
-type attributeValue interface{}
-
-// NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
-	var db *sql.DB
-	var err error
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
-	if err != nil {
-		return nil, err
-	}
-	if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
-		return nil, err
-	}
-	storage := PublicRoomsServerDatabase{
-		db:              db,
-		localServerName: localServerName,
-	}
-	if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil {
-		return nil, err
-	}
-	if err = storage.statements.prepare(db); err != nil {
-		return nil, err
-	}
-	return &storage, nil
-}
-
-// GetRoomVisibility returns the room visibility as a boolean: true if the room
-// is publicly visible, false if not.
-// Returns an error if the retrieval failed.
-func (d *PublicRoomsServerDatabase) GetRoomVisibility(
-	ctx context.Context, roomID string,
-) (bool, error) {
-	return d.statements.selectRoomVisibility(ctx, roomID)
-}
-
-// SetRoomVisibility updates the visibility attribute of a room. This attribute
-// must be set to true if the room is publicly visible, false if not.
-// Returns an error if the update failed.
-func (d *PublicRoomsServerDatabase) SetRoomVisibility(
-	ctx context.Context, visible bool, roomID string,
-) error {
-	return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID)
-}
-
-// CountPublicRooms returns the number of room set as publicly visible on the server.
-// Returns an error if the retrieval failed.
-func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
-	return d.statements.countPublicRooms(ctx)
-}
-
-// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number
-// of joined members. This array can be limited by a given number of elements, and offset by a given value.
-// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all
-// the rooms set as publicly visible on the server.
-// Returns an error if the retrieval failed.
-func (d *PublicRoomsServerDatabase) GetPublicRooms(
-	ctx context.Context, offset int64, limit int16, filter string,
-) ([]gomatrixserverlib.PublicRoom, error) {
-	return d.statements.selectPublicRooms(ctx, offset, limit, filter)
-}
-
-// UpdateRoomFromEvents iterate over a slice of state events and call
-// UpdateRoomFromEvent on each of them to update the database representation of
-// the rooms updated by each event.
-// The slice of events to remove is used to update the number of joined members
-// for the room in the database.
-// If the update triggered by one of the events failed, aborts the process and
-// returns an error.
-func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(
-	ctx context.Context,
-	eventsToAdd []gomatrixserverlib.Event,
-	eventsToRemove []gomatrixserverlib.Event,
-) error {
-	for _, event := range eventsToAdd {
-		if err := d.UpdateRoomFromEvent(ctx, event); err != nil {
-			return err
-		}
-	}
-
-	for _, event := range eventsToRemove {
-		if event.Type() == "m.room.member" {
-			if err := d.updateNumJoinedUsers(ctx, event, true); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by
-// checking the event's type to know which attribute to change and using the event's content
-// to define the new value of the attribute.
-// If the event doesn't match with any property used to compute the public room directory,
-// does nothing.
-// If something went wrong during the process, returns an error.
-func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(
-	ctx context.Context, event gomatrixserverlib.Event,
-) error {
-	// Process the event according to its type
-	switch event.Type() {
-	case "m.room.create":
-		return d.statements.insertNewRoom(ctx, event.RoomID())
-	case "m.room.member":
-		return d.updateNumJoinedUsers(ctx, event, false)
-	case "m.room.aliases":
-		return d.updateRoomAliases(ctx, event)
-	case "m.room.canonical_alias":
-		var content eventutil.CanonicalAliasContent
-		field := &(content.Alias)
-		attrName := "canonical_alias"
-		return d.updateStringAttribute(ctx, attrName, event, &content, field)
-	case "m.room.name":
-		var content eventutil.NameContent
-		field := &(content.Name)
-		attrName := "name"
-		return d.updateStringAttribute(ctx, attrName, event, &content, field)
-	case "m.room.topic":
-		var content eventutil.TopicContent
-		field := &(content.Topic)
-		attrName := "topic"
-		return d.updateStringAttribute(ctx, attrName, event, &content, field)
-	case "m.room.avatar":
-		var content eventutil.AvatarContent
-		field := &(content.URL)
-		attrName := "avatar_url"
-		return d.updateStringAttribute(ctx, attrName, event, &content, field)
-	case "m.room.history_visibility":
-		var content eventutil.HistoryVisibilityContent
-		field := &(content.HistoryVisibility)
-		attrName := "world_readable"
-		strForTrue := "world_readable"
-		return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
-	case "m.room.guest_access":
-		var content eventutil.GuestAccessContent
-		field := &(content.GuestAccess)
-		attrName := "guest_can_join"
-		strForTrue := "can_join"
-		return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
-	}
-
-	// If the event type didn't match, return with no error
-	return nil
-}
-
-// updateNumJoinedUsers updates the number of joined user in the database representation
-// of a room using a given "m.room.member" Matrix event.
-// If the membership property of the event isn't "join", ignores it and returs nil.
-// If the remove parameter is set to false, increments the joined members counter in the
-// database, if set to truem decrements it.
-// Returns an error if the update failed.
-func (d *PublicRoomsServerDatabase) updateNumJoinedUsers(
-	ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool,
-) error {
-	membership, err := membershipEvent.Membership()
-	if err != nil {
-		return err
-	}
-
-	if membership != gomatrixserverlib.Join {
-		return nil
-	}
-
-	if remove {
-		return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
-	}
-	return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
-}
-
-// updateStringAttribute updates a given string attribute in the database
-// representation of a room using a given string data field from content of the
-// Matrix event triggering the update.
-// Returns an error if decoding the Matrix event's content or updating the attribute
-// failed.
-func (d *PublicRoomsServerDatabase) updateStringAttribute(
-	ctx context.Context, attrName string, event gomatrixserverlib.Event,
-	content interface{}, field *string,
-) error {
-	if err := json.Unmarshal(event.Content(), content); err != nil {
-		return err
-	}
-
-	return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID())
-}
-
-// updateBooleanAttribute updates a given boolean attribute in the database
-// representation of a room using a given string data field from content of the
-// Matrix event triggering the update.
-// The attribute is set to true if the field matches a given string, false if not.
-// Returns an error if decoding the Matrix event's content or updating the attribute
-// failed.
-func (d *PublicRoomsServerDatabase) updateBooleanAttribute(
-	ctx context.Context, attrName string, event gomatrixserverlib.Event,
-	content interface{}, field *string, strForTrue string,
-) error {
-	if err := json.Unmarshal(event.Content(), content); err != nil {
-		return err
-	}
-
-	var attrValue bool
-	if *field == strForTrue {
-		attrValue = true
-	} else {
-		attrValue = false
-	}
-
-	return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID())
-}
-
-// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of
-// a given room with it.
-// Returns an error if decoding the Matrix event or updating the list failed.
-func (d *PublicRoomsServerDatabase) updateRoomAliases(
-	ctx context.Context, aliasesEvent gomatrixserverlib.Event,
-) error {
-	if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) {
-		return nil // only store our own aliases
-	}
-	var content eventutil.AliasesContent
-	if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil {
-		return err
-	}
-
-	return d.statements.updateRoomAttribute(
-		ctx, "aliases", content.Aliases, aliasesEvent.RoomID(),
-	)
-}
diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go
deleted file mode 100644
index f66188040..000000000
--- a/publicroomsapi/storage/storage.go
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build !wasm
-
-package storage
-
-import (
-	"net/url"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
-	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-const schemePostgres = "postgres"
-const schemeFile = "file"
-
-// NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName)
-	}
-	switch uri.Scheme {
-	case schemePostgres:
-		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName)
-	case schemeFile:
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
-	default:
-		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName)
-	}
-}
diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go
deleted file mode 100644
index 70ceeaf85..000000000
--- a/publicroomsapi/storage/storage_wasm.go
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package storage
-
-import (
-	"fmt"
-	"net/url"
-
-	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return nil, err
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	case "file":
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
-	default:
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	}
-}

From 38caf8e5b7623c090f8949076b57d769e42011ad Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 2 Jul 2020 17:43:07 +0100
Subject: [PATCH 22/23] Yggdrasil+QUIC demo, federation sender tweaks (#1177)

* Initial QUIC work

* Update Yggdrasil demo

* Make sure that the federation sender knows how many pending events are in the database when the worker starts

* QUIC tunables

* pprof

* Don't spin

* Set build info for Yggdrasil
---
 build/gobind/build.sh                         |   2 +-
 cmd/dendrite-demo-yggdrasil/main.go           |   2 +
 cmd/dendrite-demo-yggdrasil/yggconn/node.go   |  63 ++++----
 .../yggconn/session.go                        | 122 ++++++++------
 cmd/dendrite-demo-yggdrasil/yggconn/stream.go |  20 +++
 federationsender/queue/destinationqueue.go    | 115 +++++++------
 federationsender/storage/interface.go         |   1 +
 .../storage/postgres/queue_pdus_table.go      |  23 +++
 federationsender/storage/postgres/storage.go  |   9 ++
 .../storage/sqlite3/queue_pdus_table.go       |  23 +++
 federationsender/storage/sqlite3/storage.go   |   9 ++
 go.mod                                        |   7 +-
 go.sum                                        | 151 ++++++++++++++++--
 13 files changed, 406 insertions(+), 141 deletions(-)
 create mode 100644 cmd/dendrite-demo-yggdrasil/yggconn/stream.go

diff --git a/build/gobind/build.sh b/build/gobind/build.sh
index 3a80d374a..aa2cdfc5a 100644
--- a/build/gobind/build.sh
+++ b/build/gobind/build.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
 gomobile bind -v \
-    -ldflags "-X $github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=riot-ios-p2p" \
+    -ldflags "-X github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=dendrite" \
     -target ios \
     github.com/matrix-org/dendrite/build/gobind
\ No newline at end of file
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 7476d686b..7a527d87d 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -31,6 +31,7 @@ import (
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/setup"
@@ -50,6 +51,7 @@ var (
 // nolint:gocyclo
 func main() {
 	flag.Parse()
+	internal.SetupPprof()
 
 	ygg, err := yggconn.Setup(*instanceName, *instancePeer, ".")
 	if err != nil {
diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
index 18d207a9e..eb176493e 100644
--- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go
+++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
@@ -17,6 +17,7 @@ package yggconn
 import (
 	"context"
 	"crypto/ed25519"
+	"crypto/tls"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -26,10 +27,11 @@ import (
 	"os"
 	"strings"
 	"sync"
+	"time"
 
+	"github.com/lucas-clemente/quic-go"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
 
-	"github.com/libp2p/go-yamux"
 	yggdrasiladmin "github.com/yggdrasil-network/yggdrasil-go/src/admin"
 	yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config"
 	yggdrasilmulticast "github.com/yggdrasil-network/yggdrasil-go/src/multicast"
@@ -39,16 +41,26 @@ import (
 )
 
 type Node struct {
-	core      *yggdrasil.Core
-	config    *yggdrasilconfig.NodeConfig
-	state     *yggdrasilconfig.NodeState
-	admin     *yggdrasiladmin.AdminSocket
-	multicast *yggdrasilmulticast.Multicast
-	log       *gologme.Logger
-	listener  *yggdrasil.Listener
-	dialer    *yggdrasil.Dialer
-	sessions  sync.Map // string -> yamux.Session
-	incoming  chan *yamux.Stream
+	core       *yggdrasil.Core
+	config     *yggdrasilconfig.NodeConfig
+	state      *yggdrasilconfig.NodeState
+	admin      *yggdrasiladmin.AdminSocket
+	multicast  *yggdrasilmulticast.Multicast
+	log        *gologme.Logger
+	packetConn *yggdrasil.PacketConn
+	listener   quic.Listener
+	tlsConfig  *tls.Config
+	quicConfig *quic.Config
+	sessions   sync.Map // string -> quic.Session
+	incoming   chan QUICStream
+}
+
+func (n *Node) BuildName() string {
+	return "dendrite"
+}
+
+func (n *Node) BuildVersion() string {
+	return "dev"
 }
 
 func (n *Node) Dialer(_, address string) (net.Conn, error) {
@@ -74,8 +86,9 @@ func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) {
 		admin:     &yggdrasiladmin.AdminSocket{},
 		multicast: &yggdrasilmulticast.Multicast{},
 		log:       gologme.New(os.Stdout, "YGG ", log.Flags()),
-		incoming:  make(chan *yamux.Stream),
+		incoming:  make(chan QUICStream),
 	}
+	n.core.SetBuildInfo(n)
 
 	yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
 	if _, err := os.Stat(yggfile); !os.IsNotExist(err) {
@@ -114,29 +127,21 @@ func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) {
 			panic(err)
 		}
 	}
-	/*
-		if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil {
-			panic(err)
-		}
-		if err = n.admin.Start(); err != nil {
-			panic(err)
-		}
-	*/
 	if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil {
 		panic(err)
 	}
 	if err = n.multicast.Start(); err != nil {
 		panic(err)
 	}
-	//n.admin.SetupAdminHandlers(n.admin)
-	//n.multicast.SetupAdminHandlers(n.admin)
-	n.listener, err = n.core.ConnListen()
-	if err != nil {
-		panic(err)
-	}
-	n.dialer, err = n.core.ConnDialer()
-	if err != nil {
-		panic(err)
+
+	n.packetConn = n.core.PacketConn()
+	n.tlsConfig = n.generateTLSConfig()
+	n.quicConfig = &quic.Config{
+		MaxIncomingStreams:    0,
+		MaxIncomingUniStreams: 0,
+		KeepAlive:             true,
+		MaxIdleTimeout:        time.Second * 120,
+		HandshakeTimeout:      time.Second * 30,
 	}
 
 	n.log.Println("Public curve25519:", n.core.EncryptionPublicKey())
diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/session.go b/cmd/dendrite-demo-yggdrasil/yggconn/session.go
index c50b6b73c..857b2cc9c 100644
--- a/cmd/dendrite-demo-yggdrasil/yggconn/session.go
+++ b/cmd/dendrite-demo-yggdrasil/yggconn/session.go
@@ -16,60 +16,52 @@ package yggconn
 
 import (
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/pem"
+	"errors"
+	"math/big"
 	"net"
-	"strings"
 	"time"
 
-	"github.com/libp2p/go-yamux"
+	"github.com/lucas-clemente/quic-go"
+	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 )
 
-func (n *Node) yamuxConfig() *yamux.Config {
-	cfg := yamux.DefaultConfig()
-	cfg.EnableKeepAlive = false
-	cfg.ConnectionWriteTimeout = time.Second * 15
-	cfg.MaxMessageSize = 65535
-	cfg.ReadBufSize = 655350
-	return cfg
-}
-
 func (n *Node) listenFromYgg() {
+	var err error
+	n.listener, err = quic.Listen(
+		n.packetConn, // yggdrasil.PacketConn
+		n.tlsConfig,  // TLS config
+		n.quicConfig, // QUIC config
+	)
+	if err != nil {
+		panic(err)
+	}
+
 	for {
-		conn, err := n.listener.Accept()
+		session, err := n.listener.Accept(context.TODO())
 		if err != nil {
 			n.log.Println("n.listener.Accept:", err)
 			return
 		}
-		var session *yamux.Session
-		// If the remote address is lower than ours then we'll be the
-		// server. Otherwse we'll be the client.
-		if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 {
-			session, err = yamux.Server(conn, n.yamuxConfig())
-		} else {
-			session, err = yamux.Client(conn, n.yamuxConfig())
-		}
-		if err != nil {
-			return
-		}
-		go n.listenFromYggConn(session)
+		go n.listenFromQUIC(session)
 	}
 }
 
-func (n *Node) listenFromYggConn(session *yamux.Session) {
+func (n *Node) listenFromQUIC(session quic.Session) {
 	n.sessions.Store(session.RemoteAddr().String(), session)
 	defer n.sessions.Delete(session.RemoteAddr())
-	defer func() {
-		if err := session.Close(); err != nil {
-			n.log.Println("session.Close:", err)
-		}
-	}()
-
 	for {
-		st, err := session.AcceptStream()
+		st, err := session.AcceptStream(context.TODO())
 		if err != nil {
 			n.log.Println("session.AcceptStream:", err)
 			return
 		}
-		n.incoming <- st
+		n.incoming <- QUICStream{st, session}
 	}
 }
 
@@ -96,29 +88,63 @@ func (n *Node) Dial(network, address string) (net.Conn, error) {
 // Implements http.Transport.DialContext
 func (n *Node) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
 	s, ok1 := n.sessions.Load(address)
-	session, ok2 := s.(*yamux.Session)
-	if !ok1 || !ok2 || (ok1 && ok2 && session.IsClosed()) {
-		conn, err := n.dialer.DialContext(ctx, network, address)
+	session, ok2 := s.(quic.Session)
+	if !ok1 || !ok2 || (ok1 && ok2 && session.ConnectionState().HandshakeComplete) {
+		dest, err := hex.DecodeString(address)
+		if err != nil {
+			return nil, err
+		}
+		if len(dest) != crypto.BoxPubKeyLen {
+			return nil, errors.New("invalid key length supplied")
+		}
+		var pubKey crypto.BoxPubKey
+		copy(pubKey[:], dest)
+
+		session, err = quic.Dial(
+			n.packetConn, // yggdrasil.PacketConn
+			&pubKey,      // dial address
+			address,      // dial SNI
+			n.tlsConfig,  // TLS config
+			n.quicConfig, // QUIC config
+		)
 		if err != nil {
 			n.log.Println("n.dialer.DialContext:", err)
 			return nil, err
 		}
-		// If the remote address is lower than ours then we will be the
-		// server. Otherwise we'll be the client.
-		if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 {
-			session, err = yamux.Server(conn, n.yamuxConfig())
-		} else {
-			session, err = yamux.Client(conn, n.yamuxConfig())
-		}
-		if err != nil {
-			return nil, err
-		}
-		go n.listenFromYggConn(session)
+		go n.listenFromQUIC(session)
 	}
 	st, err := session.OpenStream()
 	if err != nil {
 		n.log.Println("session.OpenStream:", err)
 		return nil, err
 	}
-	return st, nil
+	return QUICStream{st, session}, nil
+}
+
+func (n *Node) generateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		NotAfter:     time.Now().Add(time.Hour * 24 * 365),
+		DNSNames:     []string{n.DerivedSessionName()},
+	}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates:       []tls.Certificate{tlsCert},
+		NextProtos:         []string{"quic-matrix-ygg"},
+		InsecureSkipVerify: true,
+	}
 }
diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/stream.go b/cmd/dendrite-demo-yggdrasil/yggconn/stream.go
new file mode 100644
index 000000000..dac7447ee
--- /dev/null
+++ b/cmd/dendrite-demo-yggdrasil/yggconn/stream.go
@@ -0,0 +1,20 @@
+package yggconn
+
+import (
+	"net"
+
+	"github.com/lucas-clemente/quic-go"
+)
+
+type QUICStream struct {
+	quic.Stream
+	session quic.Session
+}
+
+func (s QUICStream) LocalAddr() net.Addr {
+	return s.session.LocalAddr()
+}
+
+func (s QUICStream) RemoteAddr() net.Addr {
+	return s.session.RemoteAddr()
+}
diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go
index a736b3852..ce706768e 100644
--- a/federationsender/queue/destinationqueue.go
+++ b/federationsender/queue/destinationqueue.go
@@ -52,7 +52,7 @@ type destinationQueue struct {
 	transactionIDMutex sync.Mutex                              // protects transactionID
 	transactionID      gomatrixserverlib.TransactionID         // last transaction ID
 	transactionCount   atomic.Int32                            // how many events in this transaction so far
-	pendingPDUs        atomic.Int32                            // how many PDUs are waiting to be sent
+	pendingPDUs        atomic.Int64                            // how many PDUs are waiting to be sent
 	pendingEDUs        []*gomatrixserverlib.EDU                // owned by backgroundSend
 	pendingInvites     []*gomatrixserverlib.InviteV2Request    // owned by backgroundSend
 	wakeServerCh       chan bool                               // interrupts idle wait
@@ -91,6 +91,7 @@ func (oq *destinationQueue) sendEvent(nid int64) {
 		// If the destination is blacklisted then drop the event.
 		return
 	}
+	oq.wakeQueueIfNeeded()
 	// Create a transaction ID. We'll either do this if we don't have
 	// one made up yet, or if we've exceeded the number of maximum
 	// events allowed in a single tranaction. We'll reset the counter
@@ -117,10 +118,6 @@ func (oq *destinationQueue) sendEvent(nid int64) {
 	// We've successfully added a PDU to the transaction so increase
 	// the counter.
 	oq.transactionCount.Add(1)
-	// If the queue isn't running at this point then start it.
-	if !oq.running.Load() {
-		go oq.backgroundSend()
-	}
 	// Signal that we've sent a new PDU. This will cause the queue to
 	// wake up if it's asleep. The return to the Add function will only
 	// be 1 if the previous value was 0, e.g. nothing was waiting before.
@@ -137,9 +134,7 @@ func (oq *destinationQueue) sendEDU(ev *gomatrixserverlib.EDU) {
 		// If the destination is blacklisted then drop the event.
 		return
 	}
-	if !oq.running.Load() {
-		go oq.backgroundSend()
-	}
+	oq.wakeQueueIfNeeded()
 	oq.incomingEDUs <- ev
 }
 
@@ -151,10 +146,30 @@ func (oq *destinationQueue) sendInvite(ev *gomatrixserverlib.InviteV2Request) {
 		// If the destination is blacklisted then drop the event.
 		return
 	}
+	oq.wakeQueueIfNeeded()
+	oq.incomingInvites <- ev
+}
+
+func (oq *destinationQueue) wakeQueueIfNeeded() {
 	if !oq.running.Load() {
+		// Look up how many events are pending in this queue. We need
+		// to do this so that the queue thinks it has work to do.
+		count, err := oq.db.GetPendingPDUCount(
+			context.TODO(),
+			oq.destination,
+		)
+		if err == nil {
+			oq.pendingPDUs.Store(count)
+			log.Printf("Destination queue %q has %d pending PDUs", oq.destination, count)
+		} else {
+			log.WithError(err).Errorf("Can't get pending PDU count for %q destination queue", oq.destination)
+		}
+		if count > 0 {
+			oq.wakeServerCh <- true
+		}
+		// Then start the queue.
 		go oq.backgroundSend()
 	}
-	oq.incomingInvites <- ev
 }
 
 // backgroundSend is the worker goroutine for sending events.
@@ -170,46 +185,44 @@ func (oq *destinationQueue) backgroundSend() {
 	for {
 		// If we have nothing to do then wait either for incoming events, or
 		// until we hit an idle timeout.
-		if oq.pendingPDUs.Load() == 0 && len(oq.pendingEDUs) == 0 && len(oq.pendingInvites) == 0 {
-			select {
-			case <-oq.wakeServerCh:
-				// We were woken up because there are new PDUs waiting in the
-				// database.
-			case edu := <-oq.incomingEDUs:
-				// EDUs are handled in-memory for now. We will try to keep
-				// the ordering intact.
-				// TODO: Certain EDU types need persistence, e.g. send-to-device
-				oq.pendingEDUs = append(oq.pendingEDUs, edu)
-				// If there are any more things waiting in the channel queue
-				// then read them. This is safe because we guarantee only
-				// having one goroutine per destination queue, so the channel
-				// isn't being consumed anywhere else.
-				for len(oq.incomingEDUs) > 0 {
-					oq.pendingEDUs = append(oq.pendingEDUs, <-oq.incomingEDUs)
-				}
-			case invite := <-oq.incomingInvites:
-				// There's no strict ordering requirement for invites like
-				// there is for transactions, so we put the invite onto the
-				// front of the queue. This means that if an invite that is
-				// stuck failing already, that it won't block our new invite
-				// from being sent.
-				oq.pendingInvites = append(
-					[]*gomatrixserverlib.InviteV2Request{invite},
-					oq.pendingInvites...,
-				)
-				// If there are any more things waiting in the channel queue
-				// then read them. This is safe because we guarantee only
-				// having one goroutine per destination queue, so the channel
-				// isn't being consumed anywhere else.
-				for len(oq.incomingInvites) > 0 {
-					oq.pendingInvites = append(oq.pendingInvites, <-oq.incomingInvites)
-				}
-			case <-time.After(time.Second * 30):
-				// The worker is idle so stop the goroutine. It'll get
-				// restarted automatically the next time we have an event to
-				// send.
-				return
+		select {
+		case <-oq.wakeServerCh:
+			// We were woken up because there are new PDUs waiting in the
+			// database.
+		case edu := <-oq.incomingEDUs:
+			// EDUs are handled in-memory for now. We will try to keep
+			// the ordering intact.
+			// TODO: Certain EDU types need persistence, e.g. send-to-device
+			oq.pendingEDUs = append(oq.pendingEDUs, edu)
+			// If there are any more things waiting in the channel queue
+			// then read them. This is safe because we guarantee only
+			// having one goroutine per destination queue, so the channel
+			// isn't being consumed anywhere else.
+			for len(oq.incomingEDUs) > 0 {
+				oq.pendingEDUs = append(oq.pendingEDUs, <-oq.incomingEDUs)
 			}
+		case invite := <-oq.incomingInvites:
+			// There's no strict ordering requirement for invites like
+			// there is for transactions, so we put the invite onto the
+			// front of the queue. This means that if an invite that is
+			// stuck failing already, that it won't block our new invite
+			// from being sent.
+			oq.pendingInvites = append(
+				[]*gomatrixserverlib.InviteV2Request{invite},
+				oq.pendingInvites...,
+			)
+			// If there are any more things waiting in the channel queue
+			// then read them. This is safe because we guarantee only
+			// having one goroutine per destination queue, so the channel
+			// isn't being consumed anywhere else.
+			for len(oq.incomingInvites) > 0 {
+				oq.pendingInvites = append(oq.pendingInvites, <-oq.incomingInvites)
+			}
+		case <-time.After(time.Second * 30):
+			// The worker is idle so stop the goroutine. It'll get
+			// restarted automatically the next time we have an event to
+			// send.
+			return
 		}
 
 		// If we are backing off this server then wait for the
@@ -317,8 +330,10 @@ func (oq *destinationQueue) nextTransaction(
 	// Ask the database for any pending PDUs from the next transaction.
 	// maxPDUsPerTransaction is an upper limit but we probably won't
 	// actually retrieve that many events.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
 	txid, pdus, err := oq.db.GetNextTransactionPDUs(
-		context.TODO(),        // context
+		ctx,                   // context
 		oq.destination,        // server name
 		maxPDUsPerTransaction, // max events to retrieve
 	)
@@ -366,7 +381,7 @@ func (oq *destinationQueue) nextTransaction(
 	case nil:
 		// No error was returned so the transaction looks to have
 		// been successfully sent.
-		oq.pendingPDUs.Sub(int32(len(t.PDUs)))
+		oq.pendingPDUs.Sub(int64(len(t.PDUs)))
 		// Clean up the transaction in the database.
 		if err = oq.db.CleanTransactionPDUs(
 			context.TODO(),
diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go
index f4df93fa4..09d74ed7e 100644
--- a/federationsender/storage/interface.go
+++ b/federationsender/storage/interface.go
@@ -30,4 +30,5 @@ type Database interface {
 	AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, nids []int64) error
 	GetNextTransactionPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (gomatrixserverlib.TransactionID, []*gomatrixserverlib.HeaderedEvent, error)
 	CleanTransactionPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, transactionID gomatrixserverlib.TransactionID) error
+	GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error)
 }
diff --git a/federationsender/storage/postgres/queue_pdus_table.go b/federationsender/storage/postgres/queue_pdus_table.go
index ef7a9f41e..bc22825d8 100644
--- a/federationsender/storage/postgres/queue_pdus_table.go
+++ b/federationsender/storage/postgres/queue_pdus_table.go
@@ -59,12 +59,17 @@ const selectQueueReferenceJSONCountSQL = "" +
 	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
 	" WHERE json_nid = $1"
 
+const selectQueuePDUsCountSQL = "" +
+	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
+	" WHERE server_name = $1"
+
 type queuePDUsStatements struct {
 	insertQueuePDUStmt                *sql.Stmt
 	deleteQueueTransactionPDUsStmt    *sql.Stmt
 	selectQueueNextTransactionIDStmt  *sql.Stmt
 	selectQueuePDUsByTransactionStmt  *sql.Stmt
 	selectQueueReferenceJSONCountStmt *sql.Stmt
+	selectQueuePDUsCountStmt          *sql.Stmt
 }
 
 func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) {
@@ -87,6 +92,9 @@ func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) {
 	if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueueReferenceJSONCountSQL); err != nil {
 		return
 	}
+	if s.selectQueuePDUsCountStmt, err = db.Prepare(selectQueuePDUsCountSQL); err != nil {
+		return
+	}
 	return
 }
 
@@ -144,6 +152,21 @@ func (s *queuePDUsStatements) selectQueueReferenceJSONCount(
 	return count, err
 }
 
+func (s *queuePDUsStatements) selectQueuePDUCount(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) (int64, error) {
+	var count int64
+	stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsCountStmt)
+	err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
+	if err == sql.ErrNoRows {
+		// It's acceptable for there to be no rows referencing a given
+		// JSON NID but it's not an error condition. Just return as if
+		// there's a zero count.
+		return 0, nil
+	}
+	return count, err
+}
+
 func (s *queuePDUsStatements) selectQueuePDUs(
 	ctx context.Context, txn *sql.Tx,
 	serverName gomatrixserverlib.ServerName,
diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go
index 18d1532a4..be28c15dc 100644
--- a/federationsender/storage/postgres/storage.go
+++ b/federationsender/storage/postgres/storage.go
@@ -255,3 +255,12 @@ func (d *Database) CleanTransactionPDUs(
 		return nil
 	})
 }
+
+// GetPendingPDUCount returns the number of PDUs waiting to be
+// sent for a given servername.
+func (d *Database) GetPendingPDUCount(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+) (int64, error) {
+	return d.selectQueuePDUCount(ctx, nil, serverName)
+}
diff --git a/federationsender/storage/sqlite3/queue_pdus_table.go b/federationsender/storage/sqlite3/queue_pdus_table.go
index dc08fd707..955ff507d 100644
--- a/federationsender/storage/sqlite3/queue_pdus_table.go
+++ b/federationsender/storage/sqlite3/queue_pdus_table.go
@@ -60,12 +60,17 @@ const selectQueueReferenceJSONCountSQL = "" +
 	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
 	" WHERE json_nid = $1"
 
+const selectQueuePDUsCountSQL = "" +
+	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
+	" WHERE server_name = $1"
+
 type queuePDUsStatements struct {
 	insertQueuePDUStmt                *sql.Stmt
 	deleteQueueTransactionPDUsStmt    *sql.Stmt
 	selectQueueNextTransactionIDStmt  *sql.Stmt
 	selectQueuePDUsByTransactionStmt  *sql.Stmt
 	selectQueueReferenceJSONCountStmt *sql.Stmt
+	selectQueuePDUsCountStmt          *sql.Stmt
 }
 
 func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) {
@@ -88,6 +93,9 @@ func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) {
 	if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueueReferenceJSONCountSQL); err != nil {
 		return
 	}
+	if s.selectQueuePDUsCountStmt, err = db.Prepare(selectQueuePDUsCountSQL); err != nil {
+		return
+	}
 	return
 }
 
@@ -142,6 +150,21 @@ func (s *queuePDUsStatements) selectQueueReferenceJSONCount(
 	return count, err
 }
 
+func (s *queuePDUsStatements) selectQueuePDUCount(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) (int64, error) {
+	var count int64
+	stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsCountStmt)
+	err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
+	if err == sql.ErrNoRows {
+		// It's acceptable for there to be no rows referencing a given
+		// JSON NID but it's not an error condition. Just return as if
+		// there's a zero count.
+		return 0, nil
+	}
+	return count, err
+}
+
 func (s *queuePDUsStatements) selectQueuePDUs(
 	ctx context.Context, txn *sql.Tx,
 	serverName gomatrixserverlib.ServerName,
diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go
index 7629ecd21..30ac81bfd 100644
--- a/federationsender/storage/sqlite3/storage.go
+++ b/federationsender/storage/sqlite3/storage.go
@@ -261,3 +261,12 @@ func (d *Database) CleanTransactionPDUs(
 		return nil
 	})
 }
+
+// GetPendingPDUCount returns the number of PDUs waiting to be
+// sent for a given servername.
+func (d *Database) GetPendingPDUCount(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+) (int64, error) {
+	return d.selectQueuePDUCount(ctx, nil, serverName)
+}
diff --git a/go.mod b/go.mod
index b278ec4b3..5a31df464 100644
--- a/go.mod
+++ b/go.mod
@@ -17,7 +17,8 @@ require (
 	github.com/libp2p/go-libp2p-kad-dht v0.5.0
 	github.com/libp2p/go-libp2p-pubsub v0.2.5
 	github.com/libp2p/go-libp2p-record v0.1.2
-	github.com/libp2p/go-yamux v1.3.7
+	github.com/libp2p/go-yamux v1.3.7 // indirect
+	github.com/lucas-clemente/quic-go v0.17.2
 	github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
@@ -37,9 +38,9 @@ require (
 	github.com/uber-go/atomic v1.3.0 // indirect
 	github.com/uber/jaeger-client-go v2.15.0+incompatible
 	github.com/uber/jaeger-lib v1.5.0
-	github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b
+	github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200702163833-11ecfa688d93
 	go.uber.org/atomic v1.4.0
-	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
+	golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5
 	gopkg.in/h2non/bimg.v1 v1.0.18
 	gopkg.in/yaml.v2 v2.2.8
 )
diff --git a/go.sum b/go.sum
index a98ae851c..48773c6ec 100644
--- a/go.sum
+++ b/go.sum
@@ -1,4 +1,12 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
+dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
+dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
+dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
+dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48=
@@ -17,11 +25,13 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
 github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
@@ -35,8 +45,11 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa
 github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
+github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
@@ -46,6 +59,7 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
@@ -71,14 +85,20 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
+github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
 github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -92,18 +112,29 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/gologme/log v1.2.0 h1:Ya5Ip/KD6FX7uH0S31QO87nCCSucKtF44TLbTtO7V4c=
 github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
@@ -111,13 +142,22 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
 github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
@@ -189,12 +229,14 @@ github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr1
 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
 github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
 github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
 github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0/go.mod h1:rUi0/YffDo1oXBOGn1KRq7Fr07LX48XEBecQnmwjsAo=
@@ -220,6 +262,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
@@ -361,11 +404,18 @@ github.com/libp2p/go-yamux v1.3.0 h1:FsYzT16Wq2XqUGJsBbOxoz9g+dFklvNi7jN6YFPfl7U
 github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
 github.com/libp2p/go-yamux v1.3.7 h1:v40A1eSPJDIZwz2AvrV3cxpTZEGDP11QJbukmEhYyQI=
 github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
+github.com/lucas-clemente/quic-go v0.17.2 h1:4iQInIuNQkPNZmsy9rCnwuOzpH0qGnDo4jn0QfI/qE4=
+github.com/lucas-clemente/quic-go v0.17.2/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
+github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lxn/walk v0.0.0-20191128110447-55ccb3a9f5c1/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
 github.com/lxn/win v0.0.0-20191128105842-2da648fda5b4/go.mod h1:ouWl4wViUNh8tPSIwxTVMuS014WakR1hqvBc2I0bMoA=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
+github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ=
+github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY=
 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY=
 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
@@ -376,14 +426,6 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1 h1:3yS6hw01X72jpJuAPGVOY+QFD9cpAETR/6Hq2WYKbpU=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1 h1:QDOdGCfrzuVLEess3id2a2B29oVZ9JXgJmUfwE7r/iI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d h1:v1JS+JZWwAsqAc22TGWPbRDc6O5D6geSfV5Bb5wvYIs=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328 h1:rz6aiTpUyNPRcWZBWUGDkQjI7lfeLdhzy+x/Pw2jha8=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe h1:rCjG+azihYsO+EIdm//Zx5gQ7hzeJVraeSukLsW1Mic=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
@@ -405,6 +447,7 @@ github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
+github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA=
 github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
@@ -470,6 +513,8 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
+github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
+github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww=
@@ -478,18 +523,21 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg=
 github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@@ -500,6 +548,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
@@ -509,10 +558,12 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
@@ -521,12 +572,37 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
+github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
+github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
+github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
+github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
+github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
+github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
+github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
+github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
+github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
+github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
+github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
+github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
+github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
+github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
+github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
+github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
+github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
+github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
+github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
 github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
 github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU=
 github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
@@ -546,6 +622,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc=
 github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
 github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
@@ -562,6 +639,8 @@ github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
 github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo=
 github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
+github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
@@ -581,8 +660,9 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yggdrasil-network/yggdrasil-extras v0.0.0-20200525205615-6c8a4a2e8855/go.mod h1:xQdsh08Io6nV4WRnOVTe6gI8/2iTvfLDQ0CYa5aMt+I=
-github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b h1:ELOisSxFXCcptRs4LFub+Hz5fYUvV12wZrTps99Eb3E=
-github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE=
+github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200702163833-11ecfa688d93 h1:DX2HXQHoejo9GqkvFuRS9iHrjhfv/9WgL3TjmUz/AaY=
+github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200702163833-11ecfa688d93/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE=
+go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -594,14 +674,18 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
+golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -612,7 +696,10 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -620,10 +707,15 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
@@ -635,6 +727,10 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -647,12 +743,14 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -673,19 +771,26 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200301040627-c5d0d7b4ec88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3-0.20191230102452-929e72ca90de h1:aYKJLPSrddB2N7/6OKyFqJ337SXpo61bBuvO5p1+7iY=
 golang.org/x/text v0.3.3-0.20191230102452-929e72ca90de/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
@@ -693,14 +798,32 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.zx2c4.com/wireguard v0.0.20200122-0.20200214175355-9cbcff10dd3e/go.mod h1:P2HsVp8SKwZEufsnezXZA4GRX/T49/HlU7DGuelXsU4=
 golang.zx2c4.com/wireguard v0.0.20200320/go.mod h1:lDian4Sw4poJ04SgHh35nzMVwGSYlPumkdnHcucAQoY=
 golang.zx2c4.com/wireguard/windows v0.1.0/go.mod h1:EK7CxrFnicmYJ0ZCF6crBh2/EMMeSxMlqgLlwN0Kv9s=
+google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
+google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
+google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
@@ -716,6 +839,7 @@ gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4=
 gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So=
 gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM=
 gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
 gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
 gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
@@ -740,5 +864,12 @@ gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

From 9dffeb9b67fd923cd26dc1d51a50260f8102f320 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 3 Jul 2020 10:25:26 +0100
Subject: [PATCH 23/23] Fix joins to rooms that we know about that have room
 IDs with our server part (#1178)

---
 roomserver/internal/perform_join.go | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go
index 1a4508893..c5480a5bd 100644
--- a/roomserver/internal/perform_join.go
+++ b/roomserver/internal/perform_join.go
@@ -124,7 +124,13 @@ func (r *RoomserverInternalAPI) performJoinRoomByID(
 			Msg:  fmt.Sprintf("Room ID %q is invalid: %s", req.RoomIDOrAlias, err),
 		}
 	}
-	req.ServerNames = append(req.ServerNames, domain)
+
+	// If the server name in the room ID isn't ours then it's a
+	// possible candidate for finding the room via federation. Add
+	// it to the list of servers to try.
+	if domain != r.Cfg.Matrix.ServerName {
+		req.ServerNames = append(req.ServerNames, domain)
+	}
 
 	// Prepare the template for the join event.
 	userID := req.UserID
@@ -233,13 +239,18 @@ func (r *RoomserverInternalAPI) performJoinRoomByID(
 		}
 
 	case eventutil.ErrRoomNoExists:
-		// The room doesn't exist. First of all check if the room is a local
-		// room. If it is then there's nothing more to do - the room just
-		// hasn't been created yet.
+		// The room doesn't exist locally. If the room ID looks like it should
+		// be ours then this probably means that we've nuked our database at
+		// some point.
 		if domain == r.Cfg.Matrix.ServerName {
-			return "", &api.PerformError{
-				Code: api.PerformErrorNoRoom,
-				Msg:  fmt.Sprintf("Room ID %q does not exist", req.RoomIDOrAlias),
+			// If there are no more server names to try then give up here.
+			// Otherwise we'll try a federated join as normal, since it's quite
+			// possible that the room still exists on other servers.
+			if len(req.ServerNames) == 0 {
+				return "", &api.PerformError{
+					Code: api.PerformErrorNoRoom,
+					Msg:  fmt.Sprintf("Room ID %q does not exist", req.RoomIDOrAlias),
+				}
 			}
 		}