Merge branch 'master' into master

appservice/appservice.go

@@ -44,8 +44,7 @@ func SetupAppServiceAPIComponent(
 	accountsDB accounts.Database,
 	deviceDB devices.Database,
 	federation *gomatrixserverlib.FederationClient,
-	roomserverAliasAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	roomserverQueryAPI roomserverAPI.RoomserverQueryAPI,
 	transactionsCache *transactions.Cache,
 ) appserviceAPI.AppServiceQueryAPI {
 	// Create a connection to the appservice postgres DB
@@ -87,7 +86,7 @@ func SetupAppServiceAPIComponent(
 
 	consumer := consumers.NewOutputRoomEventConsumer(
 		base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB,
-		roomserverQueryAPI, roomserverAliasAPI, workerStates,
+		rsAPI, workerStates,
 	)
 	if err := consumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
@@ -100,7 +99,7 @@ func SetupAppServiceAPIComponent(
 
 	// Set up HTTP Endpoints
 	routing.Setup(
-		base.APIMux, base.Cfg, roomserverQueryAPI, roomserverAliasAPI,
+		base.APIMux, base.Cfg, rsAPI,
 		accountsDB, federation, transactionsCache,
 	)
 

appservice/consumers/roomserver.go

@@ -35,8 +35,7 @@ type OutputRoomEventConsumer struct {
 	roomServerConsumer *common.ContinualConsumer
 	db                 accounts.Database
 	asDB               storage.Database
-	query              api.RoomserverQueryAPI
+	rsAPI              api.RoomserverInternalAPI
-	alias              api.RoomserverAliasAPI
 	serverName         string
 	workerStates       []types.ApplicationServiceWorkerState
 }
@@ -48,8 +47,7 @@ func NewOutputRoomEventConsumer(
 	kafkaConsumer sarama.Consumer,
 	store accounts.Database,
 	appserviceDB storage.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
-	aliasAPI api.RoomserverAliasAPI,
 	workerStates []types.ApplicationServiceWorkerState,
 ) *OutputRoomEventConsumer {
 	consumer := common.ContinualConsumer{
@@ -61,8 +59,7 @@ func NewOutputRoomEventConsumer(
 		roomServerConsumer: &consumer,
 		db:                 store,
 		asDB:               appserviceDB,
-		query:              queryAPI,
+		rsAPI:              rsAPI,
-		alias:              aliasAPI,
 		serverName:         string(cfg.Matrix.ServerName),
 		workerStates:       workerStates,
 	}
@@ -139,7 +136,7 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents(
 	// Request the missing events from the roomserver
 	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
 	var eventResp api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
+	if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
 		return nil, err
 	}
 
@@ -200,7 +197,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	// Check all known room aliases of the room the event came from
 	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
 	var queryRes api.GetAliasesForRoomIDResponse
-	if err := s.alias.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
+	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
 		for _, alias := range queryRes.Aliases {
 			if appservice.IsInterestedInRoomAlias(alias) {
 				return true

appservice/routing/routing.go

@@ -37,7 +37,7 @@ const pathPrefixApp = "/_matrix/app/v1"
 // nolint: gocyclo
 func Setup(
 	apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam
-	queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam
+	rsAPI api.RoomserverInternalAPI, // nolint: unparam
 	accountDB accounts.Database, // nolint: unparam
 	federation *gomatrixserverlib.FederationClient, // nolint: unparam
 	transactionsCache *transactions.Cache, // nolint: unparam

build.sh

@@ -3,4 +3,6 @@
 # Put installed packages into ./bin
 export GOBIN=$PWD/`dirname $0`/bin
 
-go install -v $PWD/`dirname $0`/cmd/...
+go install -v $PWD/`dirname $0`/cmd/...
+
+GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs

clientapi/clientapi.go

@@ -38,15 +38,13 @@ func SetupClientAPIComponent(
 	accountsDB accounts.Database,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing *gomatrixserverlib.KeyRing,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	inputAPI roomserverAPI.RoomserverInputAPI,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
 	eduInputAPI eduServerAPI.EDUServerInputAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	transactionsCache *transactions.Cache,
-	fedSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	fsAPI federationSenderAPI.FederationSenderInternalAPI,
 ) {
-	roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI)
+	roomserverProducer := producers.NewRoomserverProducer(rsAPI)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 
 	userUpdateProducer := &producers.UserUpdateProducer{
@@ -60,15 +58,15 @@ func SetupClientAPIComponent(
 	}
 
 	consumer := consumers.NewOutputRoomEventConsumer(
-		base.Cfg, base.KafkaConsumer, accountsDB, queryAPI,
+		base.Cfg, base.KafkaConsumer, accountsDB, rsAPI,
 	)
 	if err := consumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start room server consumer")
 	}
 
 	routing.Setup(
-		base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI,
+		base.APIMux, base.Cfg, roomserverProducer, rsAPI, asAPI,
 		accountsDB, deviceDB, federation, *keyRing, userUpdateProducer,
-		syncProducer, eduProducer, transactionsCache, fedSenderAPI,
+		syncProducer, eduProducer, transactionsCache, fsAPI,
 	)
 }

clientapi/consumers/roomserver.go

@@ -30,10 +30,10 @@ import (
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	roomServerConsumer *common.ContinualConsumer
+	rsAPI      api.RoomserverInternalAPI
-	db                 accounts.Database
+	rsConsumer *common.ContinualConsumer
-	query              api.RoomserverQueryAPI
+	db         accounts.Database
-	serverName         string
+	serverName string
 }
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
@@ -41,7 +41,7 @@ func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
 	store accounts.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) *OutputRoomEventConsumer {
 
 	consumer := common.ContinualConsumer{
@@ -50,10 +50,10 @@ func NewOutputRoomEventConsumer(
 		PartitionStore: store,
 	}
 	s := &OutputRoomEventConsumer{
-		roomServerConsumer: &consumer,
+		rsConsumer: &consumer,
-		db:                 store,
+		db:         store,
-		query:              queryAPI,
+		rsAPI:      rsAPI,
-		serverName:         string(cfg.Matrix.ServerName),
+		serverName: string(cfg.Matrix.ServerName),
 	}
 	consumer.ProcessMessage = s.onMessage
 
@@ -62,7 +62,7 @@ func NewOutputRoomEventConsumer(
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	return s.roomServerConsumer.Start()
+	return s.rsConsumer.Start()
 }
 
 // onMessage is called when the sync server receives a new event from the room server output log.
@@ -134,7 +134,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 	// Request the missing events from the roomserver
 	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
 	var eventResp api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
+	if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
 		return nil, err
 	}
 

clientapi/producers/roomserver.go

@@ -23,15 +23,13 @@ import (
 
 // RoomserverProducer produces events for the roomserver to consume.
 type RoomserverProducer struct {
-	InputAPI api.RoomserverInputAPI
+	RsAPI api.RoomserverInternalAPI
-	QueryAPI api.RoomserverQueryAPI
 }
 
 // NewRoomserverProducer creates a new RoomserverProducer
-func NewRoomserverProducer(inputAPI api.RoomserverInputAPI, queryAPI api.RoomserverQueryAPI) *RoomserverProducer {
+func NewRoomserverProducer(rsAPI api.RoomserverInternalAPI) *RoomserverProducer {
 	return &RoomserverProducer{
-		InputAPI: inputAPI,
+		RsAPI: rsAPI,
-		QueryAPI: queryAPI,
 	}
 }
 
@@ -95,7 +93,7 @@ func (c *RoomserverProducer) SendInputRoomEvents(
 ) (eventID string, err error) {
 	request := api.InputRoomEventsRequest{InputRoomEvents: ires}
 	var response api.InputRoomEventsResponse
-	err = c.InputAPI.InputRoomEvents(ctx, &request, &response)
+	err = c.RsAPI.InputRoomEvents(ctx, &request, &response)
 	eventID = response.EventID
 	return
 }
@@ -118,5 +116,5 @@ func (c *RoomserverProducer) SendInvite(
 		}},
 	}
 	var response api.InputRoomEventsResponse
-	return c.InputAPI.InputRoomEvents(ctx, &request, &response)
+	return c.RsAPI.InputRoomEvents(ctx, &request, &response)
 }

clientapi/routing/capabilities.go

@@ -26,11 +26,11 @@ import (
 // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
 // by building a m.room.member event then sending it to the room server
 func GetCapabilities(
-	req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI,
+	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{}
 	roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{}
-	if err := queryAPI.QueryRoomVersionCapabilities(
+	if err := rsAPI.QueryRoomVersionCapabilities(
 		req.Context(),
 		&roomVersionsQueryReq,
 		&roomVersionsQueryRes,

clientapi/routing/createroom.go

@@ -137,13 +137,13 @@ type fledglingEvent struct {
 func CreateRoom(
 	req *http.Request, device *authtypes.Device,
 	cfg *config.Dendrite, producer *producers.RoomserverProducer,
-	accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	// TODO (#267): Check room ID doesn't clash with an existing one, and we
 	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
 	roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
-	return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI, asAPI)
+	return createRoom(req, device, cfg, roomID, producer, accountDB, rsAPI, asAPI)
 }
 
 // createRoom implements /createRoom
@@ -151,7 +151,7 @@ func CreateRoom(
 func createRoom(
 	req *http.Request, device *authtypes.Device,
 	cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer,
-	accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	logger := util.GetLogger(req.Context())
@@ -340,7 +340,7 @@ func createRoom(
 		}
 
 		var aliasResp roomserverAPI.SetRoomAliasResponse
-		err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp)
+		err = rsAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
 			return jsonerror.InternalServerError()

clientapi/routing/directory.go

@@ -46,8 +46,8 @@ func DirectoryRoom(
 	roomAlias string,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.Dendrite,
-	rsAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	fedSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	fedSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
 	if err != nil {
@@ -115,7 +115,7 @@ func SetLocalAlias(
 	device *authtypes.Device,
 	alias string,
 	cfg *config.Dendrite,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', alias)
 	if err != nil {
@@ -190,7 +190,7 @@ func RemoveLocalAlias(
 	req *http.Request,
 	device *authtypes.Device,
 	alias string,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 
 	creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{

clientapi/routing/getevent.go

@@ -44,7 +44,7 @@ func GetEvent(
 	roomID string,
 	eventID string,
 	cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing gomatrixserverlib.KeyRing,
 ) util.JSONResponse {
@@ -52,7 +52,7 @@ func GetEvent(
 		EventIDs: []string{eventID},
 	}
 	var eventsResp api.QueryEventsByIDResponse
-	err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
+	err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed")
 		return jsonerror.InternalServerError()
@@ -88,7 +88,7 @@ func GetEvent(
 		}},
 	}
 	var stateResp api.QueryStateAfterEventsResponse
-	if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
+	if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/joinroom.go

@@ -27,12 +27,11 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 // JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API.
@@ -44,8 +43,8 @@ func JoinRoomByIDOrAlias(
 	cfg *config.Dendrite,
 	federation *gomatrixserverlib.FederationClient,
 	producer *producers.RoomserverProducer,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	fsAPI federationSenderAPI.FederationSenderInternalAPI,
 	keyRing gomatrixserverlib.KeyRing,
 	accountDB accounts.Database,
 ) util.JSONResponse {
@@ -79,7 +78,8 @@ func JoinRoomByIDOrAlias(
 	content["avatar_url"] = profile.AvatarURL
 
 	r := joinRoomReq{
-		req, evTime, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing,
+		req, evTime, content, device.UserID, cfg, federation, producer,
+		rsAPI, fsAPI, keyRing,
 	}
 
 	if strings.HasPrefix(roomIDOrAlias, "!") {
@@ -105,8 +105,8 @@ type joinRoomReq struct {
 	cfg        *config.Dendrite
 	federation *gomatrixserverlib.FederationClient
 	producer   *producers.RoomserverProducer
-	queryAPI   roomserverAPI.RoomserverQueryAPI
+	rsAPI      roomserverAPI.RoomserverInternalAPI
-	aliasAPI   roomserverAPI.RoomserverAliasAPI
+	fsAPI      federationSenderAPI.FederationSenderInternalAPI
 	keyRing    gomatrixserverlib.KeyRing
 }
 
@@ -122,7 +122,7 @@ func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse {
 		RoomID: roomID, TargetUserID: r.userID,
 	}
 	var queryRes roomserverAPI.QueryInvitesForUserResponse
-	if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil {
+	if err := r.rsAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil {
 		util.GetLogger(r.req.Context()).WithError(err).Error("r.queryAPI.QueryInvitesForUser failed")
 		return jsonerror.InternalServerError()
 	}
@@ -170,7 +170,7 @@ func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse {
 	if domain == r.cfg.Matrix.ServerName {
 		queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
 		var queryRes roomserverAPI.GetRoomIDForAliasResponse
-		if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil {
+		if err = r.rsAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil {
 			util.GetLogger(r.req.Context()).WithError(err).Error("r.aliasAPI.GetRoomIDForAlias failed")
 			return jsonerror.InternalServerError()
 		}
@@ -241,7 +241,7 @@ func (r joinRoomReq) joinRoomUsingServers(
 	}
 
 	queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{}
-	event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes)
+	event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.rsAPI, &queryRes)
 	if err == nil {
 		// If we have successfully built an event at this point then we can
 		// assert that the room is a local room, as BuildEvent was able to
@@ -326,71 +326,15 @@ func (r joinRoomReq) joinRoomUsingServers(
 // server was invalid this returns an error.
 // Otherwise this returns a JSONResponse.
 func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) {
-	// Ask the room server for information about room versions.
+	fedJoinReq := federationSenderAPI.PerformJoinRequest{
-	var request api.QueryRoomVersionCapabilitiesRequest
+		RoomID:     roomID,
-	var response api.QueryRoomVersionCapabilitiesResponse
+		UserID:     r.userID,
-	if err := r.queryAPI.QueryRoomVersionCapabilities(r.req.Context(), &request, &response); err != nil {
+		ServerName: server,
+	}
+	fedJoinRes := federationSenderAPI.PerformJoinResponse{}
+	if err := r.fsAPI.PerformJoin(r.req.Context(), &fedJoinReq, &fedJoinRes); err != nil {
 		return nil, err
 	}
-	var supportedVersions []gomatrixserverlib.RoomVersion
-	for version := range response.AvailableRoomVersions {
-		supportedVersions = append(supportedVersions, version)
-	}
-	respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, supportedVersions)
-	if err != nil {
-		// TODO: Check if the user was not allowed to join the room.
-		return nil, fmt.Errorf("r.federation.MakeJoin: %w", err)
-	}
-
-	// Set all the fields to be what they should be, this should be a no-op
-	// but it's possible that the remote server returned us something "odd"
-	err = r.writeToBuilder(&respMakeJoin.JoinEvent, roomID)
-	if err != nil {
-		return nil, fmt.Errorf("r.writeToBuilder: %w", err)
-	}
-
-	if respMakeJoin.RoomVersion == "" {
-		respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1
-	}
-	if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion(
-				fmt.Sprintf("Room version '%s' is not supported", respMakeJoin.RoomVersion),
-			),
-		}, nil
-	}
-
-	event, err := respMakeJoin.JoinEvent.Build(
-		r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID,
-		r.cfg.Matrix.PrivateKey, respMakeJoin.RoomVersion,
-	)
-	if err != nil {
-		return nil, fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err)
-	}
-
-	respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event, respMakeJoin.RoomVersion)
-	if err != nil {
-		return nil, fmt.Errorf("r.federation.SendJoin: %w", err)
-	}
-
-	if err = r.checkSendJoinResponse(event, server, respMakeJoin, respSendJoin); err != nil {
-		return nil, err
-	}
-
-	util.GetLogger(r.req.Context()).WithFields(logrus.Fields{
-		"room_id":          roomID,
-		"num_auth_events":  len(respSendJoin.AuthEvents),
-		"num_state_events": len(respSendJoin.StateEvents),
-	}).Info("Room join signature and auth verification passed")
-
-	if err = r.producer.SendEventWithState(
-		r.req.Context(),
-		respSendJoin.ToRespState(),
-		event.Headered(respMakeJoin.RoomVersion),
-	); err != nil {
-		util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState")
-	}
 
 	return &util.JSONResponse{
 		Code: http.StatusOK,
@@ -400,49 +344,3 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib
 		}{roomID},
 	}, nil
 }
-
-// checkSendJoinResponse checks that all of the signatures are correct
-// and that the join is allowed by the supplied state.
-func (r joinRoomReq) checkSendJoinResponse(
-	event gomatrixserverlib.Event,
-	server gomatrixserverlib.ServerName,
-	respMakeJoin gomatrixserverlib.RespMakeJoin,
-	respSendJoin gomatrixserverlib.RespSendJoin,
-) error {
-	// A list of events that we have retried, if they were not included in
-	// the auth events supplied in the send_join.
-	retries := map[string]bool{}
-
-retryCheck:
-	// TODO: Can we expand Check here to return a list of missing auth
-	// events rather than failing one at a time?
-	if err := respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil {
-		switch e := err.(type) {
-		case gomatrixserverlib.MissingAuthEventError:
-			// Check that we haven't already retried for this event, prevents
-			// us from ending up in endless loops
-			if !retries[e.AuthEventID] {
-				// Ask the server that we're talking to right now for the event
-				tx, txerr := r.federation.GetEvent(r.req.Context(), server, e.AuthEventID)
-				if txerr != nil {
-					return fmt.Errorf("r.federation.GetEvent: %w", txerr)
-				}
-				// For each event returned, add it to the auth events.
-				for _, pdu := range tx.PDUs {
-					ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion)
-					if everr != nil {
-						return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr)
-					}
-					respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev)
-				}
-				// Mark the event as retried and then give the check another go.
-				retries[e.AuthEventID] = true
-				goto retryCheck
-			}
-			return fmt.Errorf("respSendJoin (after retries): %w", e)
-		default:
-			return fmt.Errorf("respSendJoin: %w", err)
-		}
-	}
-	return nil
-}

clientapi/routing/membership.go

@@ -45,12 +45,12 @@ var errMissingUserID = errors.New("'user_id' must be supplied")
 func SendMembership(
 	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
 	roomID string, membership string, cfg *config.Dendrite,
-	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 	producer *producers.RoomserverProducer,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -71,7 +71,7 @@ func SendMembership(
 	}
 
 	inviteStored, jsonErrResp := checkAndProcessThreepid(
-		req, device, &body, cfg, queryAPI, accountDB, producer,
+		req, device, &body, cfg, rsAPI, accountDB, producer,
 		membership, roomID, evTime,
 	)
 	if jsonErrResp != nil {
@@ -89,7 +89,7 @@ func SendMembership(
 	}
 
 	event, err := buildMembershipEvent(
-		req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, queryAPI, asAPI,
+		req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, rsAPI, asAPI,
 	)
 	if err == errMissingUserID {
 		return util.JSONResponse{
@@ -153,7 +153,7 @@ func buildMembershipEvent(
 	device *authtypes.Device,
 	membership, roomID string,
 	cfg *config.Dendrite, evTime time.Time,
-	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*gomatrixserverlib.Event, error) {
 	stateKey, reason, err := getMembershipStateKey(body, device, membership)
 	if err != nil {
@@ -188,7 +188,7 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
-	return common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil)
+	return common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
 }
 
 // loadProfile lookups the profile of a given user from the database and returns
@@ -248,7 +248,7 @@ func checkAndProcessThreepid(
 	device *authtypes.Device,
 	body *threepid.MembershipRequest,
 	cfg *config.Dendrite,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 	accountDB accounts.Database,
 	producer *producers.RoomserverProducer,
 	membership, roomID string,
@@ -256,7 +256,7 @@ func checkAndProcessThreepid(
 ) (inviteStored bool, errRes *util.JSONResponse) {
 
 	inviteStored, err := threepid.CheckAndProcessInvite(
-		req.Context(), device, body, cfg, queryAPI, accountDB, producer,
+		req.Context(), device, body, cfg, rsAPI, accountDB, producer,
 		membership, roomID, evTime,
 	)
 	if err == threepid.ErrMissingParameter {

clientapi/routing/memberships.go

@@ -39,7 +39,7 @@ type getJoinedRoomsResponse struct {
 func GetMemberships(
 	req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool,
 	_ *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	queryReq := api.QueryMembershipsForRoomRequest{
 		JoinedOnly: joinedOnly,
@@ -47,8 +47,8 @@ func GetMemberships(
 		Sender:     device.UserID,
 	}
 	var queryRes api.QueryMembershipsForRoomResponse
-	if err := queryAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil {
+	if err := rsAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryMembershipsForRoom failed")
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed")
 		return jsonerror.InternalServerError()
 	}
 

clientapi/routing/profile.go

@@ -94,7 +94,7 @@ func GetAvatarURL(
 func SetAvatarURL(
 	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
 	userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite,
-	rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
+	rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -154,7 +154,7 @@ func SetAvatarURL(
 	}
 
 	events, err := buildMembershipEvents(
-		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+		req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
 	)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed")
@@ -208,7 +208,7 @@ func GetDisplayName(
 func SetDisplayName(
 	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
 	userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite,
-	rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
+	rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -268,7 +268,7 @@ func SetDisplayName(
 	}
 
 	events, err := buildMembershipEvents(
-		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+		req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
 	)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed")
@@ -337,14 +337,14 @@ func buildMembershipEvents(
 	ctx context.Context,
 	memberships []authtypes.Membership,
 	newProfile authtypes.Profile, userID string, cfg *config.Dendrite,
-	evTime time.Time, queryAPI api.RoomserverQueryAPI,
+	evTime time.Time, rsAPI api.RoomserverInternalAPI,
 ) ([]gomatrixserverlib.HeaderedEvent, error) {
 	evs := []gomatrixserverlib.HeaderedEvent{}
 
 	for _, membership := range memberships {
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
-		if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
+		if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
 			return []gomatrixserverlib.HeaderedEvent{}, err
 		}
 
@@ -366,7 +366,7 @@ func buildMembershipEvents(
 			return nil, err
 		}
 
-		event, err := common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil)
+		event, err := common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
 		if err != nil {
 			return nil, err
 		}

clientapi/routing/routing.go

@@ -49,8 +49,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable"
 func Setup(
 	apiMux *mux.Router, cfg *config.Dendrite,
 	producer *producers.RoomserverProducer,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	accountDB accounts.Database,
 	deviceDB devices.Database,
@@ -60,7 +59,7 @@ func Setup(
 	syncProducer *producers.SyncAPIProducer,
 	eduProducer *producers.EDUServerProducer,
 	transactionsCache *transactions.Cache,
-	federationSender federationSenderAPI.FederationSenderQueryAPI,
+	federationSender federationSenderAPI.FederationSenderInternalAPI,
 ) {
 
 	apiMux.Handle("/_matrix/client/versions",
@@ -91,7 +90,7 @@ func Setup(
 
 	r0mux.Handle("/createRoom",
 		common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI, asAPI)
+			return CreateRoom(req, device, cfg, producer, accountDB, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
@@ -101,7 +100,8 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return JoinRoomByIDOrAlias(
-				req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB,
+				req, device, vars["roomIDOrAlias"], cfg, federation, producer,
+				rsAPI, federationSender, keyRing, accountDB,
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -117,7 +117,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer)
+			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI, producer)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
@@ -126,7 +126,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil)
+			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, producer, nil)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
@@ -137,7 +137,7 @@ func Setup(
 			}
 			txnID := vars["txnID"]
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID,
-				nil, cfg, queryAPI, producer, transactionsCache)
+				nil, cfg, rsAPI, producer, transactionsCache)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/event/{eventID}",
@@ -146,7 +146,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing)
+			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -155,7 +155,7 @@ func Setup(
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateRequest(req.Context(), queryAPI, vars["roomID"])
+		return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
@@ -163,7 +163,7 @@ func Setup(
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], "")
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "")
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
@@ -171,7 +171,7 @@ func Setup(
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], vars["stateKey"])
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
@@ -186,7 +186,7 @@ func Setup(
 			if strings.HasSuffix(eventType, "/") {
 				eventType = eventType[:len(eventType)-1]
 			}
-			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil)
+			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, producer, nil)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
@@ -197,7 +197,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			stateKey := vars["stateKey"]
-			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil)
+			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, producer, nil)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
@@ -219,7 +219,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return DirectoryRoom(req, vars["roomAlias"], federation, cfg, aliasAPI, federationSender)
+			return DirectoryRoom(req, vars["roomAlias"], federation, cfg, rsAPI, federationSender)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -229,7 +229,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetLocalAlias(req, device, vars["roomAlias"], cfg, aliasAPI)
+			return SetLocalAlias(req, device, vars["roomAlias"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
@@ -239,7 +239,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI)
+			return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
 
@@ -353,7 +353,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI)
+			return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
@@ -375,7 +375,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI)
+			return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
@@ -488,7 +488,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI)
+			return GetMemberships(req, device, vars["roomID"], false, cfg, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -498,7 +498,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI)
+			return GetMemberships(req, device, vars["roomID"], true, cfg, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -602,7 +602,7 @@ func Setup(
 
 	r0mux.Handle("/capabilities",
 		common.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			return GetCapabilities(req, queryAPI)
+			return GetCapabilities(req, rsAPI)
 		}),
 	).Methods(http.MethodGet)
 }

clientapi/routing/sendevent.go

@@ -45,13 +45,13 @@ func SendEvent(
 	device *authtypes.Device,
 	roomID, eventType string, txnID, stateKey *string,
 	cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	producer *producers.RoomserverProducer,
 	txnCache *transactions.Cache,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -65,7 +65,7 @@ func SendEvent(
 		}
 	}
 
-	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, queryAPI)
+	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI)
 	if resErr != nil {
 		return *resErr
 	}
@@ -115,7 +115,7 @@ func generateSendEvent(
 	device *authtypes.Device,
 	roomID, eventType string, stateKey *string,
 	cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	// parse the incoming http request
 	userID := device.UserID
@@ -148,7 +148,7 @@ func generateSendEvent(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, queryAPI, &queryRes)
+	e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes)
 	if err == common.ErrRoomNoExists {
 		return nil, &util.JSONResponse{
 			Code: http.StatusNotFound,

clientapi/routing/state.go

@@ -40,7 +40,7 @@ type stateEventInStateResp struct {
 // TODO: Check if the user is in the room. If not, check if the room's history
 // is publicly visible. Current behaviour is returning an empty array if the
 // user cannot see the room's history.
-func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string) util.JSONResponse {
+func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 	stateReq := api.QueryLatestEventsAndStateRequest{
@@ -48,7 +48,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI
 	}
 	stateRes := api.QueryLatestEventsAndStateResponse{}
 
-	if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
+	if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
 		return jsonerror.InternalServerError()
 	}
@@ -98,7 +98,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI
 // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current
 // state to see if there is an event with that type and state key, if there
 // is then (by default) we return the content, otherwise a 404.
-func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string, evType, stateKey string) util.JSONResponse {
+func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string, evType, stateKey string) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 	util.GetLogger(ctx).WithFields(log.Fields{
@@ -118,7 +118,7 @@ func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQuer
 	}
 	stateRes := api.QueryLatestEventsAndStateResponse{}
 
-	if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
+	if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/threepid/invites.go

@@ -87,7 +87,7 @@ var (
 func CheckAndProcessInvite(
 	ctx context.Context,
 	device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI, db accounts.Database,
+	rsAPI api.RoomserverInternalAPI, db accounts.Database,
 	producer *producers.RoomserverProducer, membership string, roomID string,
 	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
@@ -112,7 +112,7 @@ func CheckAndProcessInvite(
 		// "m.room.third_party_invite" have to be emitted from the data in
 		// storeInviteRes.
 		err = emit3PIDInviteEvent(
-			ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer, evTime,
+			ctx, body, storeInviteRes, device, roomID, cfg, rsAPI, producer, evTime,
 		)
 		inviteStoredOnIDServer = err == nil
 
@@ -331,7 +331,7 @@ func emit3PIDInviteEvent(
 	ctx context.Context,
 	body *MembershipRequest, res *idServerStoreInviteResponse,
 	device *authtypes.Device, roomID string, cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer,
+	rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer,
 	evTime time.Time,
 ) error {
 	builder := &gomatrixserverlib.EventBuilder{
@@ -354,7 +354,7 @@ func emit3PIDInviteEvent(
 	}
 
 	queryRes := api.QueryLatestEventsAndStateResponse{}
-	event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, &queryRes)
+	event, err := common.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes)
 	if err != nil {
 		return err
 	}

cmd/dendrite-appservice-server/main.go

@@ -28,11 +28,11 @@ func main() {
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
 	federation := base.CreateFederationClient()
-	alias, _, query := base.CreateHTTPRoomserverAPIs()
+	rsAPI := base.CreateHTTPRoomserverAPIs()
 	cache := transactions.New()
 
 	appservice.SetupAppServiceAPIComponent(
-		base, accountDB, deviceDB, federation, alias, query, cache,
+		base, accountDB, deviceDB, federation, rsAPI, cache,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI))

cmd/dendrite-client-api-server/main.go

@@ -36,13 +36,14 @@ func main() {
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
 	asQuery := base.CreateHTTPAppServiceAPIs()
-	alias, input, query := base.CreateHTTPRoomserverAPIs()
+	rsAPI := base.CreateHTTPRoomserverAPIs()
-	fedSenderAPI := base.CreateHTTPFederationSenderAPIs()
+	fsAPI := base.CreateHTTPFederationSenderAPIs()
+	rsAPI.SetFederationSenderAPI(fsAPI)
 	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB, federation, &keyRing,
-		alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
+		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))

cmd/dendrite-demo-libp2p/main.go

@@ -149,27 +149,34 @@ func main() {
 	federation := createFederationClient(base)
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	alias, input, query := roomserver.SetupRoomServerComponent(&base.Base, keyRing)
+	rsAPI := roomserver.SetupRoomServerComponent(
-	eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New())
+		&base.Base, keyRing, federation,
-	asQuery := appservice.SetupAppServiceAPIComponent(
-		&base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(),
 	)
-	fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input)
+	eduInputAPI := eduserver.SetupEDUServerComponent(
+		&base.Base, cache.New(),
+	)
+	asAPI := appservice.SetupAppServiceAPIComponent(
+		&base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
+	)
+	fsAPI := federationsender.SetupFederationSenderComponent(
+		&base.Base, federation, rsAPI, &keyRing,
+	)
+	rsAPI.SetFederationSenderAPI(fsAPI)
 
 	clientapi.SetupClientAPIComponent(
 		&base.Base, deviceDB, accountDB,
-		federation, &keyRing, alias, input, query,
+		federation, &keyRing, rsAPI,
-		eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
+		eduInputAPI, asAPI, transactions.New(), fsAPI,
 	)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-	federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
+	federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB)
 	publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, query, federation, nil) // Check this later
+	publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later
-	syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, query, federation, &cfg)
+	syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.Base.APIMux)
 

cmd/dendrite-federation-api-server/main.go

@@ -32,17 +32,18 @@ func main() {
 	deviceDB := base.CreateDeviceDB()
 	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
-	federationSender := base.CreateHTTPFederationSenderAPIs()
+	fsAPI := base.CreateHTTPFederationSenderAPIs()
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	alias, input, query := base.CreateHTTPRoomserverAPIs()
+	rsAPI := base.CreateHTTPRoomserverAPIs()
-	asQuery := base.CreateHTTPAppServiceAPIs()
+	asAPI := base.CreateHTTPAppServiceAPIs()
+	rsAPI.SetFederationSenderAPI(fsAPI)
 	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 
 	federationapi.SetupFederationAPIComponent(
 		base, accountDB, deviceDB, federation, &keyRing,
-		alias, input, query, asQuery, federationSender, eduProducer,
+		rsAPI, asAPI, fsAPI, eduProducer,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))

cmd/dendrite-federation-sender-server/main.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/common/keydb"
 	"github.com/matrix-org/dendrite/federationsender"
 )
 
@@ -25,12 +26,13 @@ func main() {
 	defer base.Close() // nolint: errcheck
 
 	federation := base.CreateFederationClient()
-
+	keyDB := base.CreateKeyDB()
-	_, input, query := base.CreateHTTPRoomserverAPIs()
+	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
-
+	rsAPI := base.CreateHTTPRoomserverAPIs()
-	federationsender.SetupFederationSenderComponent(
+	fsAPI := federationsender.SetupFederationSenderComponent(
-		base, federation, query, input,
+		base, federation, rsAPI, &keyRing,
 	)
+	rsAPI.SetFederationSenderAPI(fsAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender))
 

cmd/dendrite-monolith-server/main.go

@@ -57,27 +57,34 @@ func main() {
 	federation := base.CreateFederationClient()
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing)
+	rsAPI := roomserver.SetupRoomServerComponent(
-	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
+		base, keyRing, federation,
-	asQuery := appservice.SetupAppServiceAPIComponent(
-		base, accountDB, deviceDB, federation, alias, query, transactions.New(),
 	)
-	fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input)
+	eduInputAPI := eduserver.SetupEDUServerComponent(
+		base, cache.New(),
+	)
+	asAPI := appservice.SetupAppServiceAPIComponent(
+		base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
+	)
+	fsAPI := federationsender.SetupFederationSenderComponent(
+		base, federation, rsAPI, &keyRing,
+	)
+	rsAPI.SetFederationSenderAPI(fsAPI)
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB,
-		federation, &keyRing, alias, input, query,
+		federation, &keyRing, rsAPI,
-		eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
+		eduInputAPI, asAPI, transactions.New(), fsAPI,
 	)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
+	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
 	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, nil)
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil)
-	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
+	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)
 

cmd/dendrite-public-rooms-api-server/main.go

@@ -28,12 +28,15 @@ func main() {
 
 	deviceDB := base.CreateDeviceDB()
 
-	_, _, query := base.CreateHTTPRoomserverAPIs()
+	fsAPI := base.CreateHTTPFederationSenderAPIs()
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
+	rsAPI := base.CreateHTTPRoomserverAPIs()
+	rsAPI.SetFederationSenderAPI(fsAPI)
+
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, nil, nil)
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, nil, nil)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI))
 

cmd/dendrite-room-server/main.go

@@ -30,7 +30,9 @@ func main() {
 	federation := base.CreateFederationClient()
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	roomserver.SetupRoomServerComponent(base, keyRing)
+	fsAPI := base.CreateHTTPFederationSenderAPIs()
+	rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation)
+	rsAPI.SetFederationSenderAPI(fsAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer))
 

cmd/dendrite-sync-api-server/main.go

@@ -28,9 +28,9 @@ func main() {
 	accountDB := base.CreateAccountsDB()
 	federation := base.CreateFederationClient()
 
-	_, _, query := base.CreateHTTPRoomserverAPIs()
+	rsAPI := base.CreateHTTPRoomserverAPIs()
 
-	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
+	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI))
 

cmd/dendritejs/main.go

@@ -123,27 +123,28 @@ func main() {
 	}
 	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node)
 
-	alias, input, query := roomserver.SetupRoomServerComponent(base)
+	rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation)
 	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
 	asQuery := appservice.SetupAppServiceAPIComponent(
-		base, accountDB, deviceDB, federation, alias, query, transactions.New(),
+		base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
 	)
-	fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query)
+	fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing)
+	rsAPI.SetFederationSenderAPI(fedSenderAPI)
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB,
-		federation, &keyRing, alias, input, query,
+		federation, &keyRing, rsAPI,
 		eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
 	)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
+	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
 	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, p2pPublicRoomProvider)
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider)
-	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
+	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)
 

cmd/roomserver-integration-tests/main.go

@@ -209,7 +209,7 @@ func writeToRoomServer(input []string, roomserverURL string) error {
 			return err
 		}
 	}
-	x, err := api.NewRoomserverInputAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP})
+	x, err := api.NewRoomserverInternalAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}, nil)
 	if err != nil {
 		return err
 	}
@@ -225,7 +225,7 @@ func writeToRoomServer(input []string, roomserverURL string) error {
 // Once those messages have been written it runs the checkQueries function passing
 // a api.RoomserverQueryAPI client. The caller can use this function to check the
 // behaviour of the query API.
-func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) {
+func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverInternalAPI)) {
 	dir, err := ioutil.TempDir("", "room-server-test")
 	if err != nil {
 		panic(err)
@@ -276,7 +276,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 	cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)}
 
 	gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() {
-		queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache)
+		queryAPI, _ := api.NewRoomserverInternalAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache)
 		checkQueries(queryAPI)
 	})
 	if err != nil {
@@ -410,7 +410,7 @@ func main() {
 		}}`,
 	}
 
-	testRoomserver(input, want, func(q api.RoomserverQueryAPI) {
+	testRoomserver(input, want, func(q api.RoomserverInternalAPI) {
 		var response api.QueryLatestEventsAndStateResponse
 		if err := q.QueryLatestEventsAndState(
 			context.Background(),

common/basecomponent/base.go

@@ -119,24 +119,12 @@ func (b *BaseDendrite) CreateHTTPAppServiceAPIs() appserviceAPI.AppServiceQueryA
 
 // CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI for hitting
 // the roomserver over HTTP.
-func (b *BaseDendrite) CreateHTTPRoomserverAPIs() (
+func (b *BaseDendrite) CreateHTTPRoomserverAPIs() roomserverAPI.RoomserverInternalAPI {
-	roomserverAPI.RoomserverAliasAPI,
+	rsAPI, err := roomserverAPI.NewRoomserverInternalAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache)
-	roomserverAPI.RoomserverInputAPI,
-	roomserverAPI.RoomserverQueryAPI,
-) {
-	alias, err := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient)
 	if err != nil {
-		logrus.WithError(err).Panic("NewRoomserverAliasAPIHTTP failed")
+		logrus.WithError(err).Panic("NewRoomserverInternalAPIHTTP failed", b.httpClient)
 	}
-	input, err := roomserverAPI.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient)
+	return rsAPI
-	if err != nil {
-		logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient)
-	}
-	query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache)
-	if err != nil {
-		logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient)
-	}
-	return alias, input, query
 }
 
 // CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU
@@ -149,12 +137,12 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI
 	return e
 }
 
-// CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting
+// CreateHTTPFederationSenderAPIs returns FederationSenderInternalAPI for hitting
 // the federation sender over HTTP
-func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI {
+func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderInternalAPI {
-	f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient)
+	f, err := federationSenderAPI.NewFederationSenderInternalAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient)
 	if err != nil {
-		logrus.WithError(err).Panic("NewFederationSenderQueryAPIHTTP failed", b.httpClient)
+		logrus.WithError(err).Panic("NewFederationSenderInternalAPIHTTP failed", b.httpClient)
 	}
 	return f
 }

common/events.go

@@ -39,13 +39,13 @@ var ErrRoomNoExists = errors.New("Room does not exist")
 func BuildEvent(
 	ctx context.Context,
 	builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time,
-	queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse,
+	rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse,
 ) (*gomatrixserverlib.Event, error) {
 	if queryRes == nil {
 		queryRes = &api.QueryLatestEventsAndStateResponse{}
 	}
 
-	err := AddPrevEventsToEvent(ctx, builder, queryAPI, queryRes)
+	err := AddPrevEventsToEvent(ctx, builder, rsAPI, queryRes)
 	if err != nil {
 		// This can pass through a ErrRoomNoExists to the caller
 		return nil, err
@@ -66,7 +66,7 @@ func BuildEvent(
 func AddPrevEventsToEvent(
 	ctx context.Context,
 	builder *gomatrixserverlib.EventBuilder,
-	queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse,
+	rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse,
 ) error {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
 	if err != nil {
@@ -82,8 +82,8 @@ func AddPrevEventsToEvent(
 		RoomID:       builder.RoomID,
 		StateToFetch: eventsNeeded.Tuples(),
 	}
-	if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil {
+	if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil {
-		return fmt.Errorf("queryAPI.QueryLatestEventsAndState: %w", err)
+		return fmt.Errorf("rsAPI.QueryLatestEventsAndState: %w", err)
 	}
 
 	if !queryRes.RoomExists {

federationapi/federationapi.go

@@ -36,18 +36,16 @@ func SetupFederationAPIComponent(
 	deviceDB devices.Database,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing *gomatrixserverlib.KeyRing,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	inputAPI roomserverAPI.RoomserverInputAPI,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
-	federationSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 	eduProducer *producers.EDUServerProducer,
 ) {
-	roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI)
+	roomserverProducer := producers.NewRoomserverProducer(rsAPI)
 
 	routing.Setup(
-		base.APIMux, base.Cfg, queryAPI, aliasAPI, asAPI,
+		base.APIMux, base.Cfg, rsAPI, asAPI, roomserverProducer,
-		roomserverProducer, eduProducer, federationSenderAPI, *keyRing,
+		eduProducer, federationSenderAPI, *keyRing,
 		federation, accountsDB, deviceDB,
 	)
 }

federationapi/routing/backfill.go

@@ -33,7 +33,7 @@ import (
 func Backfill(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
@@ -82,7 +82,7 @@ func Backfill(
 	}
 
 	// Query the roomserver.
-	if err = query.QueryBackfill(httpReq.Context(), &req, &res); err != nil {
+	if err = rsAPI.QueryBackfill(httpReq.Context(), &req, &res); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed")
 		return jsonerror.InternalServerError()
 	}

federationapi/routing/eventauth.go

@@ -25,13 +25,13 @@ import (
 func GetEventAuth(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 	eventID string,
 ) util.JSONResponse {
 	// TODO: Optimisation: we shouldn't be querying all the room state
 	// that is in state.StateEvents - we just ignore it.
-	state, err := getState(ctx, request, query, roomID, eventID)
+	state, err := getState(ctx, request, rsAPI, roomID, eventID)
 	if err != nil {
 		return *err
 	}

federationapi/routing/events.go

@@ -16,7 +16,9 @@ package routing
 
 import (
 	"context"
+	"encoding/json"
 	"net/http"
+	"time"
 
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -27,15 +29,22 @@ import (
 func GetEvent(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	eventID string,
+	origin gomatrixserverlib.ServerName,
 ) util.JSONResponse {
-	event, err := getEvent(ctx, request, query, eventID)
+	event, err := getEvent(ctx, request, rsAPI, eventID)
 	if err != nil {
 		return *err
 	}
 
-	return util.JSONResponse{Code: http.StatusOK, JSON: event}
+	return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.Transaction{
+		Origin:         origin,
+		OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
+		PDUs: []json.RawMessage{
+			event.JSON(),
+		},
+	}}
 }
 
 // getEvent returns the requested event,
@@ -43,11 +52,11 @@ func GetEvent(
 func getEvent(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	eventID string,
 ) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	var authResponse api.QueryServerAllowedToSeeEventResponse
-	err := query.QueryServerAllowedToSeeEvent(
+	err := rsAPI.QueryServerAllowedToSeeEvent(
 		ctx,
 		&api.QueryServerAllowedToSeeEventRequest{
 			EventID:    eventID,
@@ -66,7 +75,7 @@ func getEvent(
 	}
 
 	var eventsResponse api.QueryEventsByIDResponse
-	err = query.QueryEventsByID(
+	err = rsAPI.QueryEventsByID(
 		ctx,
 		&api.QueryEventsByIDRequest{EventIDs: []string{eventID}},
 		&eventsResponse,

federationapi/routing/join.go

@@ -33,13 +33,13 @@ func MakeJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID, userID string,
 	remoteVersions []gomatrixserverlib.RoomVersion,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.InternalServerError(),
@@ -97,7 +97,7 @@ func MakeJoin(
 	queryRes := api.QueryLatestEventsAndStateResponse{
 		RoomVersion: verRes.RoomVersion,
 	}
-	event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes)
+	event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
 	if err == common.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -136,15 +136,15 @@ func SendJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	producer *producers.RoomserverProducer,
 	keys gomatrixserverlib.KeyRing,
 	roomID, eventID string,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryRoomVersionForRoom failed")
+		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryRoomVersionForRoom failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.InternalServerError(),
@@ -216,14 +216,14 @@ func SendJoin(
 	// Fetch the state and auth chain. We do this before we send the events
 	// on, in case this fails.
 	var stateAndAuthChainResponse api.QueryStateAndAuthChainResponse
-	err = query.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{
+	err = rsAPI.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{
 		PrevEventIDs: event.PrevEventIDs(),
 		AuthEventIDs: event.AuthEventIDs(),
 		RoomID:       roomID,
 		ResolveState: true,
 	}, &stateAndAuthChainResponse)
 	if err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryStateAndAuthChain failed")
+		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryStateAndAuthChain failed")
 		return jsonerror.InternalServerError()
 	}
 

federationapi/routing/leave.go

@@ -30,7 +30,7 @@ func MakeLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID, userID string,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
@@ -61,7 +61,7 @@ func MakeLeave(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes)
+	event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
 	if err == common.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -102,7 +102,7 @@ func SendLeave(
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := producer.QueryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
+	if err := producer.RsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),

federationapi/routing/missingevents.go

@@ -34,7 +34,7 @@ type getMissingEventRequest struct {
 func GetMissingEvents(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 ) util.JSONResponse {
 	var gme getMissingEventRequest
@@ -46,7 +46,7 @@ func GetMissingEvents(
 	}
 
 	var eventsResponse api.QueryMissingEventsResponse
-	if err := query.QueryMissingEvents(
+	if err := rsAPI.QueryMissingEvents(
 		httpReq.Context(), &api.QueryMissingEventsRequest{
 			EarliestEvents: gme.EarliestEvents,
 			LatestEvents:   gme.LatestEvents,

federationapi/routing/query.go

@@ -32,8 +32,8 @@ func RoomAliasToID(
 	httpReq *http.Request,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.Dendrite,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	senderAPI federationSenderAPI.FederationSenderQueryAPI,
+	senderAPI federationSenderAPI.FederationSenderInternalAPI,
 ) util.JSONResponse {
 	roomAlias := httpReq.FormValue("room_alias")
 	if roomAlias == "" {
@@ -55,7 +55,7 @@ func RoomAliasToID(
 	if domain == cfg.Matrix.ServerName {
 		queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
 		var queryRes roomserverAPI.GetRoomIDForAliasResponse
-		if err = aliasAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil {
+		if err = rsAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil {
 			util.GetLogger(httpReq.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed")
 			return jsonerror.InternalServerError()
 		}

federationapi/routing/routing.go

@@ -44,12 +44,11 @@ const (
 func Setup(
 	apiMux *mux.Router,
 	cfg *config.Dendrite,
-	query roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	producer *producers.RoomserverProducer,
 	eduProducer *producers.EDUServerProducer,
-	federationSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 	keys gomatrixserverlib.KeyRing,
 	federation *gomatrixserverlib.FederationClient,
 	accountDB accounts.Database,
@@ -80,7 +79,7 @@ func Setup(
 			}
 			return Send(
 				httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]),
-				cfg, query, producer, eduProducer, keys, federation,
+				cfg, rsAPI, producer, eduProducer, keys, federation,
 			)
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
@@ -101,7 +100,7 @@ func Setup(
 
 	v1fedmux.Handle("/3pid/onbind", common.MakeExternalAPI("3pid_onbind",
 		func(req *http.Request) util.JSONResponse {
-			return CreateInvitesFrom3PIDInvites(req, query, asAPI, cfg, producer, federation, accountDB)
+			return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, producer, federation, accountDB)
 		},
 	)).Methods(http.MethodPost, http.MethodOptions)
 
@@ -113,7 +112,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return ExchangeThirdPartyInvite(
-				httpReq, request, vars["roomID"], query, cfg, federation, producer,
+				httpReq, request, vars["roomID"], rsAPI, cfg, federation, producer,
 			)
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
@@ -126,7 +125,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return GetEvent(
-				httpReq.Context(), request, query, vars["eventID"],
+				httpReq.Context(), request, rsAPI, vars["eventID"], cfg.Matrix.ServerName,
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -139,7 +138,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return GetState(
-				httpReq.Context(), request, query, vars["roomID"],
+				httpReq.Context(), request, rsAPI, vars["roomID"],
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -152,7 +151,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return GetStateIDs(
-				httpReq.Context(), request, query, vars["roomID"],
+				httpReq.Context(), request, rsAPI, vars["roomID"],
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -162,7 +161,7 @@ func Setup(
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars := mux.Vars(httpReq)
 			return GetEventAuth(
-				httpReq.Context(), request, query, vars["roomID"], vars["eventID"],
+				httpReq.Context(), request, rsAPI, vars["roomID"], vars["eventID"],
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -171,7 +170,7 @@ func Setup(
 		"federation_query_room_alias", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			return RoomAliasToID(
-				httpReq, federation, cfg, aliasAPI, federationSenderAPI,
+				httpReq, federation, cfg, rsAPI, federationSenderAPI,
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -222,7 +221,7 @@ func Setup(
 				remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1)
 			}
 			return MakeJoin(
-				httpReq, request, cfg, query, roomID, eventID, remoteVersions,
+				httpReq, request, cfg, rsAPI, roomID, eventID, remoteVersions,
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -237,7 +236,7 @@ func Setup(
 			roomID := vars["roomID"]
 			eventID := vars["eventID"]
 			return SendJoin(
-				httpReq, request, cfg, query, producer, keys, roomID, eventID,
+				httpReq, request, cfg, rsAPI, producer, keys, roomID, eventID,
 			)
 		},
 	)).Methods(http.MethodPut)
@@ -252,7 +251,7 @@ func Setup(
 			roomID := vars["roomID"]
 			eventID := vars["eventID"]
 			return MakeLeave(
-				httpReq, request, cfg, query, roomID, eventID,
+				httpReq, request, cfg, rsAPI, roomID, eventID,
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -286,7 +285,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetMissingEvents(httpReq, request, query, vars["roomID"])
+			return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"])
 		},
 	)).Methods(http.MethodPost)
 
@@ -297,7 +296,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return Backfill(httpReq, request, query, vars["roomID"], cfg)
+			return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg)
 		},
 	)).Methods(http.MethodGet)
 }

federationapi/routing/send.go

@@ -34,7 +34,7 @@ func Send(
 	request *gomatrixserverlib.FederationRequest,
 	txnID gomatrixserverlib.TransactionID,
 	cfg *config.Dendrite,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	producer *producers.RoomserverProducer,
 	eduProducer *producers.EDUServerProducer,
 	keys gomatrixserverlib.KeyRing,
@@ -42,7 +42,7 @@ func Send(
 ) util.JSONResponse {
 	t := txnReq{
 		context:     httpReq.Context(),
-		query:       query,
+		rsAPI:       rsAPI,
 		producer:    producer,
 		eduProducer: eduProducer,
 		keys:        keys,
@@ -99,7 +99,7 @@ func Send(
 type txnReq struct {
 	gomatrixserverlib.Transaction
 	context     context.Context
-	query       api.RoomserverQueryAPI
+	rsAPI       api.RoomserverInternalAPI
 	producer    *producers.RoomserverProducer
 	eduProducer *producers.EDUServerProducer
 	keys        gomatrixserverlib.KeyRing
@@ -120,7 +120,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
 		}
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
-		if err := t.query.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil {
+		if err := t.rsAPI.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil {
 			util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID)
 			return nil, roomNotFoundError{verReq.RoomID}
 		}
@@ -228,7 +228,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error {
 		StateToFetch: needed.Tuples(),
 	}
 	var stateResp api.QueryStateAfterEventsResponse
-	if err := t.query.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil {
+	if err := t.rsAPI.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil {
 		return err
 	}
 

federationapi/routing/state.go

@@ -27,7 +27,7 @@ import (
 func GetState(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 ) util.JSONResponse {
 	eventID, err := parseEventIDParam(request)
@@ -35,7 +35,7 @@ func GetState(
 		return *err
 	}
 
-	state, err := getState(ctx, request, query, roomID, eventID)
+	state, err := getState(ctx, request, rsAPI, roomID, eventID)
 	if err != nil {
 		return *err
 	}
@@ -47,7 +47,7 @@ func GetState(
 func GetStateIDs(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 ) util.JSONResponse {
 	eventID, err := parseEventIDParam(request)
@@ -55,7 +55,7 @@ func GetStateIDs(
 		return *err
 	}
 
-	state, err := getState(ctx, request, query, roomID, eventID)
+	state, err := getState(ctx, request, rsAPI, roomID, eventID)
 	if err != nil {
 		return *err
 	}
@@ -94,11 +94,11 @@ func parseEventIDParam(
 func getState(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 	eventID string,
 ) (*gomatrixserverlib.RespState, *util.JSONResponse) {
-	event, resErr := getEvent(ctx, request, query, eventID)
+	event, resErr := getEvent(ctx, request, rsAPI, eventID)
 	if resErr != nil {
 		return nil, resErr
 	}
@@ -110,7 +110,7 @@ func getState(
 	authEventIDs := getIDsFromEventRef(event.AuthEvents())
 
 	var response api.QueryStateAndAuthChainResponse
-	err := query.QueryStateAndAuthChain(
+	err := rsAPI.QueryStateAndAuthChain(
 		ctx,
 		&api.QueryStateAndAuthChainRequest{
 			RoomID:       roomID,

federationapi/routing/threepid.go

@@ -58,7 +58,7 @@ var (
 
 // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind
 func CreateInvitesFrom3PIDInvites(
-	req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI,
+	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite,
 	producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient,
 	accountDB accounts.Database,
@@ -72,7 +72,7 @@ func CreateInvitesFrom3PIDInvites(
 	for _, inv := range body.Invites {
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
-		if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+		if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -80,7 +80,7 @@ func CreateInvitesFrom3PIDInvites(
 		}
 
 		event, err := createInviteFrom3PIDInvite(
-			req.Context(), queryAPI, asAPI, cfg, inv, federation, accountDB,
+			req.Context(), rsAPI, asAPI, cfg, inv, federation, accountDB,
 		)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed")
@@ -108,7 +108,7 @@ func ExchangeThirdPartyInvite(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 	federation *gomatrixserverlib.FederationClient,
 	producer *producers.RoomserverProducer,
@@ -148,7 +148,7 @@ func ExchangeThirdPartyInvite(
 
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err = queryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
+	if err = rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -156,7 +156,7 @@ func ExchangeThirdPartyInvite(
 	}
 
 	// Auth and build the event from what the remote server sent us
-	event, err := buildMembershipEvent(httpReq.Context(), &builder, queryAPI, cfg)
+	event, err := buildMembershipEvent(httpReq.Context(), &builder, rsAPI, cfg)
 	if err == errNotInRoom {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -199,14 +199,14 @@ func ExchangeThirdPartyInvite(
 // Returns an error if there was a problem building the event or fetching the
 // necessary data to do so.
 func createInviteFrom3PIDInvite(
-	ctx context.Context, queryAPI roomserverAPI.RoomserverQueryAPI,
+	ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite,
 	inv invite, federation *gomatrixserverlib.FederationClient,
 	accountDB accounts.Database,
 ) (*gomatrixserverlib.Event, error) {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
 		return nil, err
 	}
 
@@ -245,7 +245,7 @@ func createInviteFrom3PIDInvite(
 		return nil, err
 	}
 
-	event, err := buildMembershipEvent(ctx, builder, queryAPI, cfg)
+	event, err := buildMembershipEvent(ctx, builder, rsAPI, cfg)
 	if err == errNotInRoom {
 		return nil, sendToRemoteServer(ctx, inv, federation, cfg, *builder)
 	}
@@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite(
 // Returns an error if something failed during the process.
 func buildMembershipEvent(
 	ctx context.Context,
-	builder *gomatrixserverlib.EventBuilder, queryAPI roomserverAPI.RoomserverQueryAPI,
+	builder *gomatrixserverlib.EventBuilder, rsAPI roomserverAPI.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 ) (*gomatrixserverlib.Event, error) {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
@@ -281,7 +281,7 @@ func buildMembershipEvent(
 		StateToFetch: eventsNeeded.Tuples(),
 	}
 	var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
-	if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil {
+	if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil {
 		return nil, err
 	}
 

federationsender/api/api.go

@@ -0,0 +1,53 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+)
+
+// FederationSenderInternalAPI is used to query information from the federation sender.
+type FederationSenderInternalAPI interface {
+	// Query the joined hosts and the membership events accounting for their participation in a room.
+	// Note that if a server has multiple users in the room, it will have multiple entries in the returned slice.
+	// See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version.
+	QueryJoinedHostsInRoom(
+		ctx context.Context,
+		request *QueryJoinedHostsInRoomRequest,
+		response *QueryJoinedHostsInRoomResponse,
+	) error
+	// Query the server names of the joined hosts in a room.
+	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
+	// containing only the server names (without information for membership events).
+	QueryJoinedHostServerNamesInRoom(
+		ctx context.Context,
+		request *QueryJoinedHostServerNamesInRoomRequest,
+		response *QueryJoinedHostServerNamesInRoomResponse,
+	) error
+	// Handle an instruction to make_join & send_join with a remote server.
+	PerformJoin(
+		ctx context.Context,
+		request *PerformJoinRequest,
+		response *PerformJoinResponse,
+	) error
+	// Handle an instruction to make_leave & send_leave with a remote server.
+	PerformLeave(
+		ctx context.Context,
+		request *PerformLeaveRequest,
+		response *PerformLeaveResponse,
+	) error
+}
+
+// NewFederationSenderInternalAPIHTTP creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API.
+// If httpClient is nil an error is returned
+func NewFederationSenderInternalAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderInternalAPI, error) {
+	if httpClient == nil {
+		return nil, errors.New("NewFederationSenderInternalAPIHTTP: httpClient is <nil>")
+	}
+	return &httpFederationSenderInternalAPI{federationSenderURL, httpClient}, nil
+}
+
+type httpFederationSenderInternalAPI struct {
+	federationSenderURL string
+	httpClient          *http.Client
+}

federationsender/api/perform.go

@@ -0,0 +1,60 @@
+package api
+
+import (
+	"context"
+
+	commonHTTP "github.com/matrix-org/dendrite/common/http"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/opentracing/opentracing-go"
+)
+
+const (
+	// FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API.
+	FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest"
+
+	// FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API.
+	FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest"
+)
+
+type PerformJoinRequest struct {
+	RoomID     string                       `json:"room_id"`
+	UserID     string                       `json:"user_id"`
+	ServerName gomatrixserverlib.ServerName `json:"server_name"`
+	Content    map[string]interface{}       `json:"content"`
+}
+
+type PerformJoinResponse struct {
+}
+
+// Handle an instruction to make_join & send_join with a remote server.
+func (h *httpFederationSenderInternalAPI) PerformJoin(
+	ctx context.Context,
+	request *PerformJoinRequest,
+	response *PerformJoinResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
+type PerformLeaveRequest struct {
+	RoomID string `json:"room_id"`
+}
+
+type PerformLeaveResponse struct {
+}
+
+// Handle an instruction to make_leave & send_leave with a remote server.
+func (h *httpFederationSenderInternalAPI) PerformLeave(
+	ctx context.Context,
+	request *PerformLeaveRequest,
+	response *PerformLeaveResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

federationsender/api/query.go

@@ -2,16 +2,20 @@ package api
 
 import (
 	"context"
-	"errors"
-	"net/http"
 
 	commonHTTP "github.com/matrix-org/dendrite/common/http"
+	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/gomatrixserverlib"
 
-	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/opentracing/opentracing-go"
 )
 
+// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API.
+const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom"
+
+// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API.
+const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom"
+
 // QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom
 type QueryJoinedHostsInRoomRequest struct {
 	RoomID string `json:"room_id"`
@@ -22,6 +26,19 @@ type QueryJoinedHostsInRoomResponse struct {
 	JoinedHosts []types.JoinedHost `json:"joined_hosts"`
 }
 
+// QueryJoinedHostsInRoom implements FederationSenderInternalAPI
+func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom(
+	ctx context.Context,
+	request *QueryJoinedHostsInRoomRequest,
+	response *QueryJoinedHostsInRoomResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
 // QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames
 type QueryJoinedHostServerNamesInRoomRequest struct {
 	RoomID string `json:"room_id"`
@@ -32,61 +49,8 @@ type QueryJoinedHostServerNamesInRoomResponse struct {
 	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
 }
 
-// FederationSenderQueryAPI is used to query information from the federation sender.
+// QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI
-type FederationSenderQueryAPI interface {
+func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom(
-	// Query the joined hosts and the membership events accounting for their participation in a room.
-	// Note that if a server has multiple users in the room, it will have multiple entries in the returned slice.
-	// See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version.
-	QueryJoinedHostsInRoom(
-		ctx context.Context,
-		request *QueryJoinedHostsInRoomRequest,
-		response *QueryJoinedHostsInRoomResponse,
-	) error
-	// Query the server names of the joined hosts in a room.
-	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
-	// containing only the server names (without information for membership events).
-	QueryJoinedHostServerNamesInRoom(
-		ctx context.Context,
-		request *QueryJoinedHostServerNamesInRoomRequest,
-		response *QueryJoinedHostServerNamesInRoomResponse,
-	) error
-}
-
-// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API.
-const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom"
-
-// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API.
-const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom"
-
-// NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API.
-// If httpClient is nil an error is returned
-func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderQueryAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewFederationSenderQueryAPIHTTP: httpClient is <nil>")
-	}
-	return &httpFederationSenderQueryAPI{federationSenderURL, httpClient}, nil
-}
-
-type httpFederationSenderQueryAPI struct {
-	federationSenderURL string
-	httpClient          *http.Client
-}
-
-// QueryJoinedHostsInRoom implements FederationSenderQueryAPI
-func (h *httpFederationSenderQueryAPI) QueryJoinedHostsInRoom(
-	ctx context.Context,
-	request *QueryJoinedHostsInRoomRequest,
-	response *QueryJoinedHostsInRoomResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom")
-	defer span.Finish()
-
-	apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath
-	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}
-
-// QueryJoinedHostServerNamesInRoom implements FederationSenderQueryAPI
-func (h *httpFederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom(
 	ctx context.Context,
 	request *QueryJoinedHostServerNamesInRoomRequest,
 	response *QueryJoinedHostServerNamesInRoomResponse,

federationsender/consumers/roomserver.go

@@ -33,11 +33,11 @@ import (
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	cfg                *config.Dendrite
+	cfg        *config.Dendrite
-	roomServerConsumer *common.ContinualConsumer
+	rsAPI      api.RoomserverInternalAPI
-	db                 storage.Database
+	rsConsumer *common.ContinualConsumer
-	queues             *queue.OutgoingQueues
+	db         storage.Database
-	query              api.RoomserverQueryAPI
+	queues     *queue.OutgoingQueues
 }
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
@@ -46,7 +46,7 @@ func NewOutputRoomEventConsumer(
 	kafkaConsumer sarama.Consumer,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) *OutputRoomEventConsumer {
 	consumer := common.ContinualConsumer{
 		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
@@ -54,11 +54,11 @@ func NewOutputRoomEventConsumer(
 		PartitionStore: store,
 	}
 	s := &OutputRoomEventConsumer{
-		cfg:                cfg,
+		cfg:        cfg,
-		roomServerConsumer: &consumer,
+		rsConsumer: &consumer,
-		db:                 store,
+		db:         store,
-		queues:             queues,
+		queues:     queues,
-		query:              queryAPI,
+		rsAPI:      rsAPI,
 	}
 	consumer.ProcessMessage = s.onMessage
 
@@ -67,7 +67,7 @@ func NewOutputRoomEventConsumer(
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	return s.roomServerConsumer.Start()
+	return s.rsConsumer.Start()
 }
 
 // onMessage is called when the federation server receives a new event from the room server output log.
@@ -369,7 +369,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 	// from the roomserver using the query API.
 	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
 	var eventResp api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
+	if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
 		return nil, err
 	}
 

federationsender/federationsender.go

@@ -34,21 +34,21 @@ import (
 func SetupFederationSenderComponent(
 	base *basecomponent.BaseDendrite,
 	federation *gomatrixserverlib.FederationClient,
-	rsQueryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	rsInputAPI roomserverAPI.RoomserverInputAPI,
+	keyRing *gomatrixserverlib.KeyRing,
-) api.FederationSenderQueryAPI {
+) api.FederationSenderInternalAPI {
-	federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties())
+	federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender))
 	if err != nil {
 		logrus.WithError(err).Panic("failed to connect to federation sender db")
 	}
 
-	roomserverProducer := producers.NewRoomserverProducer(rsInputAPI, base.Cfg.Matrix.ServerName)
+	roomserverProducer := producers.NewRoomserverProducer(rsAPI, base.Cfg.Matrix.ServerName)
 
 	queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation, roomserverProducer)
 
 	rsConsumer := consumers.NewOutputRoomEventConsumer(
 		base.Cfg, base.KafkaConsumer, queues,
-		federationSenderDB, rsQueryAPI,
+		federationSenderDB, rsAPI,
 	)
 	if err = rsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start room server consumer")
@@ -61,10 +61,10 @@ func SetupFederationSenderComponent(
 		logrus.WithError(err).Panic("failed to start typing server consumer")
 	}
 
-	queryAPI := query.FederationSenderQueryAPI{
+	queryAPI := query.NewFederationSenderInternalAPI(
-		DB: federationSenderDB,
+		federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing,
-	}
+	)
 	queryAPI.SetupHTTP(http.DefaultServeMux)
 
-	return &queryAPI
+	return queryAPI
 }

federationsender/producers/roomserver.go

@@ -23,16 +23,16 @@ import (
 
 // RoomserverProducer produces events for the roomserver to consume.
 type RoomserverProducer struct {
-	InputAPI   api.RoomserverInputAPI
+	InputAPI   api.RoomserverInternalAPI
 	serverName gomatrixserverlib.ServerName
 }
 
 // NewRoomserverProducer creates a new RoomserverProducer
 func NewRoomserverProducer(
-	inputAPI api.RoomserverInputAPI, serverName gomatrixserverlib.ServerName,
+	rsAPI api.RoomserverInternalAPI, serverName gomatrixserverlib.ServerName,
 ) *RoomserverProducer {
 	return &RoomserverProducer{
-		InputAPI:   inputAPI,
+		InputAPI:   rsAPI,
 		serverName: serverName,
 	}
 }
@@ -54,6 +54,42 @@ func (c *RoomserverProducer) SendInviteResponse(
 	return c.SendInputRoomEvents(ctx, []api.InputRoomEvent{ire})
 }
 
+// SendEventWithState writes an event with KindNew to the roomserver input log
+// with the state at the event as KindOutlier before it.
+func (c *RoomserverProducer) SendEventWithState(
+	ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent,
+) error {
+	outliers, err := state.Events()
+	if err != nil {
+		return err
+	}
+
+	var ires []api.InputRoomEvent
+	for _, outlier := range outliers {
+		ires = append(ires, api.InputRoomEvent{
+			Kind:         api.KindOutlier,
+			Event:        outlier.Headered(event.RoomVersion),
+			AuthEventIDs: outlier.AuthEventIDs(),
+		})
+	}
+
+	stateEventIDs := make([]string, len(state.StateEvents))
+	for i := range state.StateEvents {
+		stateEventIDs[i] = state.StateEvents[i].EventID()
+	}
+
+	ires = append(ires, api.InputRoomEvent{
+		Kind:          api.KindNew,
+		Event:         event,
+		AuthEventIDs:  event.AuthEventIDs(),
+		HasState:      true,
+		StateEventIDs: stateEventIDs,
+	})
+
+	_, err = c.SendInputRoomEvents(ctx, ires)
+	return err
+}
+
 // SendInputRoomEvents writes the given input room events to the roomserver input API.
 func (c *RoomserverProducer) SendInputRoomEvents(
 	ctx context.Context, ires []api.InputRoomEvent,

federationsender/query/api.go

@@ -0,0 +1,97 @@
+package query
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/federationsender/producers"
+	"github.com/matrix-org/dendrite/federationsender/storage"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI
+type FederationSenderInternalAPI struct {
+	api.FederationSenderInternalAPI
+	db         storage.Database
+	cfg        *config.Dendrite
+	producer   *producers.RoomserverProducer
+	federation *gomatrixserverlib.FederationClient
+	keyRing    *gomatrixserverlib.KeyRing
+}
+
+func NewFederationSenderInternalAPI(
+	db storage.Database, cfg *config.Dendrite,
+	producer *producers.RoomserverProducer,
+	federation *gomatrixserverlib.FederationClient,
+	keyRing *gomatrixserverlib.KeyRing,
+) *FederationSenderInternalAPI {
+	return &FederationSenderInternalAPI{
+		db:         db,
+		cfg:        cfg,
+		producer:   producer,
+		federation: federation,
+		keyRing:    keyRing,
+	}
+}
+
+// SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux.
+func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) {
+	servMux.Handle(
+		api.FederationSenderQueryJoinedHostsInRoomPath,
+		common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse {
+			var request api.QueryJoinedHostsInRoomRequest
+			var response api.QueryJoinedHostsInRoomResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.FederationSenderQueryJoinedHostServerNamesInRoomPath,
+		common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
+			var request api.QueryJoinedHostServerNamesInRoomRequest
+			var response api.QueryJoinedHostServerNamesInRoomResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(api.FederationSenderPerformJoinRequestPath,
+		common.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse {
+			var request api.PerformJoinRequest
+			var response api.PerformJoinResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := f.PerformJoin(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(api.FederationSenderPerformLeaveRequestPath,
+		common.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse {
+			var request api.PerformLeaveRequest
+			var response api.PerformLeaveResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := f.PerformLeave(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+}

federationsender/query/perform.go

@@ -0,0 +1,120 @@
+package query
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/federationsender/query/perform"
+	"github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// PerformJoinRequest implements api.FederationSenderInternalAPI
+func (r *FederationSenderInternalAPI) PerformJoin(
+	ctx context.Context,
+	request *api.PerformJoinRequest,
+	response *api.PerformJoinResponse,
+) (err error) {
+	// Look up the supported room versions.
+	var supportedVersions []gomatrixserverlib.RoomVersion
+	for version := range version.SupportedRoomVersions() {
+		supportedVersions = append(supportedVersions, version)
+	}
+
+	// Try to perform a make_join using the information supplied in the
+	// request.
+	respMakeJoin, err := r.federation.MakeJoin(
+		ctx,
+		request.ServerName,
+		request.RoomID,
+		request.UserID,
+		supportedVersions,
+	)
+	if err != nil {
+		// TODO: Check if the user was not allowed to join the room.
+		return fmt.Errorf("r.federation.MakeJoin: %w", err)
+	}
+
+	// Set all the fields to be what they should be, this should be a no-op
+	// but it's possible that the remote server returned us something "odd"
+	respMakeJoin.JoinEvent.Type = "m.room.member"
+	respMakeJoin.JoinEvent.Sender = request.UserID
+	respMakeJoin.JoinEvent.StateKey = &request.UserID
+	respMakeJoin.JoinEvent.RoomID = request.RoomID
+	respMakeJoin.JoinEvent.Redacts = ""
+	if request.Content == nil {
+		request.Content = map[string]interface{}{}
+	}
+	request.Content["membership"] = "join"
+	if err = respMakeJoin.JoinEvent.SetContent(request.Content); err != nil {
+		return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err)
+	}
+	if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil {
+		return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err)
+	}
+
+	// Work out if we support the room version that has been supplied in
+	// the make_join response.
+	if respMakeJoin.RoomVersion == "" {
+		respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1
+	}
+	if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil {
+		return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err)
+	}
+
+	// Build the join event.
+	event, err := respMakeJoin.JoinEvent.Build(
+		time.Now(),
+		r.cfg.Matrix.ServerName,
+		r.cfg.Matrix.KeyID,
+		r.cfg.Matrix.PrivateKey,
+		respMakeJoin.RoomVersion,
+	)
+	if err != nil {
+		return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err)
+	}
+
+	// Try to perform a send_join using the newly built event.
+	respSendJoin, err := r.federation.SendJoin(
+		ctx,
+		request.ServerName,
+		event,
+		respMakeJoin.RoomVersion,
+	)
+	if err != nil {
+		return fmt.Errorf("r.federation.SendJoin: %w", err)
+	}
+
+	// Check that the send_join response was valid.
+	joinCtx := perform.JoinContext(r.federation, r.keyRing)
+	if err = joinCtx.CheckSendJoinResponse(
+		ctx, event, request.ServerName, respMakeJoin, respSendJoin,
+	); err != nil {
+		return fmt.Errorf("perform.JoinRequest.CheckSendJoinResponse: %w", err)
+	}
+
+	// If we successfully performed a send_join above then the other
+	// server now thinks we're a part of the room. Send the newly
+	// returned state to the roomserver to update our local view.
+	if err = r.producer.SendEventWithState(
+		ctx,
+		respSendJoin.ToRespState(),
+		event.Headered(respMakeJoin.RoomVersion),
+	); err != nil {
+		return fmt.Errorf("r.producer.SendEventWithState: %w", err)
+	}
+
+	// Everything went to plan.
+	return nil
+}
+
+// PerformLeaveRequest implements api.FederationSenderInternalAPI
+func (r *FederationSenderInternalAPI) PerformLeave(
+	ctx context.Context,
+	request *api.PerformLeaveRequest,
+	response *api.PerformLeaveResponse,
+) (err error) {
+	return nil
+}

federationsender/query/perform/join.go

@@ -0,0 +1,70 @@
+package perform
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// This file contains helpers for the PerformJoin function.
+
+type joinContext struct {
+	federation *gomatrixserverlib.FederationClient
+	keyRing    *gomatrixserverlib.KeyRing
+}
+
+// Returns a new join context.
+func JoinContext(f *gomatrixserverlib.FederationClient, k *gomatrixserverlib.KeyRing) *joinContext {
+	return &joinContext{
+		federation: f,
+		keyRing:    k,
+	}
+}
+
+// checkSendJoinResponse checks that all of the signatures are correct
+// and that the join is allowed by the supplied state.
+func (r joinContext) CheckSendJoinResponse(
+	ctx context.Context,
+	event gomatrixserverlib.Event,
+	server gomatrixserverlib.ServerName,
+	respMakeJoin gomatrixserverlib.RespMakeJoin,
+	respSendJoin gomatrixserverlib.RespSendJoin,
+) error {
+	// A list of events that we have retried, if they were not included in
+	// the auth events supplied in the send_join.
+	retries := map[string]bool{}
+
+retryCheck:
+	// TODO: Can we expand Check here to return a list of missing auth
+	// events rather than failing one at a time?
+	if err := respSendJoin.Check(ctx, r.keyRing, event); err != nil {
+		switch e := err.(type) {
+		case gomatrixserverlib.MissingAuthEventError:
+			// Check that we haven't already retried for this event, prevents
+			// us from ending up in endless loops
+			if !retries[e.AuthEventID] {
+				// Ask the server that we're talking to right now for the event
+				tx, txerr := r.federation.GetEvent(ctx, server, e.AuthEventID)
+				if txerr != nil {
+					return fmt.Errorf("r.federation.GetEvent: %w", txerr)
+				}
+				// For each event returned, add it to the auth events.
+				for _, pdu := range tx.PDUs {
+					ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion)
+					if everr != nil {
+						return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr)
+					}
+					respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev)
+				}
+				// Mark the event as retried and then give the check another go.
+				retries[e.AuthEventID] = true
+				goto retryCheck
+			}
+			return fmt.Errorf("respSendJoin (after retries): %w", e)
+		default:
+			return fmt.Errorf("respSendJoin: %w", err)
+		}
+	}
+	return nil
+}

federationsender/query/query.go

@@ -2,45 +2,28 @@ package query
 
 import (
 	"context"
-	"encoding/json"
-	"net/http"
 
-	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
-// FederationSenderQueryDatabase has the APIs needed to implement the query API.
+// QueryJoinedHostsInRoom implements api.FederationSenderInternalAPI
-type FederationSenderQueryDatabase interface {
+func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom(
-	GetJoinedHosts(
-		ctx context.Context, roomID string,
-	) ([]types.JoinedHost, error)
-}
-
-// FederationSenderQueryAPI is an implementation of api.FederationSenderQueryAPI
-type FederationSenderQueryAPI struct {
-	DB FederationSenderQueryDatabase
-}
-
-// QueryJoinedHostsInRoom implements api.FederationSenderQueryAPI
-func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom(
 	ctx context.Context,
 	request *api.QueryJoinedHostsInRoomRequest,
 	response *api.QueryJoinedHostsInRoomResponse,
 ) (err error) {
-	response.JoinedHosts, err = f.DB.GetJoinedHosts(ctx, request.RoomID)
+	response.JoinedHosts, err = f.db.GetJoinedHosts(ctx, request.RoomID)
 	return
 }
 
-// QueryJoinedHostServerNamesInRoom implements api.FederationSenderQueryAPI
+// QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI
-func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom(
+func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom(
 	ctx context.Context,
 	request *api.QueryJoinedHostServerNamesInRoomRequest,
 	response *api.QueryJoinedHostServerNamesInRoomResponse,
 ) (err error) {
-	joinedHosts, err := f.DB.GetJoinedHosts(ctx, request.RoomID)
+	joinedHosts, err := f.db.GetJoinedHosts(ctx, request.RoomID)
 	if err != nil {
 		return
 	}
@@ -54,35 +37,3 @@ func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom(
 
 	return
 }
-
-// SetupHTTP adds the FederationSenderQueryAPI handlers to the http.ServeMux.
-func (f *FederationSenderQueryAPI) SetupHTTP(servMux *http.ServeMux) {
-	servMux.Handle(
-		api.FederationSenderQueryJoinedHostsInRoomPath,
-		common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryJoinedHostsInRoomRequest
-			var response api.QueryJoinedHostsInRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.FederationSenderQueryJoinedHostServerNamesInRoomPath,
-		common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryJoinedHostServerNamesInRoomRequest
-			var response api.QueryJoinedHostServerNamesInRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-}

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible

go.sum

@@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2 h1:sy2QOqJhb4WXzq8bJhsCntAUYb64Dl6txsFtXWtxxSg=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec h1:9MvZSZzBKvCWqM5KXMGZ1PBDrSLcxs5zfc561UPgcYA=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk=
 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=

publicroomsapi/consumers/roomserver.go

@@ -29,9 +29,9 @@ import (
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	roomServerConsumer *common.ContinualConsumer
+	rsAPI      api.RoomserverInternalAPI
-	db                 storage.Database
+	rsConsumer *common.ContinualConsumer
-	query              api.RoomserverQueryAPI
+	db         storage.Database
 }
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
@@ -39,7 +39,7 @@ func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
 	store storage.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) *OutputRoomEventConsumer {
 	consumer := common.ContinualConsumer{
 		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
@@ -47,9 +47,9 @@ func NewOutputRoomEventConsumer(
 		PartitionStore: store,
 	}
 	s := &OutputRoomEventConsumer{
-		roomServerConsumer: &consumer,
+		rsConsumer: &consumer,
-		db:                 store,
+		db:         store,
-		query:              queryAPI,
+		rsAPI:      rsAPI,
 	}
 	consumer.ProcessMessage = s.onMessage
 
@@ -58,7 +58,7 @@ func NewOutputRoomEventConsumer(
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	return s.roomServerConsumer.Start()
+	return s.rsConsumer.Start()
 }
 
 // onMessage is called when the sync server receives a new event from the room server output log.
@@ -87,14 +87,14 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 
 	addQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.AddsStateEventIDs}
 	var addQueryRes api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil {
+	if err := s.rsAPI.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil {
 		log.Warn(err)
 		return err
 	}
 
 	remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs}
 	var remQueryRes api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil {
+	if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil {
 		log.Warn(err)
 		return err
 	}

publicroomsapi/directory/directory.go

@@ -59,7 +59,7 @@ func GetVisibility(
 // SetVisibility implements PUT /directory/list/room/{roomID}
 // TODO: Allow admin users to edit the room visibility
 func SetVisibility(
-	req *http.Request, publicRoomsDatabase storage.Database, queryAPI api.RoomserverQueryAPI, dev *authtypes.Device,
+	req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *authtypes.Device,
 	roomID string,
 ) util.JSONResponse {
 	queryMembershipReq := api.QueryMembershipForUserRequest{
@@ -67,7 +67,7 @@ func SetVisibility(
 		UserID: dev.UserID,
 	}
 	var queryMembershipRes api.QueryMembershipForUserResponse
-	err := queryAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes)
+	err := rsAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user")
 		return jsonerror.InternalServerError()
@@ -87,7 +87,7 @@ func SetVisibility(
 		}},
 	}
 	var queryEventsRes api.QueryLatestEventsAndStateResponse
-	err = queryAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
+	err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
 	if err != nil || len(queryEventsRes.StateEvents) == 0 {
 		util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
 		return jsonerror.InternalServerError()

publicroomsapi/publicroomsapi.go

@@ -32,16 +32,16 @@ func SetupPublicRoomsAPIComponent(
 	base *basecomponent.BaseDendrite,
 	deviceDB devices.Database,
 	publicRoomsDB storage.Database,
-	rsQueryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 	fedClient *gomatrixserverlib.FederationClient,
 	extRoomsProvider types.ExternalPublicRoomsProvider,
 ) {
 	rsConsumer := consumers.NewOutputRoomEventConsumer(
-		base.Cfg, base.KafkaConsumer, publicRoomsDB, rsQueryAPI,
+		base.Cfg, base.KafkaConsumer, publicRoomsDB, rsAPI,
 	)
 	if err := rsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start public rooms server consumer")
 	}
 
-	routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsQueryAPI, fedClient, extRoomsProvider)
+	routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider)
 }

publicroomsapi/routing/routing.go

@@ -39,7 +39,7 @@ const pathPrefixR0 = "/_matrix/client/r0"
 // applied:
 // nolint: gocyclo
 func Setup(
-	apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, queryAPI api.RoomserverQueryAPI,
+	apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI,
 	fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider,
 ) {
 	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
@@ -66,7 +66,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return directory.SetVisibility(req, publicRoomsDB, queryAPI, device, vars["roomID"])
+			return directory.SetVisibility(req, publicRoomsDB, rsAPI, device, vars["roomID"])
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/publicRooms",

roomserver/alias/alias_test.go

@@ -1,209 +0,0 @@
-// Copyright 2019 Serra Allgood
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alias
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"testing"
-
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-type MockRoomserverAliasAPIDatabase struct {
-	mode     string
-	attempts int
-}
-
-// These methods can be essentially noop
-func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error {
-	return nil
-}
-
-func (db MockRoomserverAliasAPIDatabase) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) {
-	aliases := make([]string, 0)
-	return aliases, nil
-}
-
-func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, alias string) error {
-	return nil
-}
-
-func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias(
-	ctx context.Context, alias string,
-) (string, error) {
-	return "", nil
-}
-
-func (db *MockRoomserverAliasAPIDatabase) GetRoomVersionForRoom(
-	ctx context.Context, roomID string,
-) (gomatrixserverlib.RoomVersion, error) {
-	return gomatrixserverlib.RoomVersionV1, nil
-}
-
-// This method needs to change depending on test case
-func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias(
-	ctx context.Context,
-	alias string,
-) (string, error) {
-	switch db.mode {
-	case "empty":
-		return "", nil
-	case "error":
-		return "", fmt.Errorf("found an error from GetRoomIDForAlias")
-	case "found":
-		return "123", nil
-	case "emptyFound":
-		switch db.attempts {
-		case 0:
-			db.attempts = 1
-			return "", nil
-		case 1:
-			db.attempts = 0
-			return "123", nil
-		default:
-			return "", nil
-		}
-	default:
-		return "", fmt.Errorf("unknown option used")
-	}
-}
-
-type MockAppServiceQueryAPI struct {
-	mode string
-}
-
-// This method can be noop
-func (q MockAppServiceQueryAPI) UserIDExists(
-	ctx context.Context,
-	req *appserviceAPI.UserIDExistsRequest,
-	resp *appserviceAPI.UserIDExistsResponse,
-) error {
-	return nil
-}
-
-func (q MockAppServiceQueryAPI) RoomAliasExists(
-	ctx context.Context,
-	req *appserviceAPI.RoomAliasExistsRequest,
-	resp *appserviceAPI.RoomAliasExistsResponse,
-) error {
-	switch q.mode {
-	case "error":
-		return fmt.Errorf("found an error from RoomAliasExists")
-	case "found":
-		resp.AliasExists = true
-		return nil
-	case "empty":
-		resp.AliasExists = false
-		return nil
-	default:
-		return fmt.Errorf("Unknown option used")
-	}
-}
-
-func TestGetRoomIDForAlias(t *testing.T) {
-	type arguments struct {
-		ctx      context.Context
-		request  *roomserverAPI.GetRoomIDForAliasRequest
-		response *roomserverAPI.GetRoomIDForAliasResponse
-	}
-	args := arguments{
-		context.Background(),
-		&roomserverAPI.GetRoomIDForAliasRequest{},
-		&roomserverAPI.GetRoomIDForAliasResponse{},
-	}
-	type testCase struct {
-		name      string
-		dbMode    string
-		queryMode string
-		wantError bool
-		errorMsg  string
-		want      string
-	}
-	tt := []testCase{
-		{
-			"found local alias",
-			"found",
-			"error",
-			false,
-			"",
-			"123",
-		},
-		{
-			"found appservice alias",
-			"emptyFound",
-			"found",
-			false,
-			"",
-			"123",
-		},
-		{
-			"error returned from DB",
-			"error",
-			"",
-			true,
-			"GetRoomIDForAlias",
-			"",
-		},
-		{
-			"error returned from appserviceAPI",
-			"empty",
-			"error",
-			true,
-			"RoomAliasExists",
-			"",
-		},
-		{
-			"no errors but no alias",
-			"empty",
-			"empty",
-			false,
-			"",
-			"",
-		},
-	}
-
-	setup := func(dbMode, queryMode string) *RoomserverAliasAPI {
-		mockAliasAPIDB := &MockRoomserverAliasAPIDatabase{dbMode, 0}
-		mockAppServiceQueryAPI := MockAppServiceQueryAPI{queryMode}
-
-		return &RoomserverAliasAPI{
-			DB:            mockAliasAPIDB,
-			AppserviceAPI: mockAppServiceQueryAPI,
-		}
-	}
-
-	for _, tc := range tt {
-		t.Run(tc.name, func(t *testing.T) {
-			aliasAPI := setup(tc.dbMode, tc.queryMode)
-
-			err := aliasAPI.GetRoomIDForAlias(args.ctx, args.request, args.response)
-			if tc.wantError {
-				if err == nil {
-					t.Fatalf("Got no error; wanted error from %s", tc.errorMsg)
-				} else if !strings.Contains(err.Error(), tc.errorMsg) {
-					t.Fatalf("Got %s; wanted error from %s", err, tc.errorMsg)
-				}
-			} else if err != nil {
-				t.Fatalf("Got %s; wanted no error", err)
-			} else if args.response.RoomID != tc.want {
-				t.Errorf("Got '%s'; wanted '%s'", args.response.RoomID, tc.want)
-			}
-		})
-	}
-}

roomserver/api/alias.go

@@ -16,8 +16,6 @@ package api
 
 import (
 	"context"
-	"errors"
-	"net/http"
 
 	commonHTTP "github.com/matrix-org/dendrite/common/http"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -86,44 +84,6 @@ type RemoveRoomAliasRequest struct {
 // RemoveRoomAliasResponse is a response to RemoveRoomAlias
 type RemoveRoomAliasResponse struct{}
 
-// RoomserverAliasAPI is used to save, lookup or remove a room alias
-type RoomserverAliasAPI interface {
-	// Set a room alias
-	SetRoomAlias(
-		ctx context.Context,
-		req *SetRoomAliasRequest,
-		response *SetRoomAliasResponse,
-	) error
-
-	// Get the room ID for an alias
-	GetRoomIDForAlias(
-		ctx context.Context,
-		req *GetRoomIDForAliasRequest,
-		response *GetRoomIDForAliasResponse,
-	) error
-
-	// Get all known aliases for a room ID
-	GetAliasesForRoomID(
-		ctx context.Context,
-		req *GetAliasesForRoomIDRequest,
-		response *GetAliasesForRoomIDResponse,
-	) error
-
-	// Get the user ID of the creator of an alias
-	GetCreatorIDForAlias(
-		ctx context.Context,
-		req *GetCreatorIDForAliasRequest,
-		response *GetCreatorIDForAliasResponse,
-	) error
-
-	// Remove a room alias
-	RemoveRoomAlias(
-		ctx context.Context,
-		req *RemoveRoomAliasRequest,
-		response *RemoveRoomAliasResponse,
-	) error
-}
-
 // RoomserverSetRoomAliasPath is the HTTP path for the SetRoomAlias API.
 const RoomserverSetRoomAliasPath = "/api/roomserver/setRoomAlias"
 
@@ -139,22 +99,8 @@ const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias
 // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API.
 const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias"
 
-// NewRoomserverAliasAPIHTTP creates a RoomserverAliasAPI implemented by talking to a HTTP POST API.
-// If httpClient is nil an error is returned
-func NewRoomserverAliasAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverAliasAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is <nil>")
-	}
-	return &httpRoomserverAliasAPI{roomserverURL, httpClient}, nil
-}
-
-type httpRoomserverAliasAPI struct {
-	roomserverURL string
-	httpClient    *http.Client
-}
-
 // SetRoomAlias implements RoomserverAliasAPI
-func (h *httpRoomserverAliasAPI) SetRoomAlias(
+func (h *httpRoomserverInternalAPI) SetRoomAlias(
 	ctx context.Context,
 	request *SetRoomAliasRequest,
 	response *SetRoomAliasResponse,
@@ -167,7 +113,7 @@ func (h *httpRoomserverAliasAPI) SetRoomAlias(
 }
 
 // GetRoomIDForAlias implements RoomserverAliasAPI
-func (h *httpRoomserverAliasAPI) GetRoomIDForAlias(
+func (h *httpRoomserverInternalAPI) GetRoomIDForAlias(
 	ctx context.Context,
 	request *GetRoomIDForAliasRequest,
 	response *GetRoomIDForAliasResponse,
@@ -180,7 +126,7 @@ func (h *httpRoomserverAliasAPI) GetRoomIDForAlias(
 }
 
 // GetAliasesForRoomID implements RoomserverAliasAPI
-func (h *httpRoomserverAliasAPI) GetAliasesForRoomID(
+func (h *httpRoomserverInternalAPI) GetAliasesForRoomID(
 	ctx context.Context,
 	request *GetAliasesForRoomIDRequest,
 	response *GetAliasesForRoomIDResponse,
@@ -193,7 +139,7 @@ func (h *httpRoomserverAliasAPI) GetAliasesForRoomID(
 }
 
 // GetCreatorIDForAlias implements RoomserverAliasAPI
-func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias(
+func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias(
 	ctx context.Context,
 	request *GetCreatorIDForAliasRequest,
 	response *GetCreatorIDForAliasResponse,
@@ -206,7 +152,7 @@ func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias(
 }
 
 // RemoveRoomAlias implements RoomserverAliasAPI
-func (h *httpRoomserverAliasAPI) RemoveRoomAlias(
+func (h *httpRoomserverInternalAPI) RemoveRoomAlias(
 	ctx context.Context,
 	request *RemoveRoomAliasRequest,
 	response *RemoveRoomAliasResponse,

roomserver/api/api.go

@@ -0,0 +1,141 @@
+package api
+
+import (
+	"context"
+
+	fsAPI "github.com/matrix-org/dendrite/federationsender/api"
+)
+
+// RoomserverInputAPI is used to write events to the room server.
+type RoomserverInternalAPI interface {
+	// needed to avoid chicken and egg scenario when setting up the
+	// interdependencies between the roomserver and other input APIs
+	SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI)
+
+	InputRoomEvents(
+		ctx context.Context,
+		request *InputRoomEventsRequest,
+		response *InputRoomEventsResponse,
+	) error
+
+	// Query the latest events and state for a room from the room server.
+	QueryLatestEventsAndState(
+		ctx context.Context,
+		request *QueryLatestEventsAndStateRequest,
+		response *QueryLatestEventsAndStateResponse,
+	) error
+
+	// Query the state after a list of events in a room from the room server.
+	QueryStateAfterEvents(
+		ctx context.Context,
+		request *QueryStateAfterEventsRequest,
+		response *QueryStateAfterEventsResponse,
+	) error
+
+	// Query a list of events by event ID.
+	QueryEventsByID(
+		ctx context.Context,
+		request *QueryEventsByIDRequest,
+		response *QueryEventsByIDResponse,
+	) error
+
+	// Query the membership event for an user for a room.
+	QueryMembershipForUser(
+		ctx context.Context,
+		request *QueryMembershipForUserRequest,
+		response *QueryMembershipForUserResponse,
+	) error
+
+	// Query a list of membership events for a room
+	QueryMembershipsForRoom(
+		ctx context.Context,
+		request *QueryMembershipsForRoomRequest,
+		response *QueryMembershipsForRoomResponse,
+	) error
+
+	// Query a list of invite event senders for a user in a room.
+	QueryInvitesForUser(
+		ctx context.Context,
+		request *QueryInvitesForUserRequest,
+		response *QueryInvitesForUserResponse,
+	) error
+
+	// Query whether a server is allowed to see an event
+	QueryServerAllowedToSeeEvent(
+		ctx context.Context,
+		request *QueryServerAllowedToSeeEventRequest,
+		response *QueryServerAllowedToSeeEventResponse,
+	) error
+
+	// Query missing events for a room from roomserver
+	QueryMissingEvents(
+		ctx context.Context,
+		request *QueryMissingEventsRequest,
+		response *QueryMissingEventsResponse,
+	) error
+
+	// Query to get state and auth chain for a (potentially hypothetical) event.
+	// Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate
+	// the state and auth chain to return.
+	QueryStateAndAuthChain(
+		ctx context.Context,
+		request *QueryStateAndAuthChainRequest,
+		response *QueryStateAndAuthChainResponse,
+	) error
+
+	// Query a given amount (or less) of events prior to a given set of events.
+	QueryBackfill(
+		ctx context.Context,
+		request *QueryBackfillRequest,
+		response *QueryBackfillResponse,
+	) error
+
+	// Asks for the default room version as preferred by the server.
+	QueryRoomVersionCapabilities(
+		ctx context.Context,
+		request *QueryRoomVersionCapabilitiesRequest,
+		response *QueryRoomVersionCapabilitiesResponse,
+	) error
+
+	// Asks for the room version for a given room.
+	QueryRoomVersionForRoom(
+		ctx context.Context,
+		request *QueryRoomVersionForRoomRequest,
+		response *QueryRoomVersionForRoomResponse,
+	) error
+
+	// Set a room alias
+	SetRoomAlias(
+		ctx context.Context,
+		req *SetRoomAliasRequest,
+		response *SetRoomAliasResponse,
+	) error
+
+	// Get the room ID for an alias
+	GetRoomIDForAlias(
+		ctx context.Context,
+		req *GetRoomIDForAliasRequest,
+		response *GetRoomIDForAliasResponse,
+	) error
+
+	// Get all known aliases for a room ID
+	GetAliasesForRoomID(
+		ctx context.Context,
+		req *GetAliasesForRoomIDRequest,
+		response *GetAliasesForRoomIDResponse,
+	) error
+
+	// Get the user ID of the creator of an alias
+	GetCreatorIDForAlias(
+		ctx context.Context,
+		req *GetCreatorIDForAliasRequest,
+		response *GetCreatorIDForAliasResponse,
+	) error
+
+	// Remove a room alias
+	RemoveRoomAlias(
+		ctx context.Context,
+		req *RemoveRoomAliasRequest,
+		response *RemoveRoomAliasResponse,
+	) error
+}

roomserver/api/http.go

@@ -0,0 +1,41 @@
+package api
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/common/caching"
+	fsInputAPI "github.com/matrix-org/dendrite/federationsender/api"
+)
+
+type httpRoomserverInternalAPI struct {
+	roomserverURL  string
+	httpClient     *http.Client
+	fsAPI          fsInputAPI.FederationSenderInternalAPI
+	immutableCache caching.ImmutableCache
+}
+
+// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API.
+// If httpClient is nil an error is returned
+func NewRoomserverInternalAPIHTTP(
+	roomserverURL string,
+	httpClient *http.Client,
+	//fsInputAPI fsAPI.FederationSenderInternalAPI,
+	immutableCache caching.ImmutableCache,
+) (RoomserverInternalAPI, error) {
+	if httpClient == nil {
+		return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is <nil>")
+	}
+	return &httpRoomserverInternalAPI{
+		roomserverURL:  roomserverURL,
+		httpClient:     httpClient,
+		immutableCache: immutableCache,
+	}, nil
+}
+
+// SetFederationSenderInputAPI passes in a federation sender input API reference
+// so that we can avoid the chicken-and-egg problem of both the roomserver input API
+// and the federation sender input API being interdependent.
+func (h *httpRoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsInputAPI.FederationSenderInternalAPI) {
+	h.fsAPI = fsAPI
+}

roomserver/api/input.go

@@ -17,8 +17,6 @@ package api
 
 import (
 	"context"
-	"errors"
-	"net/http"
 
 	commonHTTP "github.com/matrix-org/dendrite/common/http"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -104,34 +102,11 @@ type InputRoomEventsResponse struct {
 	EventID string `json:"event_id"`
 }
 
-// RoomserverInputAPI is used to write events to the room server.
-type RoomserverInputAPI interface {
-	InputRoomEvents(
-		ctx context.Context,
-		request *InputRoomEventsRequest,
-		response *InputRoomEventsResponse,
-	) error
-}
-
 // RoomserverInputRoomEventsPath is the HTTP path for the InputRoomEvents API.
 const RoomserverInputRoomEventsPath = "/api/roomserver/inputRoomEvents"
 
-// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API.
-// If httpClient is nil an error is returned
-func NewRoomserverInputAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverInputAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewRoomserverInputAPIHTTP: httpClient is <nil>")
-	}
-	return &httpRoomserverInputAPI{roomserverURL, httpClient}, nil
-}
-
-type httpRoomserverInputAPI struct {
-	roomserverURL string
-	httpClient    *http.Client
-}
-
 // InputRoomEvents implements RoomserverInputAPI
-func (h *httpRoomserverInputAPI) InputRoomEvents(
+func (h *httpRoomserverInternalAPI) InputRoomEvents(
 	ctx context.Context,
 	request *InputRoomEventsRequest,
 	response *InputRoomEventsResponse,

roomserver/api/query.go

@@ -18,10 +18,7 @@ package api
 
 import (
 	"context"
-	"errors"
-	"net/http"
 
-	"github.com/matrix-org/dendrite/common/caching"
 	commonHTTP "github.com/matrix-org/dendrite/common/http"
 	"github.com/matrix-org/gomatrixserverlib"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -264,95 +261,6 @@ type QueryRoomVersionForRoomResponse struct {
 	RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
 }
 
-// RoomserverQueryAPI is used to query information from the room server.
-type RoomserverQueryAPI interface {
-	// Query the latest events and state for a room from the room server.
-	QueryLatestEventsAndState(
-		ctx context.Context,
-		request *QueryLatestEventsAndStateRequest,
-		response *QueryLatestEventsAndStateResponse,
-	) error
-
-	// Query the state after a list of events in a room from the room server.
-	QueryStateAfterEvents(
-		ctx context.Context,
-		request *QueryStateAfterEventsRequest,
-		response *QueryStateAfterEventsResponse,
-	) error
-
-	// Query a list of events by event ID.
-	QueryEventsByID(
-		ctx context.Context,
-		request *QueryEventsByIDRequest,
-		response *QueryEventsByIDResponse,
-	) error
-
-	// Query the membership event for an user for a room.
-	QueryMembershipForUser(
-		ctx context.Context,
-		request *QueryMembershipForUserRequest,
-		response *QueryMembershipForUserResponse,
-	) error
-
-	// Query a list of membership events for a room
-	QueryMembershipsForRoom(
-		ctx context.Context,
-		request *QueryMembershipsForRoomRequest,
-		response *QueryMembershipsForRoomResponse,
-	) error
-
-	// Query a list of invite event senders for a user in a room.
-	QueryInvitesForUser(
-		ctx context.Context,
-		request *QueryInvitesForUserRequest,
-		response *QueryInvitesForUserResponse,
-	) error
-
-	// Query whether a server is allowed to see an event
-	QueryServerAllowedToSeeEvent(
-		ctx context.Context,
-		request *QueryServerAllowedToSeeEventRequest,
-		response *QueryServerAllowedToSeeEventResponse,
-	) error
-
-	// Query missing events for a room from roomserver
-	QueryMissingEvents(
-		ctx context.Context,
-		request *QueryMissingEventsRequest,
-		response *QueryMissingEventsResponse,
-	) error
-
-	// Query to get state and auth chain for a (potentially hypothetical) event.
-	// Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate
-	// the state and auth chain to return.
-	QueryStateAndAuthChain(
-		ctx context.Context,
-		request *QueryStateAndAuthChainRequest,
-		response *QueryStateAndAuthChainResponse,
-	) error
-
-	// Query a given amount (or less) of events prior to a given set of events.
-	QueryBackfill(
-		ctx context.Context,
-		request *QueryBackfillRequest,
-		response *QueryBackfillResponse,
-	) error
-
-	// Asks for the default room version as preferred by the server.
-	QueryRoomVersionCapabilities(
-		ctx context.Context,
-		request *QueryRoomVersionCapabilitiesRequest,
-		response *QueryRoomVersionCapabilitiesResponse,
-	) error
-
-	// Asks for the room version for a given room.
-	QueryRoomVersionForRoom(
-		ctx context.Context,
-		request *QueryRoomVersionForRoomRequest,
-		response *QueryRoomVersionForRoomResponse,
-	) error
-}
-
 // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API.
 const RoomserverQueryLatestEventsAndStatePath = "/api/roomserver/queryLatestEventsAndState"
 
@@ -389,23 +297,8 @@ const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVer
 // RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API
 const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom"
 
-// NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API.
-// If httpClient is nil an error is returned
-func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client, cache caching.ImmutableCache) (RoomserverQueryAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewRoomserverQueryAPIHTTP: httpClient is <nil>")
-	}
-	return &httpRoomserverQueryAPI{roomserverURL, httpClient, cache}, nil
-}
-
-type httpRoomserverQueryAPI struct {
-	roomserverURL  string
-	httpClient     *http.Client
-	immutableCache caching.ImmutableCache
-}
-
 // QueryLatestEventsAndState implements RoomserverQueryAPI
-func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState(
+func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState(
 	ctx context.Context,
 	request *QueryLatestEventsAndStateRequest,
 	response *QueryLatestEventsAndStateResponse,
@@ -418,7 +311,7 @@ func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState(
 }
 
 // QueryStateAfterEvents implements RoomserverQueryAPI
-func (h *httpRoomserverQueryAPI) QueryStateAfterEvents(
+func (h *httpRoomserverInternalAPI) QueryStateAfterEvents(
 	ctx context.Context,
 	request *QueryStateAfterEventsRequest,
 	response *QueryStateAfterEventsResponse,
@@ -431,7 +324,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAfterEvents(
 }
 
 // QueryEventsByID implements RoomserverQueryAPI
-func (h *httpRoomserverQueryAPI) QueryEventsByID(
+func (h *httpRoomserverInternalAPI) QueryEventsByID(
 	ctx context.Context,
 	request *QueryEventsByIDRequest,
 	response *QueryEventsByIDResponse,
@@ -444,7 +337,7 @@ func (h *httpRoomserverQueryAPI) QueryEventsByID(
 }
 
 // QueryMembershipForUser implements RoomserverQueryAPI
-func (h *httpRoomserverQueryAPI) QueryMembershipForUser(
+func (h *httpRoomserverInternalAPI) QueryMembershipForUser(
 	ctx context.Context,
 	request *QueryMembershipForUserRequest,
 	response *QueryMembershipForUserResponse,
@@ -457,7 +350,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipForUser(
 }
 
 // QueryMembershipsForRoom implements RoomserverQueryAPI
-func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom(
+func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom(
 	ctx context.Context,
 	request *QueryMembershipsForRoomRequest,
 	response *QueryMembershipsForRoomResponse,
@@ -470,7 +363,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom(
 }
 
 // QueryInvitesForUser implements RoomserverQueryAPI
-func (h *httpRoomserverQueryAPI) QueryInvitesForUser(
+func (h *httpRoomserverInternalAPI) QueryInvitesForUser(
 	ctx context.Context,
 	request *QueryInvitesForUserRequest,
 	response *QueryInvitesForUserResponse,
@@ -483,7 +376,7 @@ func (h *httpRoomserverQueryAPI) QueryInvitesForUser(
 }
 
 // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI
-func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent(
+func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent(
 	ctx context.Context,
 	request *QueryServerAllowedToSeeEventRequest,
 	response *QueryServerAllowedToSeeEventResponse,
@@ -496,7 +389,7 @@ func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent(
 }
 
 // QueryMissingEvents implements RoomServerQueryAPI
-func (h *httpRoomserverQueryAPI) QueryMissingEvents(
+func (h *httpRoomserverInternalAPI) QueryMissingEvents(
 	ctx context.Context,
 	request *QueryMissingEventsRequest,
 	response *QueryMissingEventsResponse,
@@ -509,7 +402,7 @@ func (h *httpRoomserverQueryAPI) QueryMissingEvents(
 }
 
 // QueryStateAndAuthChain implements RoomserverQueryAPI
-func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain(
+func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain(
 	ctx context.Context,
 	request *QueryStateAndAuthChainRequest,
 	response *QueryStateAndAuthChainResponse,
@@ -522,7 +415,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain(
 }
 
 // QueryBackfill implements RoomServerQueryAPI
-func (h *httpRoomserverQueryAPI) QueryBackfill(
+func (h *httpRoomserverInternalAPI) QueryBackfill(
 	ctx context.Context,
 	request *QueryBackfillRequest,
 	response *QueryBackfillResponse,
@@ -535,7 +428,7 @@ func (h *httpRoomserverQueryAPI) QueryBackfill(
 }
 
 // QueryRoomVersionCapabilities implements RoomServerQueryAPI
-func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities(
+func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities(
 	ctx context.Context,
 	request *QueryRoomVersionCapabilitiesRequest,
 	response *QueryRoomVersionCapabilitiesResponse,
@@ -548,7 +441,7 @@ func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities(
 }
 
 // QueryRoomVersionForRoom implements RoomServerQueryAPI
-func (h *httpRoomserverQueryAPI) QueryRoomVersionForRoom(
+func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom(
 	ctx context.Context,
 	request *QueryRoomVersionForRoomRequest,
 	response *QueryRoomVersionForRoomResponse,

roomserver/auth/auth.go

@@ -27,7 +27,7 @@ func IsServerAllowed(
 	serverCurrentlyInRoom bool,
 	authEvents []gomatrixserverlib.Event,
 ) bool {
-	historyVisibility := historyVisibilityForRoom(authEvents)
+	historyVisibility := HistoryVisibilityForRoom(authEvents)
 
 	// 1. If the history_visibility was set to world_readable, allow.
 	if historyVisibility == "world_readable" {
@@ -52,7 +52,7 @@ func IsServerAllowed(
 	return false
 }
 
-func historyVisibilityForRoom(authEvents []gomatrixserverlib.Event) string {
+func HistoryVisibilityForRoom(authEvents []gomatrixserverlib.Event) string {
 	// https://matrix.org/docs/spec/client_server/r0.6.0#id87
 	// By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared.
 	visibility := "shared"

roomserver/internal/alias.go

@@ -12,25 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package alias
+package internal
 
 import (
 	"context"
 	"encoding/json"
 	"errors"
-	"net/http"
 	"time"
 
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/common/config"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
-// RoomserverAliasAPIDatabase has the storage APIs needed to implement the alias API.
+// RoomserverInternalAPIDatabase has the storage APIs needed to implement the alias API.
-type RoomserverAliasAPIDatabase interface {
+type RoomserverInternalAPIDatabase interface {
 	// Save a given room alias with the room ID it refers to.
 	// Returns an error if there was a problem talking to the database.
 	SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error
@@ -52,20 +47,11 @@ type RoomserverAliasAPIDatabase interface {
 	) (gomatrixserverlib.RoomVersion, error)
 }
 
-// RoomserverAliasAPI is an implementation of alias.RoomserverAliasAPI
+// SetRoomAlias implements alias.RoomserverInternalAPI
-type RoomserverAliasAPI struct {
+func (r *RoomserverInternalAPI) SetRoomAlias(
-	DB            RoomserverAliasAPIDatabase
-	Cfg           *config.Dendrite
-	InputAPI      roomserverAPI.RoomserverInputAPI
-	QueryAPI      roomserverAPI.RoomserverQueryAPI
-	AppserviceAPI appserviceAPI.AppServiceQueryAPI
-}
-
-// SetRoomAlias implements alias.RoomserverAliasAPI
-func (r *RoomserverAliasAPI) SetRoomAlias(
 	ctx context.Context,
-	request *roomserverAPI.SetRoomAliasRequest,
+	request *api.SetRoomAliasRequest,
-	response *roomserverAPI.SetRoomAliasResponse,
+	response *api.SetRoomAliasResponse,
 ) error {
 	// Check if the alias isn't already referring to a room
 	roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias)
@@ -91,11 +77,11 @@ func (r *RoomserverAliasAPI) SetRoomAlias(
 	return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, request.RoomID)
 }
 
-// GetRoomIDForAlias implements alias.RoomserverAliasAPI
+// GetRoomIDForAlias implements alias.RoomserverInternalAPI
-func (r *RoomserverAliasAPI) GetRoomIDForAlias(
+func (r *RoomserverInternalAPI) GetRoomIDForAlias(
 	ctx context.Context,
-	request *roomserverAPI.GetRoomIDForAliasRequest,
+	request *api.GetRoomIDForAliasRequest,
-	response *roomserverAPI.GetRoomIDForAliasResponse,
+	response *api.GetRoomIDForAliasResponse,
 ) error {
 	// Look up the room ID in the database
 	roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias)
@@ -103,32 +89,38 @@ func (r *RoomserverAliasAPI) GetRoomIDForAlias(
 		return err
 	}
 
-	if roomID == "" {
+	/*
-		// No room found locally, try our application services by making a call to
+		TODO: Why is this here? It creates an unnecessary dependency
-		// the appservice component
+		from the roomserver to the appservice component, which should be
-		aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias}
+		altogether optional.
-		var aliasResp appserviceAPI.RoomAliasExistsResponse
-		if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil {
-			return err
-		}
 
-		if aliasResp.AliasExists {
+		if roomID == "" {
-			roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias)
+			// No room found locally, try our application services by making a call to
-			if err != nil {
+			// the appservice component
+			aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias}
+			var aliasResp appserviceAPI.RoomAliasExistsResponse
+			if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil {
 				return err
 			}
+
+			if aliasResp.AliasExists {
+				roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias)
+				if err != nil {
+					return err
+				}
+			}
 		}
-	}
+	*/
 
 	response.RoomID = roomID
 	return nil
 }
 
-// GetAliasesForRoomID implements alias.RoomserverAliasAPI
+// GetAliasesForRoomID implements alias.RoomserverInternalAPI
-func (r *RoomserverAliasAPI) GetAliasesForRoomID(
+func (r *RoomserverInternalAPI) GetAliasesForRoomID(
 	ctx context.Context,
-	request *roomserverAPI.GetAliasesForRoomIDRequest,
+	request *api.GetAliasesForRoomIDRequest,
-	response *roomserverAPI.GetAliasesForRoomIDResponse,
+	response *api.GetAliasesForRoomIDResponse,
 ) error {
 	// Look up the aliases in the database for the given RoomID
 	aliases, err := r.DB.GetAliasesForRoomID(ctx, request.RoomID)
@@ -140,11 +132,11 @@ func (r *RoomserverAliasAPI) GetAliasesForRoomID(
 	return nil
 }
 
-// GetCreatorIDForAlias implements alias.RoomserverAliasAPI
+// GetCreatorIDForAlias implements alias.RoomserverInternalAPI
-func (r *RoomserverAliasAPI) GetCreatorIDForAlias(
+func (r *RoomserverInternalAPI) GetCreatorIDForAlias(
 	ctx context.Context,
-	request *roomserverAPI.GetCreatorIDForAliasRequest,
+	request *api.GetCreatorIDForAliasRequest,
-	response *roomserverAPI.GetCreatorIDForAliasResponse,
+	response *api.GetCreatorIDForAliasResponse,
 ) error {
 	// Look up the aliases in the database for the given RoomID
 	creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias)
@@ -156,11 +148,11 @@ func (r *RoomserverAliasAPI) GetCreatorIDForAlias(
 	return nil
 }
 
-// RemoveRoomAlias implements alias.RoomserverAliasAPI
+// RemoveRoomAlias implements alias.RoomserverInternalAPI
-func (r *RoomserverAliasAPI) RemoveRoomAlias(
+func (r *RoomserverInternalAPI) RemoveRoomAlias(
 	ctx context.Context,
-	request *roomserverAPI.RemoveRoomAliasRequest,
+	request *api.RemoveRoomAliasRequest,
-	response *roomserverAPI.RemoveRoomAliasResponse,
+	response *api.RemoveRoomAliasResponse,
 ) error {
 	// Look up the room ID in the database
 	roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias)
@@ -186,7 +178,7 @@ type roomAliasesContent struct {
 
 // Build the updated m.room.aliases event to send to the room after addition or
 // removal of an alias
-func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent(
+func (r *RoomserverInternalAPI) sendUpdatedAliasesEvent(
 	ctx context.Context, userID string, roomID string,
 ) error {
 	serverName := string(r.Cfg.Matrix.ServerName)
@@ -222,12 +214,12 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent(
 	if len(eventsNeeded.Tuples()) == 0 {
 		return errors.New("expecting state tuples for event builder, got none")
 	}
-	req := roomserverAPI.QueryLatestEventsAndStateRequest{
+	req := api.QueryLatestEventsAndStateRequest{
 		RoomID:       roomID,
 		StateToFetch: eventsNeeded.Tuples(),
 	}
-	var res roomserverAPI.QueryLatestEventsAndStateResponse
+	var res api.QueryLatestEventsAndStateResponse
-	if err = r.QueryAPI.QueryLatestEventsAndState(ctx, &req, &res); err != nil {
+	if err = r.QueryLatestEventsAndState(ctx, &req, &res); err != nil {
 		return err
 	}
 	builder.Depth = res.Depth
@@ -263,91 +255,17 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent(
 	}
 
 	// Create the request
-	ire := roomserverAPI.InputRoomEvent{
+	ire := api.InputRoomEvent{
-		Kind:         roomserverAPI.KindNew,
+		Kind:         api.KindNew,
 		Event:        event.Headered(roomVersion),
 		AuthEventIDs: event.AuthEventIDs(),
 		SendAsServer: serverName,
 	}
-	inputReq := roomserverAPI.InputRoomEventsRequest{
+	inputReq := api.InputRoomEventsRequest{
-		InputRoomEvents: []roomserverAPI.InputRoomEvent{ire},
+		InputRoomEvents: []api.InputRoomEvent{ire},
 	}
-	var inputRes roomserverAPI.InputRoomEventsResponse
+	var inputRes api.InputRoomEventsResponse
 
 	// Send the request
-	return r.InputAPI.InputRoomEvents(ctx, &inputReq, &inputRes)
+	return r.InputRoomEvents(ctx, &inputReq, &inputRes)
-}
-
-// SetupHTTP adds the RoomserverAliasAPI handlers to the http.ServeMux.
-func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) {
-	servMux.Handle(
-		roomserverAPI.RoomserverSetRoomAliasPath,
-		common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse {
-			var request roomserverAPI.SetRoomAliasRequest
-			var response roomserverAPI.SetRoomAliasResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		roomserverAPI.RoomserverGetRoomIDForAliasPath,
-		common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse {
-			var request roomserverAPI.GetRoomIDForAliasRequest
-			var response roomserverAPI.GetRoomIDForAliasResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		roomserverAPI.RoomserverGetCreatorIDForAliasPath,
-		common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse {
-			var request roomserverAPI.GetCreatorIDForAliasRequest
-			var response roomserverAPI.GetCreatorIDForAliasResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		roomserverAPI.RoomserverGetAliasesForRoomIDPath,
-		common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse {
-			var request roomserverAPI.GetAliasesForRoomIDRequest
-			var response roomserverAPI.GetAliasesForRoomIDResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		roomserverAPI.RoomserverRemoveRoomAliasPath,
-		common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse {
-			var request roomserverAPI.RemoveRoomAliasRequest
-			var response roomserverAPI.RemoveRoomAliasResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
 }

roomserver/internal/api.go

@@ -0,0 +1,287 @@
+package internal
+
+import (
+	"encoding/json"
+	"net/http"
+	"sync"
+
+	"github.com/Shopify/sarama"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/caching"
+	"github.com/matrix-org/dendrite/common/config"
+	fsAPI "github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// RoomserverInternalAPI is an implementation of api.RoomserverInternalAPI
+type RoomserverInternalAPI struct {
+	DB                   storage.Database
+	Cfg                  *config.Dendrite
+	Producer             sarama.SyncProducer
+	ImmutableCache       caching.ImmutableCache
+	ServerName           gomatrixserverlib.ServerName
+	KeyRing              gomatrixserverlib.JSONVerifier
+	FedClient            *gomatrixserverlib.FederationClient
+	OutputRoomEventTopic string     // Kafka topic for new output room events
+	mutex                sync.Mutex // Protects calls to processRoomEvent
+	fsAPI                fsAPI.FederationSenderInternalAPI
+}
+
+// SetupHTTP adds the RoomserverInternalAPI handlers to the http.ServeMux.
+// nolint: gocyclo
+func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
+	servMux.Handle(api.RoomserverInputRoomEventsPath,
+		common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse {
+			var request api.InputRoomEventsRequest
+			var response api.InputRoomEventsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryLatestEventsAndStatePath,
+		common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse {
+			var request api.QueryLatestEventsAndStateRequest
+			var response api.QueryLatestEventsAndStateResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryStateAfterEventsPath,
+		common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse {
+			var request api.QueryStateAfterEventsRequest
+			var response api.QueryStateAfterEventsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryEventsByIDPath,
+		common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse {
+			var request api.QueryEventsByIDRequest
+			var response api.QueryEventsByIDResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryMembershipForUserPath,
+		common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse {
+			var request api.QueryMembershipForUserRequest
+			var response api.QueryMembershipForUserResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryMembershipsForRoomPath,
+		common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse {
+			var request api.QueryMembershipsForRoomRequest
+			var response api.QueryMembershipsForRoomResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryInvitesForUserPath,
+		common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse {
+			var request api.QueryInvitesForUserRequest
+			var response api.QueryInvitesForUserResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryServerAllowedToSeeEventPath,
+		common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse {
+			var request api.QueryServerAllowedToSeeEventRequest
+			var response api.QueryServerAllowedToSeeEventResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryMissingEventsPath,
+		common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse {
+			var request api.QueryMissingEventsRequest
+			var response api.QueryMissingEventsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryStateAndAuthChainPath,
+		common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse {
+			var request api.QueryStateAndAuthChainRequest
+			var response api.QueryStateAndAuthChainResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryBackfillPath,
+		common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse {
+			var request api.QueryBackfillRequest
+			var response api.QueryBackfillResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryBackfill(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryRoomVersionCapabilitiesPath,
+		common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse {
+			var request api.QueryRoomVersionCapabilitiesRequest
+			var response api.QueryRoomVersionCapabilitiesResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverQueryRoomVersionForRoomPath,
+		common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse {
+			var request api.QueryRoomVersionForRoomRequest
+			var response api.QueryRoomVersionForRoomResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverSetRoomAliasPath,
+		common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse {
+			var request api.SetRoomAliasRequest
+			var response api.SetRoomAliasResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverGetRoomIDForAliasPath,
+		common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse {
+			var request api.GetRoomIDForAliasRequest
+			var response api.GetRoomIDForAliasResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverGetCreatorIDForAliasPath,
+		common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse {
+			var request api.GetCreatorIDForAliasRequest
+			var response api.GetCreatorIDForAliasResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverGetAliasesForRoomIDPath,
+		common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse {
+			var request api.GetAliasesForRoomIDRequest
+			var response api.GetAliasesForRoomIDResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.RoomserverRemoveRoomAliasPath,
+		common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse {
+			var request api.RemoveRoomAliasRequest
+			var response api.RemoveRoomAliasResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+}

roomserver/internal/input.go

@@ -13,34 +13,27 @@
 // limitations under the License.
 
 // Package input contains the code processes new room events
-package input
+package internal
 
 import (
 	"context"
 	"encoding/json"
-	"net/http"
-	"sync"
 
 	"github.com/Shopify/sarama"
-	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/storage"
+
-	"github.com/matrix-org/util"
+	fsAPI "github.com/matrix-org/dendrite/federationsender/api"
 )
 
-// RoomserverInputAPI implements api.RoomserverInputAPI
+// SetFederationSenderInputAPI passes in a federation sender input API reference
-type RoomserverInputAPI struct {
+// so that we can avoid the chicken-and-egg problem of both the roomserver input API
-	DB       storage.Database
+// and the federation sender input API being interdependent.
-	Producer sarama.SyncProducer
+func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) {
-	// The kafkaesque topic to output new room events to.
+	r.fsAPI = fsAPI
-	// This is the name used in kafka to identify the stream to write events to.
-	OutputRoomEventTopic string
-	// Protects calls to processRoomEvent
-	mutex sync.Mutex
 }
 
 // WriteOutputEvents implements OutputRoomEventWriter
-func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error {
+func (r *RoomserverInternalAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error {
 	messages := make([]*sarama.ProducerMessage, len(updates))
 	for i := range updates {
 		value, err := json.Marshal(updates[i])
@@ -56,8 +49,8 @@ func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.Outp
 	return r.Producer.SendMessages(messages)
 }
 
-// InputRoomEvents implements api.RoomserverInputAPI
+// InputRoomEvents implements api.RoomserverInternalAPI
-func (r *RoomserverInputAPI) InputRoomEvents(
+func (r *RoomserverInternalAPI) InputRoomEvents(
 	ctx context.Context,
 	request *api.InputRoomEventsRequest,
 	response *api.InputRoomEventsResponse,
@@ -77,20 +70,3 @@ func (r *RoomserverInputAPI) InputRoomEvents(
 	}
 	return nil
 }
-
-// SetupHTTP adds the RoomserverInputAPI handlers to the http.ServeMux.
-func (r *RoomserverInputAPI) SetupHTTP(servMux *http.ServeMux) {
-	servMux.Handle(api.RoomserverInputRoomEventsPath,
-		common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse {
-			var request api.InputRoomEventsRequest
-			var response api.InputRoomEventsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-}

roomserver/internal/input_authevents.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package input
+package internal
 
 import (
 	"context"

roomserver/internal/input_authevents_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package input
+package internal
 
 import (
 	"testing"

roomserver/internal/input_events.go

@@ -14,7 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package input
+package internal
 
 import (
 	"context"

roomserver/internal/input_latest_events.go

@@ -14,7 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package input
+package internal
 
 import (
 	"bytes"

roomserver/internal/input_membership.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package input
+package internal
 
 import (
 	"context"

roomserver/internal/query.go

@@ -14,16 +14,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package query
+package internal
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
-	"net/http"
 
-	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/common/caching"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/auth"
 	"github.com/matrix-org/dendrite/roomserver/state"
@@ -35,17 +31,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI
+// QueryLatestEventsAndState implements api.RoomserverInternalAPI
-type RoomserverQueryAPI struct {
+func (r *RoomserverInternalAPI) QueryLatestEventsAndState(
-	DB             storage.Database
-	ImmutableCache caching.ImmutableCache
-	ServerName     gomatrixserverlib.ServerName
-	KeyRing        gomatrixserverlib.JSONVerifier
-	FedClient      *gomatrixserverlib.FederationClient
-}
-
-// QueryLatestEventsAndState implements api.RoomserverQueryAPI
-func (r *RoomserverQueryAPI) QueryLatestEventsAndState(
 	ctx context.Context,
 	request *api.QueryLatestEventsAndStateRequest,
 	response *api.QueryLatestEventsAndStateResponse,
@@ -104,8 +91,8 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState(
 	return nil
 }
 
-// QueryStateAfterEvents implements api.RoomserverQueryAPI
+// QueryStateAfterEvents implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryStateAfterEvents(
+func (r *RoomserverInternalAPI) QueryStateAfterEvents(
 	ctx context.Context,
 	request *api.QueryStateAfterEventsRequest,
 	response *api.QueryStateAfterEventsResponse,
@@ -160,8 +147,8 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents(
 	return nil
 }
 
-// QueryEventsByID implements api.RoomserverQueryAPI
+// QueryEventsByID implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryEventsByID(
+func (r *RoomserverInternalAPI) QueryEventsByID(
 	ctx context.Context,
 	request *api.QueryEventsByIDRequest,
 	response *api.QueryEventsByIDResponse,
@@ -195,7 +182,7 @@ func (r *RoomserverQueryAPI) QueryEventsByID(
 	return nil
 }
 
-func (r *RoomserverQueryAPI) loadStateEvents(
+func (r *RoomserverInternalAPI) loadStateEvents(
 	ctx context.Context, stateEntries []types.StateEntry,
 ) ([]gomatrixserverlib.Event, error) {
 	eventNIDs := make([]types.EventNID, len(stateEntries))
@@ -205,7 +192,7 @@ func (r *RoomserverQueryAPI) loadStateEvents(
 	return r.loadEvents(ctx, eventNIDs)
 }
 
-func (r *RoomserverQueryAPI) loadEvents(
+func (r *RoomserverInternalAPI) loadEvents(
 	ctx context.Context, eventNIDs []types.EventNID,
 ) ([]gomatrixserverlib.Event, error) {
 	stateEvents, err := r.DB.Events(ctx, eventNIDs)
@@ -220,8 +207,8 @@ func (r *RoomserverQueryAPI) loadEvents(
 	return result, nil
 }
 
-// QueryMembershipForUser implements api.RoomserverQueryAPI
+// QueryMembershipForUser implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryMembershipForUser(
+func (r *RoomserverInternalAPI) QueryMembershipForUser(
 	ctx context.Context,
 	request *api.QueryMembershipForUserRequest,
 	response *api.QueryMembershipForUserResponse,
@@ -251,8 +238,8 @@ func (r *RoomserverQueryAPI) QueryMembershipForUser(
 	return nil
 }
 
-// QueryMembershipsForRoom implements api.RoomserverQueryAPI
+// QueryMembershipsForRoom implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryMembershipsForRoom(
+func (r *RoomserverInternalAPI) QueryMembershipsForRoom(
 	ctx context.Context,
 	request *api.QueryMembershipsForRoomRequest,
 	response *api.QueryMembershipsForRoomResponse,
@@ -277,6 +264,7 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom(
 	response.JoinEvents = []gomatrixserverlib.ClientEvent{}
 
 	var events []types.Event
+	var stateEntries []types.StateEntry
 	if stillInRoom {
 		var eventNIDs []types.EventNID
 		eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, request.JoinedOnly)
@@ -286,7 +274,12 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom(
 
 		events, err = r.DB.Events(ctx, eventNIDs)
 	} else {
-		events, err = getMembershipsBeforeEventNID(ctx, r.DB, membershipEventNID, request.JoinedOnly)
+		stateEntries, err = stateBeforeEvent(ctx, r.DB, membershipEventNID)
+		if err != nil {
+			logrus.WithField("membership_event_nid", membershipEventNID).WithError(err).Error("failed to load state before event")
+			return err
+		}
+		events, err = getMembershipsAtState(ctx, r.DB, stateEntries, request.JoinedOnly)
 	}
 
 	if err != nil {
@@ -301,15 +294,8 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom(
 	return nil
 }
 
-// getMembershipsBeforeEventNID takes the numeric ID of an event and fetches the state
+func stateBeforeEvent(ctx context.Context, db storage.Database, eventNID types.EventNID) ([]types.StateEntry, error) {
-// of the event's room as it was when this event was fired, then filters the state events to
-// only keep the "m.room.member" events with a "join" membership. These events are returned.
-// Returns an error if there was an issue fetching the events.
-func getMembershipsBeforeEventNID(
-	ctx context.Context, db storage.Database, eventNID types.EventNID, joinedOnly bool,
-) ([]types.Event, error) {
 	roomState := state.NewStateResolution(db)
-	events := []types.Event{}
 	// Lookup the event NID
 	eIDs, err := db.EventIDs(ctx, []types.EventNID{eventNID})
 	if err != nil {
@@ -323,10 +309,15 @@ func getMembershipsBeforeEventNID(
 	}
 
 	// Fetch the state as it was when this event was fired
-	stateEntries, err := roomState.LoadCombinedStateAfterEvents(ctx, prevState)
+	return roomState.LoadCombinedStateAfterEvents(ctx, prevState)
-	if err != nil {
+}
-		return nil, err
+
-	}
+// getMembershipsAtState filters the state events to
+// only keep the "m.room.member" events with a "join" membership. These events are returned.
+// Returns an error if there was an issue fetching the events.
+func getMembershipsAtState(
+	ctx context.Context, db storage.Database, stateEntries []types.StateEntry, joinedOnly bool,
+) ([]types.Event, error) {
 
 	var eventNIDs []types.EventNID
 	for _, entry := range stateEntries {
@@ -347,6 +338,7 @@ func getMembershipsBeforeEventNID(
 	}
 
 	// Filter the events to only keep the "join" membership events
+	var events []types.Event
 	for _, event := range stateEvents {
 		membership, err := event.Membership()
 		if err != nil {
@@ -361,8 +353,8 @@ func getMembershipsBeforeEventNID(
 	return events, nil
 }
 
-// QueryInvitesForUser implements api.RoomserverQueryAPI
+// QueryInvitesForUser implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryInvitesForUser(
+func (r *RoomserverInternalAPI) QueryInvitesForUser(
 	ctx context.Context,
 	request *api.QueryInvitesForUserRequest,
 	response *api.QueryInvitesForUserResponse,
@@ -395,8 +387,8 @@ func (r *RoomserverQueryAPI) QueryInvitesForUser(
 	return nil
 }
 
-// QueryServerAllowedToSeeEvent implements api.RoomserverQueryAPI
+// QueryServerAllowedToSeeEvent implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent(
+func (r *RoomserverInternalAPI) QueryServerAllowedToSeeEvent(
 	ctx context.Context,
 	request *api.QueryServerAllowedToSeeEventRequest,
 	response *api.QueryServerAllowedToSeeEventResponse,
@@ -419,7 +411,7 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent(
 	return
 }
 
-func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent(
+func (r *RoomserverInternalAPI) checkServerAllowedToSeeEvent(
 	ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, isServerInRoom bool,
 ) (bool, error) {
 	roomState := state.NewStateResolution(r.DB)
@@ -438,8 +430,8 @@ func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent(
 	return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil
 }
 
-// QueryMissingEvents implements api.RoomserverQueryAPI
+// QueryMissingEvents implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryMissingEvents(
+func (r *RoomserverInternalAPI) QueryMissingEvents(
 	ctx context.Context,
 	request *api.QueryMissingEventsRequest,
 	response *api.QueryMissingEventsResponse,
@@ -484,7 +476,7 @@ func (r *RoomserverQueryAPI) QueryMissingEvents(
 }
 
 // QueryBackfill implements api.RoomServerQueryAPI
-func (r *RoomserverQueryAPI) QueryBackfill(
+func (r *RoomserverInternalAPI) QueryBackfill(
 	ctx context.Context,
 	request *api.QueryBackfillRequest,
 	response *api.QueryBackfillResponse,
@@ -537,7 +529,7 @@ func (r *RoomserverQueryAPI) QueryBackfill(
 	return err
 }
 
-func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error {
+func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error {
 	roomVer, err := r.DB.GetRoomVersionForRoom(ctx, req.RoomID)
 	if err != nil {
 		return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err)
@@ -563,20 +555,29 @@ func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api
 		if !ok {
 			// this should be impossible as all events returned must have pass Step 5 of the PDU checks
 			// which requires a list of state IDs.
-			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to find state IDs for event which passed auth checks")
+			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to find state IDs for event which passed auth checks")
 			continue
 		}
 		var entries []types.StateEntry
 		if entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs); err != nil {
-			return err
+			// attempt to fetch the missing events
+			r.fetchAndStoreMissingEvents(ctx, roomVer, requester, stateIDs)
+			// try again
+			entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs)
+			if err != nil {
+				logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to get state entries for event")
+				return err
+			}
 		}
 
 		var beforeStateSnapshotNID types.StateSnapshotNID
 		if beforeStateSnapshotNID, err = r.DB.AddState(ctx, roomNID, nil, entries); err != nil {
+			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist state entries to get snapshot nid")
 			return err
 		}
+		util.GetLogger(ctx).Infof("Backfilled event %s (nid=%d) getting snapshot %v with entries %+v", ev.EventID(), ev.EventNID, beforeStateSnapshotNID, entries)
 		if err = r.DB.SetState(ctx, ev.EventNID, beforeStateSnapshotNID); err != nil {
-			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to set state before event")
+			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist snapshot nid")
 		}
 	}
 
@@ -586,7 +587,7 @@ func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api
 	return nil
 }
 
-func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) {
+func (r *RoomserverInternalAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) {
 	roomNID, err := r.DB.RoomNID(ctx, roomID)
 	if err != nil {
 		return false, err
@@ -608,9 +609,69 @@ func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, server
 	return auth.IsAnyUserOnServerWithMembership(serverName, gmslEvents, gomatrixserverlib.Join), nil
 }
 
+// fetchAndStoreMissingEvents does a best-effort fetch and store of missing events specified in stateIDs. Returns no error as it is just
+// best effort.
+func (r *RoomserverInternalAPI) fetchAndStoreMissingEvents(ctx context.Context, roomVer gomatrixserverlib.RoomVersion,
+	backfillRequester *backfillRequester, stateIDs []string) {
+
+	servers := backfillRequester.servers
+
+	// work out which are missing
+	nidMap, err := r.DB.EventNIDs(ctx, stateIDs)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Warn("cannot query missing events")
+		return
+	}
+	missingMap := make(map[string]*gomatrixserverlib.HeaderedEvent) // id -> event
+	for _, id := range stateIDs {
+		if _, ok := nidMap[id]; !ok {
+			missingMap[id] = nil
+		}
+	}
+	util.GetLogger(ctx).Infof("Fetching %d missing state events (from %d possible servers)", len(missingMap), len(servers))
+
+	// fetch the events from federation. Loop the servers first so if we find one that works we stick with them
+	for _, srv := range servers {
+		for id, ev := range missingMap {
+			if ev != nil {
+				continue // already found
+			}
+			logger := util.GetLogger(ctx).WithField("server", srv).WithField("event_id", id)
+			res, err := r.FedClient.GetEvent(ctx, srv, id)
+			if err != nil {
+				logger.WithError(err).Warn("failed to get event from server")
+				continue
+			}
+			loader := gomatrixserverlib.NewEventsLoader(roomVer, r.KeyRing, backfillRequester, backfillRequester.ProvideEvents, false)
+			result, err := loader.LoadAndVerify(ctx, res.PDUs, gomatrixserverlib.TopologicalOrderByPrevEvents)
+			if err != nil {
+				logger.WithError(err).Warn("failed to load and verify event")
+				continue
+			}
+			logger.Infof("returned %d PDUs which made events %+v", len(res.PDUs), result)
+			for _, res := range result {
+				if res.Error != nil {
+					logger.WithError(err).Warn("event failed PDU checks")
+					continue
+				}
+				missingMap[id] = res.Event
+			}
+		}
+	}
+
+	var newEvents []gomatrixserverlib.HeaderedEvent
+	for _, ev := range missingMap {
+		if ev != nil {
+			newEvents = append(newEvents, *ev)
+		}
+	}
+	util.GetLogger(ctx).Infof("Persisting %d new events", len(newEvents))
+	persistEvents(ctx, r.DB, newEvents)
+}
+
 // TODO: Remove this when we have tests to assert correctness of this function
 // nolint:gocyclo
-func (r *RoomserverQueryAPI) scanEventTree(
+func (r *RoomserverInternalAPI) scanEventTree(
 	ctx context.Context, front []string, visited map[string]bool, limit int,
 	serverName gomatrixserverlib.ServerName,
 ) ([]types.EventNID, error) {
@@ -703,8 +764,8 @@ BFSLoop:
 	return resultNIDs, err
 }
 
-// QueryStateAndAuthChain implements api.RoomserverQueryAPI
+// QueryStateAndAuthChain implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryStateAndAuthChain(
+func (r *RoomserverInternalAPI) QueryStateAndAuthChain(
 	ctx context.Context,
 	request *api.QueryStateAndAuthChainRequest,
 	response *api.QueryStateAndAuthChainResponse,
@@ -763,7 +824,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain(
 	return err
 }
 
-func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) {
+func (r *RoomserverInternalAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) {
 	roomState := state.NewStateResolution(r.DB)
 	prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs)
 	if err != nil {
@@ -857,7 +918,7 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse
 		var stateAtEvent types.StateAtEvent
 		roomNID, stateAtEvent, err = db.StoreEvent(ctx, ev.Unwrap(), nil, authNids)
 		if err != nil {
-			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to store backfilled event")
+			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to persist event")
 			continue
 		}
 		backfilledEventMap[ev.EventID()] = types.Event{
@@ -868,8 +929,8 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse
 	return roomNID, backfilledEventMap
 }
 
-// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI
+// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities(
+func (r *RoomserverInternalAPI) QueryRoomVersionCapabilities(
 	ctx context.Context,
 	request *api.QueryRoomVersionCapabilitiesRequest,
 	response *api.QueryRoomVersionCapabilitiesResponse,
@@ -886,8 +947,8 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities(
 	return nil
 }
 
-// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI
+// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI
-func (r *RoomserverQueryAPI) QueryRoomVersionForRoom(
+func (r *RoomserverInternalAPI) QueryRoomVersionForRoom(
 	ctx context.Context,
 	request *api.QueryRoomVersionForRoomRequest,
 	response *api.QueryRoomVersionForRoomResponse,
@@ -905,176 +966,3 @@ func (r *RoomserverQueryAPI) QueryRoomVersionForRoom(
 	r.ImmutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion)
 	return nil
 }
-
-// SetupHTTP adds the RoomserverQueryAPI handlers to the http.ServeMux.
-// nolint: gocyclo
-func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) {
-	servMux.Handle(
-		api.RoomserverQueryLatestEventsAndStatePath,
-		common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse {
-			var request api.QueryLatestEventsAndStateRequest
-			var response api.QueryLatestEventsAndStateResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryStateAfterEventsPath,
-		common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse {
-			var request api.QueryStateAfterEventsRequest
-			var response api.QueryStateAfterEventsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryEventsByIDPath,
-		common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse {
-			var request api.QueryEventsByIDRequest
-			var response api.QueryEventsByIDResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryMembershipForUserPath,
-		common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse {
-			var request api.QueryMembershipForUserRequest
-			var response api.QueryMembershipForUserResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryMembershipsForRoomPath,
-		common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryMembershipsForRoomRequest
-			var response api.QueryMembershipsForRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryInvitesForUserPath,
-		common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse {
-			var request api.QueryInvitesForUserRequest
-			var response api.QueryInvitesForUserResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryServerAllowedToSeeEventPath,
-		common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse {
-			var request api.QueryServerAllowedToSeeEventRequest
-			var response api.QueryServerAllowedToSeeEventResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryMissingEventsPath,
-		common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse {
-			var request api.QueryMissingEventsRequest
-			var response api.QueryMissingEventsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryStateAndAuthChainPath,
-		common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse {
-			var request api.QueryStateAndAuthChainRequest
-			var response api.QueryStateAndAuthChainResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryBackfillPath,
-		common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse {
-			var request api.QueryBackfillRequest
-			var response api.QueryBackfillResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryBackfill(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryRoomVersionCapabilitiesPath,
-		common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse {
-			var request api.QueryRoomVersionCapabilitiesRequest
-			var response api.QueryRoomVersionCapabilitiesResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.RoomserverQueryRoomVersionForRoomPath,
-		common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryRoomVersionForRoomRequest
-			var response api.QueryRoomVersionForRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-}

roomserver/internal/query_backfill.go

@@ -1,8 +1,9 @@
-package query
+package internal
 
 import (
 	"context"
 
+	"github.com/matrix-org/dendrite/roomserver/auth"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -62,9 +63,9 @@ FederationHit:
 	logrus.WithField("event_id", targetEvent.EventID()).Info("Requesting /state_ids at event")
 	for _, srv := range b.servers { // hit any valid server
 		c := gomatrixserverlib.FederatedStateProvider{
-			FedClient:      b.fedClient,
+			FedClient:          b.fedClient,
-			AuthEventsOnly: false,
+			RememberAuthEvents: false,
-			Server:         srv,
+			Server:             srv,
 		}
 		res, err := c.StateIDsBeforeEvent(ctx, targetEvent)
 		if err != nil {
@@ -114,7 +115,9 @@ func (b *backfillRequester) calculateNewStateIDs(targetEvent, prevEvent gomatrix
 	return nil
 }
 
-func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) {
+func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion,
+	event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) {
+
 	// try to fetch the events from the database first
 	events, err := b.ProvideEvents(roomVer, eventIDs)
 	if err != nil {
@@ -133,9 +136,9 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr
 	}
 
 	c := gomatrixserverlib.FederatedStateProvider{
-		FedClient:      b.fedClient,
+		FedClient:          b.fedClient,
-		AuthEventsOnly: false,
+		RememberAuthEvents: false,
-		Server:         b.servers[0],
+		Server:             b.servers[0],
 	}
 	result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs)
 	if err != nil {
@@ -160,18 +163,33 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID
 		return
 	}
 
+	stateEntries, err := stateBeforeEvent(ctx, b.db, NIDs[eventID])
+	if err != nil {
+		logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to load state before event")
+		return
+	}
+
+	// possibly return all joined servers depending on history visiblity
+	memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries)
+	if err != nil {
+		logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules")
+		return
+	}
+	logrus.Infof("ServersAtEvent including %d current events from history visibility", len(memberEventsFromVis))
+
 	// Retrieve all "m.room.member" state events of "join" membership, which
 	// contains the list of users in the room before the event, therefore all
 	// the servers in it at that moment.
-	events, err := getMembershipsBeforeEventNID(ctx, b.db, NIDs[eventID], true)
+	memberEvents, err := getMembershipsAtState(ctx, b.db, stateEntries, true)
 	if err != nil {
 		logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get memberships before event")
 		return
 	}
+	memberEvents = append(memberEvents, memberEventsFromVis...)
 
 	// Store the server names in a temporary map to avoid duplicates.
 	serverSet := make(map[gomatrixserverlib.ServerName]bool)
-	for _, event := range events {
+	for _, event := range memberEvents {
 		serverSet[event.Origin()] = true
 	}
 	for server := range serverSet {
@@ -186,7 +204,9 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID
 
 // Backfill performs a backfill request to the given server.
 // https://matrix.org/docs/spec/server_server/latest#get-matrix-federation-v1-backfill-roomid
-func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string, fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) {
+func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string,
+	fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) {
+
 	tx, err := b.fedClient.Backfill(ctx, server, roomID, limit, fromEventIDs)
 	return &tx, err
 }
@@ -215,3 +235,44 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion,
 	}
 	return events, nil
 }
+
+// joinEventsFromHistoryVisibility returns all CURRENTLY joined members if the provided state indicated a 'shared' history visibility.
+// TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just
+//       pull all events and then filter by that table.
+func joinEventsFromHistoryVisibility(
+	ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry) ([]types.Event, error) {
+
+	var eventNIDs []types.EventNID
+	for _, entry := range stateEntries {
+		// Filter the events to retrieve to only keep the membership events
+		if entry.EventTypeNID == types.MRoomHistoryVisibilityNID && entry.EventStateKeyNID == types.EmptyStateKeyNID {
+			eventNIDs = append(eventNIDs, entry.EventNID)
+			break
+		}
+	}
+
+	// Get all of the events in this state
+	stateEvents, err := db.Events(ctx, eventNIDs)
+	if err != nil {
+		return nil, err
+	}
+	events := make([]gomatrixserverlib.Event, len(stateEvents))
+	for i := range stateEvents {
+		events[i] = stateEvents[i].Event
+	}
+	visibility := auth.HistoryVisibilityForRoom(events)
+	if visibility != "shared" {
+		logrus.Infof("ServersAtEvent history visibility not shared: %s", visibility)
+		return nil, nil
+	}
+	// get joined members
+	roomNID, err := db.RoomNID(ctx, roomID)
+	if err != nil {
+		return nil, err
+	}
+	joinEventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, roomNID, true)
+	if err != nil {
+		return nil, err
+	}
+	return db.Events(ctx, joinEventNIDs)
+}

roomserver/internal/query_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package query
+package internal
 
 import (
 	"context"
@@ -24,7 +24,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// used to implement RoomserverQueryAPIEventDB to test getAuthChain
+// used to implement RoomserverInternalAPIEventDB to test getAuthChain
 type getEventDB struct {
 	eventMap map[string]gomatrixserverlib.Event
 }
@@ -79,7 +79,7 @@ func (db *getEventDB) addFakeEvents(graph map[string][]string) error {
 	return nil
 }
 
-// EventsFromIDs implements RoomserverQueryAPIEventDB
+// EventsFromIDs implements RoomserverInternalAPIEventDB
 func (db *getEventDB) EventsFromIDs(ctx context.Context, eventIDs []string) (res []types.Event, err error) {
 	for _, evID := range eventIDs {
 		res = append(res, types.Event{

roomserver/roomserver.go

@@ -20,11 +20,8 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 
-	asQuery "github.com/matrix-org/dendrite/appservice/query"
 	"github.com/matrix-org/dendrite/common/basecomponent"
-	"github.com/matrix-org/dendrite/roomserver/alias"
+	"github.com/matrix-org/dendrite/roomserver/internal"
-	"github.com/matrix-org/dendrite/roomserver/input"
-	"github.com/matrix-org/dendrite/roomserver/query"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/sirupsen/logrus"
 )
@@ -34,44 +31,27 @@ import (
 // allowing other components running in the same process to hit the query the
 // APIs directly instead of having to use HTTP.
 func SetupRoomServerComponent(
-	base *basecomponent.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier,
+	base *basecomponent.BaseDendrite,
-) (api.RoomserverAliasAPI, api.RoomserverInputAPI, api.RoomserverQueryAPI) {
+	keyRing gomatrixserverlib.JSONVerifier,
-	roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties())
+	fedClient *gomatrixserverlib.FederationClient,
+) api.RoomserverInternalAPI {
+	roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer))
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to room server db")
 	}
 
-	inputAPI := input.RoomserverInputAPI{
+	internalAPI := internal.RoomserverInternalAPI{
 		DB:                   roomserverDB,
+		Cfg:                  base.Cfg,
 		Producer:             base.KafkaProducer,
 		OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent),
+		ImmutableCache:       base.ImmutableCache,
+		ServerName:           base.Cfg.Matrix.ServerName,
+		FedClient:            fedClient,
+		KeyRing:              keyRing,
 	}
 
-	inputAPI.SetupHTTP(http.DefaultServeMux)
+	internalAPI.SetupHTTP(http.DefaultServeMux)
 
-	queryAPI := query.RoomserverQueryAPI{
+	return &internalAPI
-		DB:             roomserverDB,
-		ImmutableCache: base.ImmutableCache,
-		ServerName:     base.Cfg.Matrix.ServerName,
-		FedClient:      base.CreateFederationClient(),
-		// TODO: We should have a key server so we don't keep adding components
-		// which talk to the same DB.
-		KeyRing: keyRing,
-	}
-
-	queryAPI.SetupHTTP(http.DefaultServeMux)
-
-	asAPI := asQuery.AppServiceQueryAPI{Cfg: base.Cfg}
-
-	aliasAPI := alias.RoomserverAliasAPI{
-		DB:            roomserverDB,
-		Cfg:           base.Cfg,
-		InputAPI:      &inputAPI,
-		QueryAPI:      &queryAPI,
-		AppserviceAPI: &asAPI,
-	}
-
-	aliasAPI.SetupHTTP(http.DefaultServeMux)
-
-	return &aliasAPI, &inputAPI, &queryAPI
 }

roomserver/state/state.go

@@ -86,7 +86,10 @@ func (v StateResolution) LoadStateAtEvent(
 ) ([]types.StateEntry, error) {
 	snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %s", eventID, err)
+	}
+	if snapshotNID == 0 {
+		return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID(%s) returned 0 NID, was this event stored?", eventID)
 	}
 
 	stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID)

roomserver/storage/sqlite3/events_table.go

@@ -48,11 +48,6 @@ const insertEventSQL = `
 	  ON CONFLICT DO NOTHING;
 `
 
-const insertEventResultSQL = `
-	SELECT event_nid, state_snapshot_nid FROM roomserver_events
-		WHERE rowid = last_insert_rowid();
-`
-
 const selectEventSQL = "" +
 	"SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1"
 
@@ -102,7 +97,6 @@ const selectRoomNIDForEventNIDSQL = "" +
 type eventStatements struct {
 	db                                     *sql.DB
 	insertEventStmt                        *sql.Stmt
-	insertEventResultStmt                  *sql.Stmt
 	selectEventStmt                        *sql.Stmt
 	bulkSelectStateEventByIDStmt           *sql.Stmt
 	bulkSelectStateAtEventByIDStmt         *sql.Stmt
@@ -126,7 +120,6 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) {
 
 	return statementList{
 		{&s.insertEventStmt, insertEventSQL},
-		{&s.insertEventResultStmt, insertEventResultSQL},
 		{&s.selectEventStmt, selectEventSQL},
 		{&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL},
 		{&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL},
@@ -152,19 +145,22 @@ func (s *eventStatements) insertEvent(
 	referenceSHA256 []byte,
 	authEventNIDs []types.EventNID,
 	depth int64,
-) (types.EventNID, types.StateSnapshotNID, error) {
+) (types.EventNID, error) {
-	var eventNID int64
+	// attempt to insert: the last_row_id is the event NID
-	var stateNID int64
-	var err error
 	insertStmt := common.TxStmt(txn, s.insertEventStmt)
-	resultStmt := common.TxStmt(txn, s.insertEventResultStmt)
+	result, err := insertStmt.ExecContext(
-	if _, err = insertStmt.ExecContext(
 		ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID),
 		eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth,
-	); err == nil {
+	)
-		err = resultStmt.QueryRowContext(ctx).Scan(&eventNID, &stateNID)
+	if err != nil {
+		return 0, err
 	}
-	return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err
+	modified, err := result.RowsAffected()
+	if modified == 0 && err == nil {
+		return 0, sql.ErrNoRows
+	}
+	eventNID, err := result.LastInsertId()
+	return types.EventNID(eventNID), err
 }
 
 func (s *eventStatements) selectEvent(

roomserver/storage/sqlite3/storage.go

@@ -124,7 +124,7 @@ func (d *Database) StoreEvent(
 			}
 		}
 
-		if eventNID, stateNID, err = d.statements.insertEvent(
+		if eventNID, err = d.statements.insertEvent(
 			ctx,
 			txn,
 			roomNID,

roomserver/version/version.go

@@ -51,15 +51,15 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{
 		Stable:    true,
 	},
 	gomatrixserverlib.RoomVersionV5: RoomVersionDescription{
-		Supported: false,
+		Supported: true,
-		Stable:    false,
+		Stable:    true,
 	},
 }
 
 // DefaultRoomVersion contains the room version that will, by
 // default, be used to create new rooms on this server.
 func DefaultRoomVersion() gomatrixserverlib.RoomVersion {
-	return gomatrixserverlib.RoomVersionV4
+	return gomatrixserverlib.RoomVersionV5
 }
 
 // RoomVersions returns a map of all known room versions to this

syncapi/consumers/roomserver.go

@@ -32,10 +32,10 @@ import (
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	roomServerConsumer *common.ContinualConsumer
+	rsAPI      api.RoomserverInternalAPI
-	db                 storage.Database
+	rsConsumer *common.ContinualConsumer
-	notifier           *sync.Notifier
+	db         storage.Database
-	query              api.RoomserverQueryAPI
+	notifier   *sync.Notifier
 }
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
@@ -44,7 +44,7 @@ func NewOutputRoomEventConsumer(
 	kafkaConsumer sarama.Consumer,
 	n *sync.Notifier,
 	store storage.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) *OutputRoomEventConsumer {
 
 	consumer := common.ContinualConsumer{
@@ -53,10 +53,10 @@ func NewOutputRoomEventConsumer(
 		PartitionStore: store,
 	}
 	s := &OutputRoomEventConsumer{
-		roomServerConsumer: &consumer,
+		rsConsumer: &consumer,
-		db:                 store,
+		db:         store,
-		notifier:           n,
+		notifier:   n,
-		query:              queryAPI,
+		rsAPI:      rsAPI,
 	}
 	consumer.ProcessMessage = s.onMessage
 
@@ -65,7 +65,7 @@ func NewOutputRoomEventConsumer(
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	return s.roomServerConsumer.Start()
+	return s.rsConsumer.Start()
 }
 
 // onMessage is called when the sync server receives a new event from the room server output log.
@@ -226,7 +226,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 	// from the roomserver using the query API.
 	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
 	var eventResp api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
+	if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
 		return nil, err
 	}
 

syncapi/routing/messages.go

@@ -34,7 +34,7 @@ import (
 type messagesReq struct {
 	ctx              context.Context
 	db               storage.Database
-	queryAPI         api.RoomserverQueryAPI
+	rsAPI            api.RoomserverInternalAPI
 	federation       *gomatrixserverlib.FederationClient
 	cfg              *config.Dendrite
 	roomID           string
@@ -59,7 +59,7 @@ const defaultMessagesLimit = 10
 func OnIncomingMessagesRequest(
 	req *http.Request, db storage.Database, roomID string,
 	federation *gomatrixserverlib.FederationClient,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var err error
@@ -135,7 +135,7 @@ func OnIncomingMessagesRequest(
 	mReq := messagesReq{
 		ctx:              req.Context(),
 		db:               db,
-		queryAPI:         queryAPI,
+		rsAPI:            rsAPI,
 		federation:       federation,
 		cfg:              cfg,
 		roomID:           roomID,
@@ -229,14 +229,14 @@ func (r *messagesReq) retrieveEvents() (
 	// change the way topological positions are defined (as depth isn't the most
 	// reliable way to define it), it would be easier and less troublesome to
 	// only have to change it in one place, i.e. the database.
-	startPos, err := r.db.EventPositionInTopology(
+	startPos, startStreamPos, err := r.db.EventPositionInTopology(
 		r.ctx, events[0].EventID(),
 	)
 	if err != nil {
 		err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err)
 		return
 	}
-	endPos, err := r.db.EventPositionInTopology(
+	endPos, endStreamPos, err := r.db.EventPositionInTopology(
 		r.ctx, events[len(events)-1].EventID(),
 	)
 	if err != nil {
@@ -246,10 +246,10 @@ func (r *messagesReq) retrieveEvents() (
 	// Generate pagination tokens to send to the client using the positions
 	// retrieved previously.
 	start = types.NewPaginationTokenFromTypeAndPosition(
-		types.PaginationTokenTypeTopology, startPos, 0,
+		types.PaginationTokenTypeTopology, startPos, startStreamPos,
 	)
 	end = types.NewPaginationTokenFromTypeAndPosition(
-		types.PaginationTokenTypeTopology, endPos, 0,
+		types.PaginationTokenTypeTopology, endPos, endStreamPos,
 	)
 
 	if r.backwardOrdering {
@@ -360,7 +360,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent
 // the room or sending the request.
 func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) {
 	var res api.QueryBackfillResponse
-	err := r.queryAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{
+	err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{
 		RoomID:            roomID,
 		EarliestEventsIDs: fromEventIDs,
 		Limit:             limit,
@@ -407,13 +407,13 @@ func setToDefault(
 		// go 1 earlier than the first event so we correctly fetch the earliest event
 		to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0)
 	} else {
-		var pos types.StreamPosition
+		var pos, stream types.StreamPosition
-		pos, err = db.MaxTopologicalPosition(ctx, roomID)
+		pos, stream, err = db.MaxTopologicalPosition(ctx, roomID)
 		if err != nil {
 			return
 		}
 
-		to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, 0)
+		to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, stream)
 	}
 
 	return

syncapi/routing/routing.go

@@ -40,7 +40,7 @@ const pathPrefixR0 = "/_matrix/client/r0"
 func Setup(
 	apiMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database,
 	deviceDB devices.Database, federation *gomatrixserverlib.FederationClient,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 ) {
 	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
@@ -61,6 +61,6 @@ func Setup(
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, queryAPI, cfg)
+		return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, rsAPI, cfg)
 	})).Methods(http.MethodGet, http.MethodOptions)
 }

syncapi/storage/interface.go

@@ -28,26 +28,83 @@ import (
 
 type Database interface {
 	common.PartitionStorer
+	// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs.
 	AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error)
+	// Events lookups a list of event by their event ID.
+	// Returns a list of events matching the requested IDs found in the database.
+	// If an event is not found in the database then it will be omitted from the list.
+	// Returns an error if there was a problem talking with the database.
+	// Does not include any transaction IDs in the returned events.
 	Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error)
-	WriteEvent(context.Context, *gomatrixserverlib.HeaderedEvent, []gomatrixserverlib.HeaderedEvent, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error)
+	// WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races
+	// when generating the sync stream position for this event. Returns the sync stream position for the inserted event.
+	// Returns an error if there was a problem inserting this event.
+	WriteEvent(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, addStateEvents []gomatrixserverlib.HeaderedEvent,
+		addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool) (types.StreamPosition, error)
+	// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key
+	// If no event could be found, returns nil
+	// If there was an issue during the retrieval, returns an error
 	GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
+	// GetStateEventsForRoom fetches the state events for a given room.
+	// Returns an empty slice if no state events could be found for this room.
+	// Returns an error if there was an issue with the retrieval.
 	GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error)
+	// SyncPosition returns the latest positions for syncing.
 	SyncPosition(ctx context.Context) (types.PaginationToken, error)
+	// IncrementalSync returns all the data needed in order to create an incremental
+	// sync response for the given user. Events returned will include any client
+	// transaction IDs associated with the given device. These transaction IDs come
+	// from when the device sent the event via an API that included a transaction
+	// ID.
 	IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error)
+	// CompleteSync returns a complete /sync API response for the given user.
 	CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error)
+	// GetAccountDataInRange returns all account data for a given user inserted or
+	// updated between two given positions
+	// Returns a map following the format data[roomID] = []dataTypes
+	// If no data is retrieved, returns an empty map
+	// If there was an issue with the retrieval, returns an error
 	GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error)
+	// UpsertAccountData keeps track of new or updated account data, by saving the type
+	// of the new/updated data, and the user ID and room ID the data is related to (empty)
+	// room ID means the data isn't specific to any room)
+	// If no data with the given type, user ID and room ID exists in the database,
+	// creates a new row, else update the existing one
+	// Returns an error if there was an issue with the upsert
 	UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error)
+	// AddInviteEvent stores a new invite event for a user.
+	// If the invite was successfully stored this returns the stream ID it was stored at.
+	// Returns an error if there was a problem communicating with the database.
 	AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error)
+	// RetireInviteEvent removes an old invite event from the database.
+	// Returns an error if there was a problem communicating with the database.
 	RetireInviteEvent(ctx context.Context, inviteEventID string) error
+	// SetTypingTimeoutCallback sets a callback function that is called right after
+	// a user is removed from the typing user list due to timeout.
 	SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn)
+	// AddTypingUser adds a typing user to the typing cache.
+	// Returns the newly calculated sync position for typing notifications.
 	AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition
+	// RemoveTypingUser removes a typing user from the typing cache.
+	// Returns the newly calculated sync position for typing notifications.
 	RemoveTypingUser(userID, roomID string) types.StreamPosition
+	// GetEventsInRange retrieves all of the events on a given ordering using the
+	// given extremities and limit.
 	GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error)
-	EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error)
+	// EventPositionInTopology returns the depth and stream position of the given event.
+	EventPositionInTopology(ctx context.Context, eventID string) (depth types.StreamPosition, stream types.StreamPosition, err error)
+	// EventsAtTopologicalPosition returns all of the events matching a given
+	// position in the topology of a given room.
 	EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error)
+	// BackwardExtremitiesForRoom returns the event IDs of all of the backward
+	// extremities we know of for a given room.
 	BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error)
-	MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error)
+	// MaxTopologicalPosition returns the highest topological position for a given room.
+	MaxTopologicalPosition(ctx context.Context, roomID string) (depth types.StreamPosition, stream types.StreamPosition, err error)
+	// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and
+	// matches the streamevent.transactionID device then the transaction ID gets
+	// added to the unsigned section of the output event.
 	StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent
+	// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet.
 	SyncStreamPosition(ctx context.Context) (types.StreamPosition, error)
 }

syncapi/storage/postgres/backward_extremities_table.go

@@ -1,124 +0,0 @@
-// Copyright 2018 New Vector Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"context"
-	"database/sql"
-
-	"github.com/matrix-org/dendrite/common"
-)
-
-// The purpose of this table is to keep track of backwards extremities for a room.
-// Backwards extremities are the earliest (DAG-wise) known events which we have
-// the entire event JSON. These event IDs are used in federation requests to fetch
-// even earlier events.
-//
-// We persist the previous event IDs as well, one per row, so when we do fetch even
-// earlier events we can simply delete rows which referenced it. Consider the graph:
-//        A
-//        |   Event C has 1 prev_event ID: A.
-//    B   C
-//    |___|   Event D has 2 prev_event IDs: B and C.
-//      |
-//      D
-// The earliest known event we have is D, so this table has 2 rows.
-// A backfill request gives us C but not B. We delete rows where prev_event=C. This
-// still means that D is a backwards extremity as we do not have event B. However, event
-// C is *also* a backwards extremity at this point as we do not have event A. Later,
-// when we fetch event B, we delete rows where prev_event=B. This then removes D as
-// a backwards extremity because there are no more rows with event_id=B.
-const backwardExtremitiesSchema = `
--- Stores output room events received from the roomserver.
-CREATE TABLE IF NOT EXISTS syncapi_backward_extremities (
-	-- The 'room_id' key for the event.
-	room_id TEXT NOT NULL,
-	-- The event ID for the last known event. This is the backwards extremity.
-	event_id TEXT NOT NULL,
-	-- The prev_events for the last known event. This is used to update extremities.
-	prev_event_id TEXT NOT NULL,
-
-	PRIMARY KEY(room_id, event_id, prev_event_id)
-);
-`
-
-const insertBackwardExtremitySQL = "" +
-	"INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" +
-	" VALUES ($1, $2, $3)" +
-	" ON CONFLICT DO NOTHING"
-
-const selectBackwardExtremitiesForRoomSQL = "" +
-	"SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1"
-
-const deleteBackwardExtremitySQL = "" +
-	"DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2"
-
-type backwardExtremitiesStatements struct {
-	insertBackwardExtremityStmt          *sql.Stmt
-	selectBackwardExtremitiesForRoomStmt *sql.Stmt
-	deleteBackwardExtremityStmt          *sql.Stmt
-}
-
-func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(backwardExtremitiesSchema)
-	if err != nil {
-		return
-	}
-	if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil {
-		return
-	}
-	if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil {
-		return
-	}
-	if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil {
-		return
-	}
-	return
-}
-
-func (s *backwardExtremitiesStatements) insertsBackwardExtremity(
-	ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string,
-) (err error) {
-	_, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID)
-	return
-}
-
-func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom(
-	ctx context.Context, roomID string,
-) (eventIDs []string, err error) {
-	rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID)
-	if err != nil {
-		return
-	}
-	defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed")
-
-	for rows.Next() {
-		var eID string
-		if err = rows.Scan(&eID); err != nil {
-			return
-		}
-
-		eventIDs = append(eventIDs, eID)
-	}
-
-	return eventIDs, rows.Err()
-}
-
-func (s *backwardExtremitiesStatements) deleteBackwardExtremity(
-	ctx context.Context, txn *sql.Tx, roomID, knownEventID string,
-) (err error) {
-	_, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID)
-	return
-}

syncapi/storage/postgres/output_room_events_table.go

@@ -94,6 +94,9 @@ const selectEarlyEventsSQL = "" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3" +
 	" ORDER BY id ASC LIMIT $4"
 
+const selectStreamPositionForEventIDSQL = "" +
+	"SELECT id FROM syncapi_output_room_events WHERE event_id = $1"
+
 const selectMaxEventIDSQL = "" +
 	"SELECT MAX(id) FROM syncapi_output_room_events"
 
@@ -111,13 +114,14 @@ const selectStateInRangeSQL = "" +
 	" LIMIT $8"
 
 type outputRoomEventsStatements struct {
-	insertEventStmt               *sql.Stmt
+	insertEventStmt                    *sql.Stmt
-	selectEventsStmt              *sql.Stmt
+	selectEventsStmt                   *sql.Stmt
-	selectMaxEventIDStmt          *sql.Stmt
+	selectMaxEventIDStmt               *sql.Stmt
-	selectRecentEventsStmt        *sql.Stmt
+	selectRecentEventsStmt             *sql.Stmt
-	selectRecentEventsForSyncStmt *sql.Stmt
+	selectRecentEventsForSyncStmt      *sql.Stmt
-	selectEarlyEventsStmt         *sql.Stmt
+	selectEarlyEventsStmt              *sql.Stmt
-	selectStateInRangeStmt        *sql.Stmt
+	selectStateInRangeStmt             *sql.Stmt
+	selectStreamPositionForEventIDStmt *sql.Stmt
 }
 
 func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) {
@@ -146,9 +150,18 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) {
 	if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil {
 		return
 	}
+	if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil {
+		return
+	}
 	return
 }
 
+func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) {
+	var id int64
+	err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id)
+	return types.StreamPosition(id), err
+}
+
 // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos.
 // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the
 // two positions, only the most recent state is returned.

syncapi/storage/postgres/output_room_events_topology_table.go

@@ -32,35 +32,44 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology (
 	-- The place of the event in the room's topology. This can usually be determined
 	-- from the event's depth.
 	topological_position BIGINT NOT NULL,
+	stream_position BIGINT NOT NULL,
     -- The 'room_id' key for the event.
     room_id TEXT NOT NULL
 );
 -- The topological order will be used in events selection and ordering
-CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id);
+CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id);
 `
 
 const insertEventInTopologySQL = "" +
-	"INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" +
+	"INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" +
-	" VALUES ($1, $2, $3)" +
+	" VALUES ($1, $2, $3, $4)" +
-	" ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1"
+	" ON CONFLICT (topological_position, stream_position, room_id) DO UPDATE SET event_id = $1"
 
 const selectEventIDsInRangeASCSQL = "" +
 	"SELECT event_id FROM syncapi_output_room_events_topology" +
-	" WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" +
+	" WHERE room_id = $1 AND" +
-	" ORDER BY topological_position ASC LIMIT $4"
+	"(topological_position > $2 AND topological_position < $3) OR" +
+	"(topological_position = $4 AND stream_position <= $5)" +
+	" ORDER BY topological_position ASC, stream_position ASC LIMIT $6"
 
 const selectEventIDsInRangeDESCSQL = "" +
 	"SELECT event_id FROM syncapi_output_room_events_topology" +
-	" WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" +
+	" WHERE room_id = $1 AND" +
-	" ORDER BY topological_position DESC LIMIT $4"
+	"(topological_position > $2 AND topological_position < $3) OR" +
+	"(topological_position = $4 AND stream_position <= $5)" +
+	" ORDER BY topological_position DESC, stream_position DESC LIMIT $6"
 
 const selectPositionInTopologySQL = "" +
 	"SELECT topological_position FROM syncapi_output_room_events_topology" +
 	" WHERE event_id = $1"
 
+	// Select the max topological position for the room, then sort by stream position and take the highest,
+	// returning both topological and stream positions.
 const selectMaxPositionInTopologySQL = "" +
-	"SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" +
+	"SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" +
-	" WHERE room_id = $1"
+	" WHERE topological_position=(" +
+	"SELECT MAX(topological_position) FROM syncapi_output_room_events_topology WHERE room_id=$1" +
+	") ORDER BY stream_position DESC LIMIT 1"
 
 const selectEventIDsFromPositionSQL = "" +
 	"SELECT event_id FROM syncapi_output_room_events_topology" +
@@ -104,10 +113,10 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) {
 // insertEventInTopology inserts the given event in the room's topology, based
 // on the event's depth.
 func (s *outputRoomEventsTopologyStatements) insertEventInTopology(
-	ctx context.Context, event *gomatrixserverlib.HeaderedEvent,
+	ctx context.Context, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition,
 ) (err error) {
 	_, err = s.insertEventInTopologyStmt.ExecContext(
-		ctx, event.EventID(), event.Depth(), event.RoomID(),
+		ctx, event.EventID(), event.Depth(), event.RoomID(), pos,
 	)
 	return
 }
@@ -116,7 +125,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology(
 // given range in a given room's topological order.
 // Returns an empty slice if no events match the given range.
 func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange(
-	ctx context.Context, roomID string, fromPos, toPos types.StreamPosition,
+	ctx context.Context, roomID string, fromPos, toPos, toMicroPos types.StreamPosition,
 	limit int, chronologicalOrder bool,
 ) (eventIDs []string, err error) {
 	// Decide on the selection's order according to whether chronological order
@@ -129,7 +138,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange(
 	}
 
 	// Query the event IDs.
-	rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit)
+	rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit)
 	if err == sql.ErrNoRows {
 		// If no event matched the request, return an empty slice.
 		return []string{}, nil
@@ -161,8 +170,8 @@ func (s *outputRoomEventsTopologyStatements) selectPositionInTopology(
 
 func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology(
 	ctx context.Context, roomID string,
-) (pos types.StreamPosition, err error) {
+) (pos types.StreamPosition, spos types.StreamPosition, err error) {
-	err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos)
+	err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos)
 	return
 }
 

syncapi/storage/postgres/syncserver.go

@@ -32,6 +32,7 @@ import (
 	_ "github.com/lib/pq"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -56,7 +57,7 @@ type SyncServerDatasource struct {
 	invites             inviteEventsStatements
 	eduCache            *cache.EDUCache
 	topology            outputRoomEventsTopologyStatements
-	backwardExtremities backwardExtremitiesStatements
+	backwardExtremities tables.BackwardsExtremities
 }
 
 // NewSyncServerDatasource creates a new sync server database
@@ -75,32 +76,27 @@ func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProp
 	if err = d.events.prepare(d.db); err != nil {
 		return nil, err
 	}
-	if err := d.roomstate.prepare(d.db); err != nil {
+	if err = d.roomstate.prepare(d.db); err != nil {
 		return nil, err
 	}
-	if err := d.invites.prepare(d.db); err != nil {
+	if err = d.invites.prepare(d.db); err != nil {
 		return nil, err
 	}
-	if err := d.topology.prepare(d.db); err != nil {
+	if err = d.topology.prepare(d.db); err != nil {
 		return nil, err
 	}
-	if err := d.backwardExtremities.prepare(d.db); err != nil {
+	d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.PostgresBackwardsExtremitiesStatements{})
+	if err != nil {
 		return nil, err
 	}
 	d.eduCache = cache.New()
 	return &d, nil
 }
 
-// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs.
 func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) {
 	return d.roomstate.selectJoinedUsers(ctx)
 }
 
-// Events lookups a list of event by their event ID.
-// Returns a list of events matching the requested IDs found in the database.
-// If an event is not found in the database then it will be omitted from the list.
-// Returns an error if there was a problem talking with the database.
-// Does not include any transaction IDs in the returned events.
 func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) {
 	streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs)
 	if err != nil {
@@ -116,7 +112,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([
 // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table
 // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such.
 func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error {
-	if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil {
+	if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil {
 		return err
 	}
 
@@ -137,7 +133,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx
 
 		// If the event is missing, consider it a backward extremity.
 		if !found {
-			if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil {
+			if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil {
 				return err
 			}
 		}
@@ -146,9 +142,6 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx
 	return nil
 }
 
-// WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races
-// when generating the sync stream position for this event. Returns the sync stream position for the inserted event.
-// Returns an error if there was a problem inserting this event.
 func (d *SyncServerDatasource) WriteEvent(
 	ctx context.Context,
 	ev *gomatrixserverlib.HeaderedEvent,
@@ -166,7 +159,7 @@ func (d *SyncServerDatasource) WriteEvent(
 		}
 		pduPosition = pos
 
-		if err = d.topology.insertEventInTopology(ctx, ev); err != nil {
+		if err = d.topology.insertEventInTopology(ctx, ev, pos); err != nil {
 			return err
 		}
 
@@ -219,18 +212,12 @@ func (d *SyncServerDatasource) updateRoomState(
 	return nil
 }
 
-// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key
-// If no event could be found, returns nil
-// If there was an issue during the retrieval, returns an error
 func (d *SyncServerDatasource) GetStateEvent(
 	ctx context.Context, roomID, evType, stateKey string,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
 	return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey)
 }
 
-// GetStateEventsForRoom fetches the state events for a given room.
-// Returns an empty slice if no state events could be found for this room.
-// Returns an error if there was an issue with the retrieval.
 func (d *SyncServerDatasource) GetStateEventsForRoom(
 	ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter,
 ) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) {
@@ -241,8 +228,6 @@ func (d *SyncServerDatasource) GetStateEventsForRoom(
 	return
 }
 
-// GetEventsInRange retrieves all of the events on a given ordering using the
-// given extremities and limit.
 func (d *SyncServerDatasource) GetEventsInRange(
 	ctx context.Context,
 	from, to *types.PaginationToken,
@@ -255,12 +240,13 @@ func (d *SyncServerDatasource) GetEventsInRange(
 	if from.Type == types.PaginationTokenTypeTopology {
 		// Determine the backward and forward limit, i.e. the upper and lower
 		// limits to the selection in the room's topology, from the direction.
-		var backwardLimit, forwardLimit types.StreamPosition
+		var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition
 		if backwardOrdering {
 			// Backward ordering is antichronological (latest event to oldest
 			// one).
 			backwardLimit = to.PDUPosition
 			forwardLimit = from.PDUPosition
+			forwardMicroLimit = from.EDUTypingPosition
 		} else {
 			// Forward ordering is chronological (oldest event to latest one).
 			backwardLimit = from.PDUPosition
@@ -270,7 +256,7 @@ func (d *SyncServerDatasource) GetEventsInRange(
 		// Select the event IDs from the defined range.
 		var eIDs []string
 		eIDs, err = d.topology.selectEventIDsInRange(
-			ctx, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering,
+			ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering,
 		)
 		if err != nil {
 			return
@@ -304,29 +290,22 @@ func (d *SyncServerDatasource) GetEventsInRange(
 	return
 }
 
-// SyncPosition returns the latest positions for syncing.
 func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.PaginationToken, error) {
 	return d.syncPositionTx(ctx, nil)
 }
 
-// BackwardExtremitiesForRoom returns the event IDs of all of the backward
-// extremities we know of for a given room.
 func (d *SyncServerDatasource) BackwardExtremitiesForRoom(
 	ctx context.Context, roomID string,
 ) (backwardExtremities []string, err error) {
-	return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID)
+	return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID)
 }
 
-// MaxTopologicalPosition returns the highest topological position for a given
-// room.
 func (d *SyncServerDatasource) MaxTopologicalPosition(
 	ctx context.Context, roomID string,
-) (types.StreamPosition, error) {
+) (depth types.StreamPosition, stream types.StreamPosition, err error) {
 	return d.topology.selectMaxPositionInTopology(ctx, roomID)
 }
 
-// EventsAtTopologicalPosition returns all of the events matching a given
-// position in the topology of a given room.
 func (d *SyncServerDatasource) EventsAtTopologicalPosition(
 	ctx context.Context, roomID string, pos types.StreamPosition,
 ) ([]types.StreamEvent, error) {
@@ -340,11 +319,15 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition(
 
 func (d *SyncServerDatasource) EventPositionInTopology(
 	ctx context.Context, eventID string,
-) (types.StreamPosition, error) {
+) (depth types.StreamPosition, stream types.StreamPosition, err error) {
-	return d.topology.selectPositionInTopology(ctx, eventID)
+	depth, err = d.topology.selectPositionInTopology(ctx, eventID)
+	if err != nil {
+		return
+	}
+	stream, err = d.events.selectStreamPositionForEventID(ctx, eventID)
+	return
 }
 
-// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet.
 func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) {
 	return d.syncStreamPositionTx(ctx, nil)
 }
@@ -509,11 +492,6 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse(
 	return
 }
 
-// IncrementalSync returns all the data needed in order to create an incremental
-// sync response for the given user. Events returned will include any client
-// transaction IDs associated with the given device. These transaction IDs come
-// from when the device sent the event via an API that included a transaction
-// ID.
 func (d *SyncServerDatasource) IncrementalSync(
 	ctx context.Context,
 	device authtypes.Device,
@@ -643,7 +621,6 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
 	return res, toPos, joinedRoomIDs, err
 }
 
-// CompleteSync returns a complete /sync API response for the given user.
 func (d *SyncServerDatasource) CompleteSync(
 	ctx context.Context, userID string, numRecentEventsPerRoom int,
 ) (*types.Response, error) {
@@ -675,11 +652,6 @@ var txReadOnlySnapshot = sql.TxOptions{
 	ReadOnly:  true,
 }
 
-// GetAccountDataInRange returns all account data for a given user inserted or
-// updated between two given positions
-// Returns a map following the format data[roomID] = []dataTypes
-// If no data is retrieved, returns an empty map
-// If there was an issue with the retrieval, returns an error
 func (d *SyncServerDatasource) GetAccountDataInRange(
 	ctx context.Context, userID string, oldPos, newPos types.StreamPosition,
 	accountDataFilterPart *gomatrixserverlib.EventFilter,
@@ -687,29 +659,18 @@ func (d *SyncServerDatasource) GetAccountDataInRange(
 	return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart)
 }
 
-// UpsertAccountData keeps track of new or updated account data, by saving the type
-// of the new/updated data, and the user ID and room ID the data is related to (empty)
-// room ID means the data isn't specific to any room)
-// If no data with the given type, user ID and room ID exists in the database,
-// creates a new row, else update the existing one
-// Returns an error if there was an issue with the upsert
 func (d *SyncServerDatasource) UpsertAccountData(
 	ctx context.Context, userID, roomID, dataType string,
 ) (types.StreamPosition, error) {
 	return d.accountData.insertAccountData(ctx, userID, roomID, dataType)
 }
 
-// AddInviteEvent stores a new invite event for a user.
-// If the invite was successfully stored this returns the stream ID it was stored at.
-// Returns an error if there was a problem communicating with the database.
 func (d *SyncServerDatasource) AddInviteEvent(
 	ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent,
 ) (types.StreamPosition, error) {
 	return d.invites.insertInviteEvent(ctx, inviteEvent)
 }
 
-// RetireInviteEvent removes an old invite event from the database.
-// Returns an error if there was a problem communicating with the database.
 func (d *SyncServerDatasource) RetireInviteEvent(
 	ctx context.Context, inviteEventID string,
 ) error {
@@ -723,16 +684,12 @@ func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallback
 	d.eduCache.SetTimeoutCallback(fn)
 }
 
-// AddTypingUser adds a typing user to the typing cache.
-// Returns the newly calculated sync position for typing notifications.
 func (d *SyncServerDatasource) AddTypingUser(
 	userID, roomID string, expireTime *time.Time,
 ) types.StreamPosition {
 	return types.StreamPosition(d.eduCache.AddTypingUser(userID, roomID, expireTime))
 }
 
-// RemoveTypingUser removes a typing user from the typing cache.
-// Returns the newly calculated sync position for typing notifications.
 func (d *SyncServerDatasource) RemoveTypingUser(
 	userID, roomID string,
 ) types.StreamPosition {
@@ -1071,9 +1028,6 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom(
 	return s, nil
 }
 
-// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and
-// matches the streamevent.transactionID device then the transaction ID gets
-// added to the unsigned section of the output event.
 func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent {
 	out := make([]gomatrixserverlib.HeaderedEvent, len(in))
 	for i := 0; i < len(in); i++ {

syncapi/storage/sqlite3/backward_extremities_table.go

@@ -1,124 +0,0 @@
-// Copyright 2018 New Vector Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-
-	"github.com/matrix-org/dendrite/common"
-)
-
-// The purpose of this table is to keep track of backwards extremities for a room.
-// Backwards extremities are the earliest (DAG-wise) known events which we have
-// the entire event JSON. These event IDs are used in federation requests to fetch
-// even earlier events.
-//
-// We persist the previous event IDs as well, one per row, so when we do fetch even
-// earlier events we can simply delete rows which referenced it. Consider the graph:
-//        A
-//        |   Event C has 1 prev_event ID: A.
-//    B   C
-//    |___|   Event D has 2 prev_event IDs: B and C.
-//      |
-//      D
-// The earliest known event we have is D, so this table has 2 rows.
-// A backfill request gives us C but not B. We delete rows where prev_event=C. This
-// still means that D is a backwards extremity as we do not have event B. However, event
-// C is *also* a backwards extremity at this point as we do not have event A. Later,
-// when we fetch event B, we delete rows where prev_event=B. This then removes D as
-// a backwards extremity because there are no more rows with event_id=B.
-const backwardExtremitiesSchema = `
--- Stores output room events received from the roomserver.
-CREATE TABLE IF NOT EXISTS syncapi_backward_extremities (
-	-- The 'room_id' key for the event.
-	room_id TEXT NOT NULL,
-	-- The event ID for the last known event. This is the backwards extremity.
-	event_id TEXT NOT NULL,
-	-- The prev_events for the last known event. This is used to update extremities.
-	prev_event_id TEXT NOT NULL,
-
-	PRIMARY KEY(room_id, event_id, prev_event_id)
-);
-`
-
-const insertBackwardExtremitySQL = "" +
-	"INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" +
-	" VALUES ($1, $2, $3)" +
-	" ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING"
-
-const selectBackwardExtremitiesForRoomSQL = "" +
-	"SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1"
-
-const deleteBackwardExtremitySQL = "" +
-	"DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2"
-
-type backwardExtremitiesStatements struct {
-	insertBackwardExtremityStmt          *sql.Stmt
-	selectBackwardExtremitiesForRoomStmt *sql.Stmt
-	deleteBackwardExtremityStmt          *sql.Stmt
-}
-
-func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(backwardExtremitiesSchema)
-	if err != nil {
-		return
-	}
-	if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil {
-		return
-	}
-	if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil {
-		return
-	}
-	if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil {
-		return
-	}
-	return
-}
-
-func (s *backwardExtremitiesStatements) insertsBackwardExtremity(
-	ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string,
-) (err error) {
-	_, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID)
-	return
-}
-
-func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom(
-	ctx context.Context, roomID string,
-) (eventIDs []string, err error) {
-	rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID)
-	if err != nil {
-		return
-	}
-	defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed")
-
-	for rows.Next() {
-		var eID string
-		if err = rows.Scan(&eID); err != nil {
-			return
-		}
-
-		eventIDs = append(eventIDs, eID)
-	}
-
-	return eventIDs, rows.Err()
-}
-
-func (s *backwardExtremitiesStatements) deleteBackwardExtremity(
-	ctx context.Context, txn *sql.Tx, roomID, knownEventID string,
-) (err error) {
-	_, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID)
-	return
-}

syncapi/storage/sqlite3/output_room_events_table.go

@@ -74,6 +74,9 @@ const selectEarlyEventsSQL = "" +
 const selectMaxEventIDSQL = "" +
 	"SELECT MAX(id) FROM syncapi_output_room_events"
 
+const selectStreamPositionForEventIDSQL = "" +
+	"SELECT id FROM syncapi_output_room_events WHERE event_id = $1"
+
 // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id).
 /*
 	$1 = oldPos,
@@ -99,14 +102,15 @@ const selectStateInRangeSQL = "" +
 	" LIMIT $8" // limit
 
 type outputRoomEventsStatements struct {
-	streamIDStatements            *streamIDStatements
+	streamIDStatements                 *streamIDStatements
-	insertEventStmt               *sql.Stmt
+	insertEventStmt                    *sql.Stmt
-	selectEventsStmt              *sql.Stmt
+	selectEventsStmt                   *sql.Stmt
-	selectMaxEventIDStmt          *sql.Stmt
+	selectMaxEventIDStmt               *sql.Stmt
-	selectRecentEventsStmt        *sql.Stmt
+	selectRecentEventsStmt             *sql.Stmt
-	selectRecentEventsForSyncStmt *sql.Stmt
+	selectRecentEventsForSyncStmt      *sql.Stmt
-	selectEarlyEventsStmt         *sql.Stmt
+	selectEarlyEventsStmt              *sql.Stmt
-	selectStateInRangeStmt        *sql.Stmt
+	selectStateInRangeStmt             *sql.Stmt
+	selectStreamPositionForEventIDStmt *sql.Stmt
 }
 
 func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) {
@@ -136,9 +140,18 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDState
 	if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil {
 		return
 	}
+	if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil {
+		return
+	}
 	return
 }
 
+func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) {
+	var id int64
+	err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id)
+	return types.StreamPosition(id), err
+}
+
 // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos.
 // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the
 // two positions, only the most recent state is returned.

syncapi/storage/sqlite3/output_room_events_topology_table.go

@@ -27,37 +27,42 @@ const outputRoomEventsTopologySchema = `
 -- Stores output room events received from the roomserver.
 CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology (
   event_id TEXT PRIMARY KEY,
-	topological_position BIGINT NOT NULL,
+  topological_position BIGINT NOT NULL,
+  stream_position BIGINT NOT NULL,
   room_id TEXT NOT NULL,
 
-	UNIQUE(topological_position, room_id)
+	UNIQUE(topological_position, room_id, stream_position)
 );
 -- The topological order will be used in events selection and ordering
--- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id);
+-- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id);
 `
 
 const insertEventInTopologySQL = "" +
-	"INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" +
+	"INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" +
-	" VALUES ($1, $2, $3)" +
+	" VALUES ($1, $2, $3, $4)" +
-	" ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1"
+	" ON CONFLICT DO NOTHING"
 
 const selectEventIDsInRangeASCSQL = "" +
 	"SELECT event_id FROM syncapi_output_room_events_topology" +
-	" WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" +
+	" WHERE room_id = $1 AND" +
-	" ORDER BY topological_position ASC LIMIT $4"
+	"(topological_position > $2 AND topological_position < $3) OR" +
+	"(topological_position = $4 AND stream_position <= $5)" +
+	" ORDER BY topological_position ASC, stream_position ASC LIMIT $6"
 
 const selectEventIDsInRangeDESCSQL = "" +
-	"SELECT event_id FROM syncapi_output_room_events_topology" +
+	"SELECT event_id  FROM syncapi_output_room_events_topology" +
-	" WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" +
+	" WHERE room_id = $1 AND" +
-	" ORDER BY topological_position DESC LIMIT $4"
+	"(topological_position > $2 AND topological_position < $3) OR" +
+	"(topological_position = $4 AND stream_position <= $5)" +
+	" ORDER BY topological_position DESC, stream_position DESC LIMIT $6"
 
 const selectPositionInTopologySQL = "" +
 	"SELECT topological_position FROM syncapi_output_room_events_topology" +
 	" WHERE event_id = $1"
 
 const selectMaxPositionInTopologySQL = "" +
-	"SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" +
+	"SELECT MAX(topological_position), stream_position FROM syncapi_output_room_events_topology" +
-	" WHERE room_id = $1"
+	" WHERE room_id = $1 ORDER BY stream_position DESC"
 
 const selectEventIDsFromPositionSQL = "" +
 	"SELECT event_id FROM syncapi_output_room_events_topology" +
@@ -101,11 +106,11 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) {
 // insertEventInTopology inserts the given event in the room's topology, based
 // on the event's depth.
 func (s *outputRoomEventsTopologyStatements) insertEventInTopology(
-	ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent,
+	ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition,
 ) (err error) {
 	stmt := common.TxStmt(txn, s.insertEventInTopologyStmt)
 	_, err = stmt.ExecContext(
-		ctx, event.EventID(), event.Depth(), event.RoomID(),
+		ctx, event.EventID(), event.Depth(), event.RoomID(), pos,
 	)
 	return
 }
@@ -115,7 +120,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology(
 // Returns an empty slice if no events match the given range.
 func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange(
 	ctx context.Context, txn *sql.Tx, roomID string,
-	fromPos, toPos types.StreamPosition,
+	fromPos, toPos, toMicroPos types.StreamPosition,
 	limit int, chronologicalOrder bool,
 ) (eventIDs []string, err error) {
 	// Decide on the selection's order according to whether chronological order
@@ -128,7 +133,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange(
 	}
 
 	// Query the event IDs.
-	rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit)
+	rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit)
 	if err == sql.ErrNoRows {
 		// If no event matched the request, return an empty slice.
 		return []string{}, nil
@@ -160,9 +165,9 @@ func (s *outputRoomEventsTopologyStatements) selectPositionInTopology(
 
 func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology(
 	ctx context.Context, txn *sql.Tx, roomID string,
-) (pos types.StreamPosition, err error) {
+) (pos types.StreamPosition, spos types.StreamPosition, err error) {
 	stmt := common.TxStmt(txn, s.selectMaxPositionInTopologyStmt)
-	err = stmt.QueryRowContext(ctx, roomID).Scan(&pos)
+	err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos)
 	return
 }
 

syncapi/storage/sqlite3/syncserver.go

@@ -35,6 +35,7 @@ import (
 
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -60,7 +61,7 @@ type SyncServerDatasource struct {
 	invites             inviteEventsStatements
 	eduCache            *cache.EDUCache
 	topology            outputRoomEventsTopologyStatements
-	backwardExtremities backwardExtremitiesStatements
+	backwardExtremities tables.BackwardsExtremities
 }
 
 // NewSyncServerDatasource creates a new sync server database
@@ -102,16 +103,17 @@ func (d *SyncServerDatasource) prepare() (err error) {
 	if err = d.events.prepare(d.db, &d.streamID); err != nil {
 		return err
 	}
-	if err := d.roomstate.prepare(d.db, &d.streamID); err != nil {
+	if err = d.roomstate.prepare(d.db, &d.streamID); err != nil {
 		return err
 	}
-	if err := d.invites.prepare(d.db, &d.streamID); err != nil {
+	if err = d.invites.prepare(d.db, &d.streamID); err != nil {
 		return err
 	}
-	if err := d.topology.prepare(d.db); err != nil {
+	if err = d.topology.prepare(d.db); err != nil {
 		return err
 	}
-	if err := d.backwardExtremities.prepare(d.db); err != nil {
+	d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.SqliteBackwardsExtremitiesStatements{})
+	if err != nil {
 		return err
 	}
 	return nil
@@ -142,7 +144,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([
 // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table
 // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such.
 func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error {
-	if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil {
+	if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil {
 		return err
 	}
 
@@ -163,7 +165,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx
 
 		// If the event is missing, consider it a backward extremity.
 		if !found {
-			if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil {
+			if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil {
 				return err
 			}
 		}
@@ -192,7 +194,7 @@ func (d *SyncServerDatasource) WriteEvent(
 		}
 		pduPosition = pos
 
-		if err = d.topology.insertEventInTopology(ctx, txn, ev); err != nil {
+		if err = d.topology.insertEventInTopology(ctx, txn, ev, pos); err != nil {
 			return err
 		}
 
@@ -279,14 +281,16 @@ func (d *SyncServerDatasource) GetEventsInRange(
 	// events must be retrieved from the rooms' topology table rather than the
 	// table contaning the syncapi server's whole stream of events.
 	if from.Type == types.PaginationTokenTypeTopology {
+		// TODO: ARGH CONFUSING
 		// Determine the backward and forward limit, i.e. the upper and lower
 		// limits to the selection in the room's topology, from the direction.
-		var backwardLimit, forwardLimit types.StreamPosition
+		var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition
 		if backwardOrdering {
 			// Backward ordering is antichronological (latest event to oldest
 			// one).
 			backwardLimit = to.PDUPosition
 			forwardLimit = from.PDUPosition
+			forwardMicroLimit = from.EDUTypingPosition
 		} else {
 			// Forward ordering is chronological (oldest event to latest one).
 			backwardLimit = from.PDUPosition
@@ -296,7 +300,7 @@ func (d *SyncServerDatasource) GetEventsInRange(
 		// Select the event IDs from the defined range.
 		var eIDs []string
 		eIDs, err = d.topology.selectEventIDsInRange(
-			ctx, nil, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering,
+			ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering,
 		)
 		if err != nil {
 			return
@@ -326,8 +330,7 @@ func (d *SyncServerDatasource) GetEventsInRange(
 			return
 		}
 	}
-
+	return events, err
-	return
 }
 
 // SyncPosition returns the latest positions for syncing.
@@ -344,14 +347,14 @@ func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.Pagi
 func (d *SyncServerDatasource) BackwardExtremitiesForRoom(
 	ctx context.Context, roomID string,
 ) (backwardExtremities []string, err error) {
-	return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID)
+	return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID)
 }
 
 // MaxTopologicalPosition returns the highest topological position for a given
 // room.
 func (d *SyncServerDatasource) MaxTopologicalPosition(
 	ctx context.Context, roomID string,
-) (types.StreamPosition, error) {
+) (types.StreamPosition, types.StreamPosition, error) {
 	return d.topology.selectMaxPositionInTopology(ctx, nil, roomID)
 }
 
@@ -370,8 +373,13 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition(
 
 func (d *SyncServerDatasource) EventPositionInTopology(
 	ctx context.Context, eventID string,
-) (types.StreamPosition, error) {
+) (depth types.StreamPosition, stream types.StreamPosition, err error) {
-	return d.topology.selectPositionInTopology(ctx, nil, eventID)
+	depth, err = d.topology.selectPositionInTopology(ctx, nil, eventID)
+	if err != nil {
+		return
+	}
+	stream, err = d.events.selectStreamPositionForEventID(ctx, eventID)
+	return
 }
 
 // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet.

syncapi/storage/storage_test.go

@@ -0,0 +1,413 @@
+package storage_test
+
+import (
+	"context"
+	"crypto/ed25519"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+var (
+	ctx             = context.Background()
+	emptyStateKey   = ""
+	testOrigin      = gomatrixserverlib.ServerName("hollow.knight")
+	testRoomID      = fmt.Sprintf("!hallownest:%s", testOrigin)
+	testUserIDA     = fmt.Sprintf("@hornet:%s", testOrigin)
+	testUserIDB     = fmt.Sprintf("@paleking:%s", testOrigin)
+	testUserDeviceA = authtypes.Device{
+		UserID:      testUserIDA,
+		ID:          "device_id_A",
+		DisplayName: "Device A",
+	}
+	testRoomVersion = gomatrixserverlib.RoomVersionV4
+	testKeyID       = gomatrixserverlib.KeyID("ed25519:storage_test")
+	testPrivateKey  = ed25519.NewKeyFromSeed([]byte{
+		1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+		17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
+	})
+)
+
+func MustCreateEvent(t *testing.T, roomID string, prevs []gomatrixserverlib.HeaderedEvent, b *gomatrixserverlib.EventBuilder) gomatrixserverlib.HeaderedEvent {
+	b.RoomID = roomID
+	if prevs != nil {
+		prevIDs := make([]string, len(prevs))
+		for i := range prevs {
+			prevIDs[i] = prevs[i].EventID()
+		}
+		b.PrevEvents = prevIDs
+	}
+	e, err := b.Build(time.Now(), testOrigin, testKeyID, testPrivateKey, testRoomVersion)
+	if err != nil {
+		t.Fatalf("failed to build event: %s", err)
+	}
+	return e.Headered(testRoomVersion)
+}
+
+func MustCreateDatabase(t *testing.T) storage.Database {
+	db, err := sqlite3.NewSyncServerDatasource("file::memory:")
+	if err != nil {
+		t.Fatalf("NewSyncServerDatasource returned %s", err)
+	}
+	return db
+}
+
+// Create a list of events which include a create event, join event and some messages.
+func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []gomatrixserverlib.HeaderedEvent, state []gomatrixserverlib.HeaderedEvent) {
+	var events []gomatrixserverlib.HeaderedEvent
+	events = append(events, MustCreateEvent(t, roomID, nil, &gomatrixserverlib.EventBuilder{
+		Content:  []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, userA)),
+		Type:     "m.room.create",
+		StateKey: &emptyStateKey,
+		Sender:   userA,
+		Depth:    int64(len(events) + 1),
+	}))
+	state = append(state, events[len(events)-1])
+	events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{
+		Content:  []byte(fmt.Sprintf(`{"membership":"join"}`)),
+		Type:     "m.room.member",
+		StateKey: &userA,
+		Sender:   userA,
+		Depth:    int64(len(events) + 1),
+	}))
+	state = append(state, events[len(events)-1])
+	for i := 0; i < 10; i++ {
+		events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{
+			Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)),
+			Type:    "m.room.message",
+			Sender:  userA,
+			Depth:   int64(len(events) + 1),
+		}))
+	}
+	events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{
+		Content:  []byte(fmt.Sprintf(`{"membership":"join"}`)),
+		Type:     "m.room.member",
+		StateKey: &userB,
+		Sender:   userB,
+		Depth:    int64(len(events) + 1),
+	}))
+	state = append(state, events[len(events)-1])
+	for i := 0; i < 10; i++ {
+		events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{
+			Content: []byte(fmt.Sprintf(`{"body":"Message B %d"}`, i+1)),
+			Type:    "m.room.message",
+			Sender:  userB,
+			Depth:   int64(len(events) + 1),
+		}))
+	}
+
+	return events, state
+}
+
+func MustWriteEvents(t *testing.T, db storage.Database, events []gomatrixserverlib.HeaderedEvent) (positions []types.StreamPosition) {
+	for _, ev := range events {
+		var addStateEvents []gomatrixserverlib.HeaderedEvent
+		var addStateEventIDs []string
+		var removeStateEventIDs []string
+		if ev.StateKey() != nil {
+			addStateEvents = append(addStateEvents, ev)
+			addStateEventIDs = append(addStateEventIDs, ev.EventID())
+		}
+		pos, err := db.WriteEvent(ctx, &ev, addStateEvents, addStateEventIDs, removeStateEventIDs, nil, false)
+		if err != nil {
+			t.Fatalf("WriteEvent failed: %s", err)
+		}
+		positions = append(positions, pos)
+	}
+	return
+}
+
+func TestWriteEvents(t *testing.T) {
+	t.Parallel()
+	db := MustCreateDatabase(t)
+	events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB)
+	MustWriteEvents(t, db, events)
+}
+
+// These tests assert basic functionality of the IncrementalSync and CompleteSync functions.
+func TestSyncResponse(t *testing.T) {
+	t.Parallel()
+	db := MustCreateDatabase(t)
+	events, state := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB)
+	positions := MustWriteEvents(t, db, events)
+	latest, err := db.SyncPosition(ctx)
+	if err != nil {
+		t.Fatalf("failed to get SyncPosition: %s", err)
+	}
+
+	testCases := []struct {
+		Name         string
+		DoSync       func() (*types.Response, error)
+		WantTimeline []gomatrixserverlib.HeaderedEvent
+		WantState    []gomatrixserverlib.HeaderedEvent
+	}{
+		// The purpose of this test is to make sure that incremental syncs are including up to the latest events.
+		// It's a basic sanity test that sync works. It creates a `since` token that is on the penultimate event.
+		// It makes sure the response includes the final event.
+		{
+			Name: "IncrementalSync penultimate",
+			DoSync: func() (*types.Response, error) {
+				from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are at the penultimate event
+					types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0),
+				)
+				return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false)
+			},
+			WantTimeline: events[len(events)-1:],
+		},
+		// The purpose of this test is to check that passing a `numRecentEventsPerRoom` correctly limits the
+		// number of returned events. This is critical for big rooms hence the test here.
+		{
+			Name: "IncrementalSync limited",
+			DoSync: func() (*types.Response, error) {
+				from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are 10 events behind
+					types.PaginationTokenTypeStream, positions[len(positions)-11], types.StreamPosition(0),
+				)
+				// limit is set to 5
+				return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false)
+			},
+			// want the last 5 events, NOT the last 10.
+			WantTimeline: events[len(events)-5:],
+		},
+		// The purpose of this test is to check that CompleteSync returns all the current state as well as
+		// honouring the `numRecentEventsPerRoom` value
+		{
+			Name: "CompleteSync limited",
+			DoSync: func() (*types.Response, error) {
+				// limit set to 5
+				return db.CompleteSync(ctx, testUserIDA, 5)
+			},
+			// want the last 5 events
+			WantTimeline: events[len(events)-5:],
+			// want all state for the room
+			WantState: state,
+		},
+		// The purpose of this test is to check that CompleteSync can return everything with a high enough
+		// `numRecentEventsPerRoom`.
+		{
+			Name: "CompleteSync",
+			DoSync: func() (*types.Response, error) {
+				return db.CompleteSync(ctx, testUserIDA, len(events)+1)
+			},
+			WantTimeline: events,
+			// We want no state at all as that field in /sync is the delta between the token (beginning of time)
+			// and the START of the timeline.
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(st *testing.T) {
+			res, err := tc.DoSync()
+			if err != nil {
+				st.Fatalf("failed to do sync: %s", err)
+			}
+			next := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeStream, latest.PDUPosition, latest.EDUTypingPosition)
+			if res.NextBatch != next.String() {
+				st.Errorf("NextBatch got %s want %s", res.NextBatch, next.String())
+			}
+			roomRes, ok := res.Rooms.Join[testRoomID]
+			if !ok {
+				st.Fatalf("IncrementalSync response missing room %s - response: %+v", testRoomID, res)
+			}
+			assertEventsEqual(st, "state for "+testRoomID, false, roomRes.State.Events, tc.WantState)
+			assertEventsEqual(st, "timeline for "+testRoomID, false, roomRes.Timeline.Events, tc.WantTimeline)
+		})
+	}
+}
+
+// The purpose of this test is to ensure that backfill does indeed go backwards, using a stream token.
+func TestGetEventsInRangeWithStreamToken(t *testing.T) {
+	t.Parallel()
+	db := MustCreateDatabase(t)
+	events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB)
+	MustWriteEvents(t, db, events)
+	latest, err := db.SyncPosition(ctx)
+	if err != nil {
+		t.Fatalf("failed to get SyncPosition: %s", err)
+	}
+	// head towards the beginning of time
+	to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0)
+
+	// backpaginate 5 messages starting at the latest position.
+	paginatedEvents, err := db.GetEventsInRange(ctx, &latest, to, testRoomID, 5, true)
+	if err != nil {
+		t.Fatalf("GetEventsInRange returned an error: %s", err)
+	}
+	gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll)
+	assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:]))
+}
+
+// The purpose of this test is to ensure that backfill does indeed go backwards, using a topology token
+func TestGetEventsInRangeWithTopologyToken(t *testing.T) {
+	t.Parallel()
+	db := MustCreateDatabase(t)
+	events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB)
+	MustWriteEvents(t, db, events)
+	latest, latestStream, err := db.MaxTopologicalPosition(ctx, testRoomID)
+	if err != nil {
+		t.Fatalf("failed to get MaxTopologicalPosition: %s", err)
+	}
+	from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, latestStream)
+	// head towards the beginning of time
+	to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0)
+
+	// backpaginate 5 messages starting at the latest position.
+	paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, 5, true)
+	if err != nil {
+		t.Fatalf("GetEventsInRange returned an error: %s", err)
+	}
+	gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll)
+	assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:]))
+}
+
+// The purpose of this test is to make sure that backpagination returns all events, even if some events have the same depth.
+// For cases where events have the same depth, the streaming token should be used to tie break so events written via WriteEvent
+// will appear FIRST when going backwards. This test creates a DAG like:
+//                            .-----> Message ---.
+//     Create -> Membership --------> Message -------> Message
+//                            `-----> Message ---`
+// depth  1          2                   3                 4
+//
+// With a total depth of 4. It tests that:
+// - Backpagination over the whole fork should include all messages and not leave any out.
+// - Backpagination from the middle of the fork should not return duplicates (things later than the token).
+func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) {
+	t.Parallel()
+	db := MustCreateDatabase(t)
+
+	var events []gomatrixserverlib.HeaderedEvent
+	events = append(events, MustCreateEvent(t, testRoomID, nil, &gomatrixserverlib.EventBuilder{
+		Content:  []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, testUserIDA)),
+		Type:     "m.room.create",
+		StateKey: &emptyStateKey,
+		Sender:   testUserIDA,
+		Depth:    int64(len(events) + 1),
+	}))
+	events = append(events, MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{
+		Content:  []byte(fmt.Sprintf(`{"membership":"join"}`)),
+		Type:     "m.room.member",
+		StateKey: &testUserIDA,
+		Sender:   testUserIDA,
+		Depth:    int64(len(events) + 1),
+	}))
+	// fork the dag into three, same prev_events and depth
+	parent := []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}
+	depth := int64(len(events) + 1)
+	for i := 0; i < 3; i++ {
+		events = append(events, MustCreateEvent(t, testRoomID, parent, &gomatrixserverlib.EventBuilder{
+			Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)),
+			Type:    "m.room.message",
+			Sender:  testUserIDA,
+			Depth:   depth,
+		}))
+	}
+	// merge the fork, prev_events are all 3 messages, depth is increased by 1.
+	events = append(events, MustCreateEvent(t, testRoomID, events[len(events)-3:], &gomatrixserverlib.EventBuilder{
+		Content: []byte(fmt.Sprintf(`{"body":"Message merge"}`)),
+		Type:    "m.room.message",
+		Sender:  testUserIDA,
+		Depth:   depth + 1,
+	}))
+	MustWriteEvents(t, db, events)
+	latestPos, latestStreamPos, err := db.EventPositionInTopology(ctx, events[len(events)-1].EventID())
+	if err != nil {
+		t.Fatalf("failed to get EventPositionInTopology: %s", err)
+	}
+	topoPos, streamPos, err := db.EventPositionInTopology(ctx, events[len(events)-3].EventID()) // Message 2
+	if err != nil {
+		t.Fatalf("failed to get EventPositionInTopology for event: %s", err)
+	}
+	fromLatest := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latestPos, latestStreamPos)
+	fromFork := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, topoPos, streamPos)
+	// head towards the beginning of time
+	to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0)
+
+	testCases := []struct {
+		Name  string
+		From  *types.PaginationToken
+		Limit int
+		Wants []gomatrixserverlib.HeaderedEvent
+	}{
+		{
+			Name:  "Pagination over the whole fork",
+			From:  fromLatest,
+			Limit: 5,
+			Wants: reversed(events[len(events)-5:]),
+		},
+		{
+			Name:  "Paginating to the middle of the fork",
+			From:  fromLatest,
+			Limit: 2,
+			Wants: reversed(events[len(events)-2:]),
+		},
+		{
+			Name:  "Pagination FROM the middle of the fork",
+			From:  fromFork,
+			Limit: 3,
+			Wants: reversed(events[len(events)-5 : len(events)-2]),
+		},
+	}
+
+	for _, tc := range testCases {
+		// backpaginate messages starting at the latest position.
+		paginatedEvents, err := db.GetEventsInRange(ctx, tc.From, to, testRoomID, tc.Limit, true)
+		if err != nil {
+			t.Fatalf("%s GetEventsInRange returned an error: %s", tc.Name, err)
+		}
+		gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll)
+		assertEventsEqual(t, tc.Name, true, gots, tc.Wants)
+	}
+}
+
+func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) {
+	if len(gots) != len(wants) {
+		t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants))
+	}
+	for i := range gots {
+		g := gots[i]
+		w := wants[i]
+		if g.EventID != w.EventID() {
+			t.Errorf("%s event[%d] event_id mismatch: got %s want %s", msg, i, g.EventID, w.EventID())
+		}
+		if g.Sender != w.Sender() {
+			t.Errorf("%s event[%d] sender mismatch: got %s want %s", msg, i, g.Sender, w.Sender())
+		}
+		if checkRoomID && g.RoomID != w.RoomID() {
+			t.Errorf("%s event[%d] room_id mismatch: got %s want %s", msg, i, g.RoomID, w.RoomID())
+		}
+		if g.Type != w.Type() {
+			t.Errorf("%s event[%d] event type mismatch: got %s want %s", msg, i, g.Type, w.Type())
+		}
+		if g.OriginServerTS != w.OriginServerTS() {
+			t.Errorf("%s event[%d] origin_server_ts mismatch: got %v want %v", msg, i, g.OriginServerTS, w.OriginServerTS())
+		}
+		if string(g.Content) != string(w.Content()) {
+			t.Errorf("%s event[%d] content mismatch: got %s want %s", msg, i, string(g.Content), string(w.Content()))
+		}
+		if string(g.Unsigned) != string(w.Unsigned()) {
+			t.Errorf("%s event[%d] unsigned mismatch: got %s want %s", msg, i, string(g.Unsigned), string(w.Unsigned()))
+		}
+		if (g.StateKey == nil && w.StateKey() != nil) || (g.StateKey != nil && w.StateKey() == nil) {
+			t.Errorf("%s event[%d] state_key [not] missing: got %v want %v", msg, i, g.StateKey, w.StateKey())
+			continue
+		}
+		if g.StateKey != nil {
+			if !w.StateKeyEquals(*g.StateKey) {
+				t.Errorf("%s event[%d] state_key mismatch: got %s want %s", msg, i, *g.StateKey, *w.StateKey())
+			}
+		}
+	}
+}
+
+func reversed(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent {
+	out := make([]gomatrixserverlib.HeaderedEvent, len(in))
+	for i := 0; i < len(in); i++ {
+		out[i] = in[len(in)-i-1]
+	}
+	return out
+}

syncapi/storage/tables/backward_extremities.go

@@ -0,0 +1,175 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/common"
+)
+
+// BackwardsExtremitiesStatements contains the SQL statements to implement.
+// See BackwardsExtremities to see the parameter and response types.
+type BackwardsExtremitiesStatements interface {
+	Schema() string
+	InsertBackwardExtremity() string
+	SelectBackwardExtremitiesForRoom() string
+	DeleteBackwardExtremity() string
+}
+
+type PostgresBackwardsExtremitiesStatements struct{}
+
+func (s *PostgresBackwardsExtremitiesStatements) Schema() string {
+	return `-- Stores output room events received from the roomserver.
+	CREATE TABLE IF NOT EXISTS syncapi_backward_extremities (
+		-- The 'room_id' key for the event.
+		room_id TEXT NOT NULL,
+		-- The event ID for the last known event. This is the backwards extremity.
+		event_id TEXT NOT NULL,
+		-- The prev_events for the last known event. This is used to update extremities.
+		prev_event_id TEXT NOT NULL,
+	
+		PRIMARY KEY(room_id, event_id, prev_event_id)
+	);
+	`
+}
+func (s *PostgresBackwardsExtremitiesStatements) InsertBackwardExtremity() string {
+	return "" +
+		"INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" +
+		" VALUES ($1, $2, $3)" +
+		" ON CONFLICT DO NOTHING"
+}
+func (s *PostgresBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string {
+	return "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1"
+}
+func (s *PostgresBackwardsExtremitiesStatements) DeleteBackwardExtremity() string {
+	return "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2"
+}
+
+type SqliteBackwardsExtremitiesStatements struct{}
+
+func (s *SqliteBackwardsExtremitiesStatements) Schema() string {
+	return `-- Stores output room events received from the roomserver.
+CREATE TABLE IF NOT EXISTS syncapi_backward_extremities (
+	-- The 'room_id' key for the event.
+	room_id TEXT NOT NULL,
+	-- The event ID for the last known event. This is the backwards extremity.
+	event_id TEXT NOT NULL,
+	-- The prev_events for the last known event. This is used to update extremities.
+	prev_event_id TEXT NOT NULL,
+
+	PRIMARY KEY(room_id, event_id, prev_event_id)
+);
+`
+}
+
+func (s *SqliteBackwardsExtremitiesStatements) InsertBackwardExtremity() string {
+	return "" +
+		"INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" +
+		" VALUES ($1, $2, $3)" +
+		" ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING"
+}
+
+func (s *SqliteBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string {
+	return "" +
+		"SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1"
+}
+
+func (s *SqliteBackwardsExtremitiesStatements) DeleteBackwardExtremity() string {
+	return "" +
+		"DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2"
+}
+
+// BackwardsExtremities keeps track of backwards extremities for a room.
+// Backwards extremities are the earliest (DAG-wise) known events which we have
+// the entire event JSON. These event IDs are used in federation requests to fetch
+// even earlier events.
+//
+// We persist the previous event IDs as well, one per row, so when we do fetch even
+// earlier events we can simply delete rows which referenced it. Consider the graph:
+//        A
+//        |   Event C has 1 prev_event ID: A.
+//    B   C
+//    |___|   Event D has 2 prev_event IDs: B and C.
+//      |
+//      D
+// The earliest known event we have is D, so this table has 2 rows.
+// A backfill request gives us C but not B. We delete rows where prev_event=C. This
+// still means that D is a backwards extremity as we do not have event B. However, event
+// C is *also* a backwards extremity at this point as we do not have event A. Later,
+// when we fetch event B, we delete rows where prev_event=B. This then removes D as
+// a backwards extremity because there are no more rows with event_id=B.
+type BackwardsExtremities struct {
+	insertBackwardExtremityStmt          *sql.Stmt
+	selectBackwardExtremitiesForRoomStmt *sql.Stmt
+	deleteBackwardExtremityStmt          *sql.Stmt
+}
+
+// NewBackwardsExtremities prepares the table
+func NewBackwardsExtremities(db *sql.DB, stmts BackwardsExtremitiesStatements) (table BackwardsExtremities, err error) {
+	_, err = db.Exec(stmts.Schema())
+	if err != nil {
+		return
+	}
+	if table.insertBackwardExtremityStmt, err = db.Prepare(stmts.InsertBackwardExtremity()); err != nil {
+		return
+	}
+	if table.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(stmts.SelectBackwardExtremitiesForRoom()); err != nil {
+		return
+	}
+	if table.deleteBackwardExtremityStmt, err = db.Prepare(stmts.DeleteBackwardExtremity()); err != nil {
+		return
+	}
+	return
+}
+
+// InsertsBackwardExtremity inserts a new backwards extremity.
+func (s *BackwardsExtremities) InsertsBackwardExtremity(
+	ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string,
+) (err error) {
+	_, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID)
+	return
+}
+
+// SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room.
+func (s *BackwardsExtremities) SelectBackwardExtremitiesForRoom(
+	ctx context.Context, roomID string,
+) (eventIDs []string, err error) {
+	rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID)
+	if err != nil {
+		return
+	}
+	defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed")
+
+	for rows.Next() {
+		var eID string
+		if err = rows.Scan(&eID); err != nil {
+			return
+		}
+
+		eventIDs = append(eventIDs, eID)
+	}
+
+	return eventIDs, rows.Err()
+}
+
+// DeleteBackwardExtremity removes a backwards extremity for a room, if one existed.
+func (s *BackwardsExtremities) DeleteBackwardExtremity(
+	ctx context.Context, txn *sql.Tx, roomID, knownEventID string,
+) (err error) {
+	_, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID)
+	return
+}

syncapi/syncapi.go

@@ -38,7 +38,7 @@ func SetupSyncAPIComponent(
 	base *basecomponent.BaseDendrite,
 	deviceDB devices.Database,
 	accountsDB accounts.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.Dendrite,
 ) {
@@ -61,7 +61,7 @@ func SetupSyncAPIComponent(
 	requestPool := sync.NewRequestPool(syncDB, notifier, accountsDB)
 
 	roomConsumer := consumers.NewOutputRoomEventConsumer(
-		base.Cfg, base.KafkaConsumer, notifier, syncDB, queryAPI,
+		base.Cfg, base.KafkaConsumer, notifier, syncDB, rsAPI,
 	)
 	if err = roomConsumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start room server consumer")
@@ -81,5 +81,5 @@ func SetupSyncAPIComponent(
 		logrus.WithError(err).Panicf("failed to start typing server consumer")
 	}
 
-	routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, queryAPI, cfg)
+	routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg)
 }

syncapi/types/types.go

@@ -64,8 +64,14 @@ const (
 // /sync or /messages, for example.
 type PaginationToken struct {
 	//Position StreamPosition
-	Type              PaginationTokenType
+	Type PaginationTokenType
-	PDUPosition       StreamPosition
+	// For /sync, this is the PDU position. For /messages, this is the topological position (depth).
+	// TODO: Given how different the positions are depending on the token type, they should probably be renamed
+	//       or use different structs altogether.
+	PDUPosition StreamPosition
+	// For /sync, this is the EDU position. For /messages, this is the stream (PDU) position.
+	// TODO: Given how different the positions are depending on the token type, they should probably be renamed
+	//       or use different structs altogether.
 	EDUTypingPosition StreamPosition
 }
 

sytest-whitelist

@@ -258,3 +258,9 @@ User can invite remote user to room with version 1
 User can invite remote user to room with version 2
 User can invite remote user to room with version 3
 User can invite remote user to room with version 4
+User can create and send/receive messages in a room with version 5
+local user can join room with version 5
+User can invite local user to room with version 5
+remote user can join room with version 5
+User can invite remote user to room with version 5
+Remote user can backfill in a room with version 5