Merge branch 'master' into public-visibility-preset

appservice/appservice.go

@@ -44,12 +44,11 @@ func SetupAppServiceAPIComponent(
 	accountsDB accounts.Database,
 	deviceDB devices.Database,
 	federation *gomatrixserverlib.FederationClient,
-	roomserverAliasAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	roomserverQueryAPI roomserverAPI.RoomserverQueryAPI,
 	transactionsCache *transactions.Cache,
 ) appserviceAPI.AppServiceQueryAPI {
 	// Create a connection to the appservice postgres DB
-	appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService))
+	appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to appservice db")
 	}
@@ -87,7 +86,7 @@ func SetupAppServiceAPIComponent(
 
 	consumer := consumers.NewOutputRoomEventConsumer(
 		base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB,
-		roomserverQueryAPI, roomserverAliasAPI, workerStates,
+		rsAPI, workerStates,
 	)
 	if err := consumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
@@ -100,7 +99,7 @@ func SetupAppServiceAPIComponent(
 
 	// Set up HTTP Endpoints
 	routing.Setup(
-		base.APIMux, base.Cfg, roomserverQueryAPI, roomserverAliasAPI,
+		base.APIMux, base.Cfg, rsAPI,
 		accountsDB, federation, transactionsCache,
 	)
 

appservice/consumers/roomserver.go

@@ -26,8 +26,8 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/Shopify/sarama"
 	log "github.com/sirupsen/logrus"
-	sarama "gopkg.in/Shopify/sarama.v1"
 )
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
@@ -35,8 +35,7 @@ type OutputRoomEventConsumer struct {
 	roomServerConsumer *common.ContinualConsumer
 	db                 accounts.Database
 	asDB               storage.Database
-	query              api.RoomserverQueryAPI
+	rsAPI              api.RoomserverInternalAPI
-	alias              api.RoomserverAliasAPI
 	serverName         string
 	workerStates       []types.ApplicationServiceWorkerState
 }
@@ -48,8 +47,7 @@ func NewOutputRoomEventConsumer(
 	kafkaConsumer sarama.Consumer,
 	store accounts.Database,
 	appserviceDB storage.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
-	aliasAPI api.RoomserverAliasAPI,
 	workerStates []types.ApplicationServiceWorkerState,
 ) *OutputRoomEventConsumer {
 	consumer := common.ContinualConsumer{
@@ -61,8 +59,7 @@ func NewOutputRoomEventConsumer(
 		roomServerConsumer: &consumer,
 		db:                 store,
 		asDB:               appserviceDB,
-		query:              queryAPI,
+		rsAPI:              rsAPI,
-		alias:              aliasAPI,
 		serverName:         string(cfg.Matrix.ServerName),
 		workerStates:       workerStates,
 	}
@@ -139,7 +136,7 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents(
 	// Request the missing events from the roomserver
 	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
 	var eventResp api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
+	if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
 		return nil, err
 	}
 
@@ -200,7 +197,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	// Check all known room aliases of the room the event came from
 	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
 	var queryRes api.GetAliasesForRoomIDResponse
-	if err := s.alias.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
+	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
 		for _, alias := range queryRes.Aliases {
 			if appservice.IsInterestedInRoomAlias(alias) {
 				return true

appservice/routing/routing.go

@@ -37,7 +37,7 @@ const pathPrefixApp = "/_matrix/app/v1"
 // nolint: gocyclo
 func Setup(
 	apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam
-	queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam
+	rsAPI api.RoomserverInternalAPI, // nolint: unparam
 	accountDB accounts.Database, // nolint: unparam
 	federation *gomatrixserverlib.FederationClient, // nolint: unparam
 	transactionsCache *transactions.Cache, // nolint: unparam

appservice/storage/postgres/storage.go

@@ -21,6 +21,7 @@ import (
 
 	// Import postgres database driver
 	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -33,10 +34,10 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

appservice/storage/sqlite3/storage.go

@@ -37,7 +37,7 @@ type Database struct {
 func NewDatabase(dataSourceName string) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

appservice/storage/storage.go

@@ -21,19 +21,22 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage/postgres"
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 )
 
-func NewDatabase(dataSourceName string) (Database, error) {
+// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
+// and sets DB connection parameters
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName)
 	default:
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 }

appservice/storage/storage_wasm.go

@@ -19,9 +19,13 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 )
 
-func NewDatabase(dataSourceName string) (Database, error) {
+func NewDatabase(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

are-we-synapse-yet.list

@@ -28,6 +28,10 @@ log POST /login returns the same device_id as that in the request
 log POST /login can log in as a user with just the local part of the id 
 log POST /login as non-existing user is rejected 
 log POST /login wrong password is rejected 
+log Interactive authentication types include SSO
+log Can perform interactive authentication with SSO
+log The user must be consistent through an interactive authentication session with SSO
+log The operation must be consistent through an interactive authentication session
 v1s GET /events initially
 v1s GET /initialSync initially
 csa Version responds 200 OK with valid structure
@@ -44,6 +48,7 @@ dev DELETE /device/{deviceId}
 dev DELETE /device/{deviceId} requires UI auth user to match device owner 
 dev DELETE /device/{deviceId} with no body gives a 401 
 dev The deleted device must be consistent through an interactive auth session 
+dev Users receive device_list updates for their own devices
 pre GET /presence/:user_id/status fetches initial status 
 pre PUT /presence/:user_id/status updates my presence 
 crm POST /createRoom makes a public room

build.sh

@@ -4,3 +4,5 @@
 export GOBIN=$PWD/`dirname $0`/bin
 
 go install -v $PWD/`dirname $0`/cmd/...
+
+GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs

clientapi/auth/storage/accounts/postgres/storage.go

@@ -44,10 +44,10 @@ type Database struct {
 }
 
 // NewDatabase creates a new accounts and profiles database
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	partitions := common.PartitionOffsetStatements{}

clientapi/auth/storage/accounts/sqlite3/storage.go

@@ -50,7 +50,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	partitions := common.PartitionOffsetStatements{}

clientapi/auth/storage/accounts/storage.go

@@ -21,20 +21,23 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
+// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
+// and sets postgres connection parameters
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName, serverName)
 	default:
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	}
 }

clientapi/auth/storage/accounts/storage_wasm.go

@@ -19,10 +19,15 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
+func NewDatabase(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+	serverName gomatrixserverlib.ServerName,
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

clientapi/auth/storage/devices/postgres/storage.go

@@ -36,10 +36,10 @@ type Database struct {
 }
 
 // NewDatabase creates a new device database
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	d := devicesStatements{}

clientapi/auth/storage/devices/sqlite3/storage.go

@@ -41,7 +41,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	d := devicesStatements{}

clientapi/auth/storage/devices/storage.go

@@ -21,20 +21,23 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
+// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
+// and sets postgres connection parameters
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName, serverName)
 	default:
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	}
 }

clientapi/auth/storage/devices/storage_wasm.go

@@ -19,10 +19,15 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
+func NewDatabase(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+	serverName gomatrixserverlib.ServerName,
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

clientapi/clientapi.go

@@ -38,15 +38,13 @@ func SetupClientAPIComponent(
 	accountsDB accounts.Database,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing *gomatrixserverlib.KeyRing,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	inputAPI roomserverAPI.RoomserverInputAPI,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
 	eduInputAPI eduServerAPI.EDUServerInputAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	transactionsCache *transactions.Cache,
-	fedSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	fsAPI federationSenderAPI.FederationSenderInternalAPI,
 ) {
-	roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI)
+	roomserverProducer := producers.NewRoomserverProducer(rsAPI)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 
 	userUpdateProducer := &producers.UserUpdateProducer{
@@ -60,15 +58,15 @@ func SetupClientAPIComponent(
 	}
 
 	consumer := consumers.NewOutputRoomEventConsumer(
-		base.Cfg, base.KafkaConsumer, accountsDB, queryAPI,
+		base.Cfg, base.KafkaConsumer, accountsDB, rsAPI,
 	)
 	if err := consumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start room server consumer")
 	}
 
 	routing.Setup(
-		base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI,
+		base.APIMux, base.Cfg, roomserverProducer, rsAPI, asAPI,
 		accountsDB, deviceDB, federation, *keyRing, userUpdateProducer,
-		syncProducer, eduProducer, transactionsCache, fedSenderAPI,
+		syncProducer, eduProducer, transactionsCache, fsAPI,
 	)
 }

clientapi/consumers/roomserver.go

@@ -24,15 +24,15 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/Shopify/sarama"
 	log "github.com/sirupsen/logrus"
-	sarama "gopkg.in/Shopify/sarama.v1"
 )
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	roomServerConsumer *common.ContinualConsumer
+	rsAPI      api.RoomserverInternalAPI
+	rsConsumer *common.ContinualConsumer
 	db         accounts.Database
-	query              api.RoomserverQueryAPI
 	serverName string
 }
 
@@ -41,7 +41,7 @@ func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
 	store accounts.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) *OutputRoomEventConsumer {
 
 	consumer := common.ContinualConsumer{
@@ -50,9 +50,9 @@ func NewOutputRoomEventConsumer(
 		PartitionStore: store,
 	}
 	s := &OutputRoomEventConsumer{
-		roomServerConsumer: &consumer,
+		rsConsumer: &consumer,
 		db:         store,
-		query:              queryAPI,
+		rsAPI:      rsAPI,
 		serverName: string(cfg.Matrix.ServerName),
 	}
 	consumer.ProcessMessage = s.onMessage
@@ -62,7 +62,7 @@ func NewOutputRoomEventConsumer(
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	return s.roomServerConsumer.Start()
+	return s.rsConsumer.Start()
 }
 
 // onMessage is called when the sync server receives a new event from the room server output log.
@@ -134,7 +134,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 	// Request the missing events from the roomserver
 	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
 	var eventResp api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
+	if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
 		return nil, err
 	}
 

clientapi/jsonerror/jsonerror.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
@@ -124,6 +125,12 @@ func GuestAccessForbidden(msg string) *MatrixError {
 	return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg}
 }
 
+// IncompatibleRoomVersion is an error which is returned when the client
+// requests a room with a version that is unsupported.
+func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *MatrixError {
+	return &MatrixError{"M_INCOMPATIBLE_ROOM_VERSION", string(roomVersion)}
+}
+
 // UnsupportedRoomVersion is an error which is returned when the client
 // requests a room with a version that is unsupported.
 func UnsupportedRoomVersion(msg string) *MatrixError {

clientapi/producers/roomserver.go

@@ -23,15 +23,13 @@ import (
 
 // RoomserverProducer produces events for the roomserver to consume.
 type RoomserverProducer struct {
-	InputAPI api.RoomserverInputAPI
+	RsAPI api.RoomserverInternalAPI
-	QueryAPI api.RoomserverQueryAPI
 }
 
 // NewRoomserverProducer creates a new RoomserverProducer
-func NewRoomserverProducer(inputAPI api.RoomserverInputAPI, queryAPI api.RoomserverQueryAPI) *RoomserverProducer {
+func NewRoomserverProducer(rsAPI api.RoomserverInternalAPI) *RoomserverProducer {
 	return &RoomserverProducer{
-		InputAPI: inputAPI,
+		RsAPI: rsAPI,
-		QueryAPI: queryAPI,
 	}
 }
 
@@ -95,7 +93,7 @@ func (c *RoomserverProducer) SendInputRoomEvents(
 ) (eventID string, err error) {
 	request := api.InputRoomEventsRequest{InputRoomEvents: ires}
 	var response api.InputRoomEventsResponse
-	err = c.InputAPI.InputRoomEvents(ctx, &request, &response)
+	err = c.RsAPI.InputRoomEvents(ctx, &request, &response)
 	eventID = response.EventID
 	return
 }
@@ -106,14 +104,17 @@ func (c *RoomserverProducer) SendInputRoomEvents(
 func (c *RoomserverProducer) SendInvite(
 	ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent,
 	inviteRoomState []gomatrixserverlib.InviteV2StrippedState,
+	sendAsServer gomatrixserverlib.ServerName, txnID *api.TransactionID,
 ) error {
 	request := api.InputRoomEventsRequest{
 		InputInviteEvents: []api.InputInviteEvent{{
 			Event:           inviteEvent,
 			InviteRoomState: inviteRoomState,
 			RoomVersion:     inviteEvent.RoomVersion,
+			SendAsServer:    string(sendAsServer),
+			TransactionID:   txnID,
 		}},
 	}
 	var response api.InputRoomEventsResponse
-	return c.InputAPI.InputRoomEvents(ctx, &request, &response)
+	return c.RsAPI.InputRoomEvents(ctx, &request, &response)
 }

clientapi/producers/syncapi.go

@@ -19,7 +19,7 @@ import (
 
 	"github.com/matrix-org/dendrite/common"
 
-	sarama "gopkg.in/Shopify/sarama.v1"
+	"github.com/Shopify/sarama"
 )
 
 // SyncAPIProducer produces events for the sync API server to consume

clientapi/producers/userupdate.go

@@ -17,7 +17,7 @@ package producers
 import (
 	"encoding/json"
 
-	sarama "gopkg.in/Shopify/sarama.v1"
+	"github.com/Shopify/sarama"
 )
 
 // UserUpdateProducer produces events related to user updates.

clientapi/routing/capabilities.go

@@ -26,11 +26,11 @@ import (
 // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
 // by building a m.room.member event then sending it to the room server
 func GetCapabilities(
-	req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI,
+	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{}
 	roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{}
-	if err := queryAPI.QueryRoomVersionCapabilities(
+	if err := rsAPI.QueryRoomVersionCapabilities(
 		req.Context(),
 		&roomVersionsQueryReq,
 		&roomVersionsQueryRes,

clientapi/routing/createroom.go

@@ -164,13 +164,13 @@ type fledglingEvent struct {
 func CreateRoom(
 	req *http.Request, device *authtypes.Device,
 	cfg *config.Dendrite, producer *producers.RoomserverProducer,
-	accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	// TODO (#267): Check room ID doesn't clash with an existing one, and we
 	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
 	roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
-	return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI, asAPI)
+	return createRoom(req, device, cfg, roomID, producer, accountDB, rsAPI, asAPI)
 }
 
 // createRoom implements /createRoom
@@ -178,7 +178,7 @@ func CreateRoom(
 func createRoom(
 	req *http.Request, device *authtypes.Device,
 	cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer,
-	accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	logger := util.GetLogger(req.Context())
@@ -400,7 +400,7 @@ func createRoom(
 		}
 
 		var aliasResp roomserverAPI.SetRoomAliasResponse
-		err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp)
+		err = rsAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
 			return jsonerror.InternalServerError()

clientapi/routing/directory.go

@@ -46,8 +46,8 @@ func DirectoryRoom(
 	roomAlias string,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.Dendrite,
-	rsAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	fedSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	fedSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
 	if err != nil {
@@ -115,7 +115,7 @@ func SetLocalAlias(
 	device *authtypes.Device,
 	alias string,
 	cfg *config.Dendrite,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', alias)
 	if err != nil {
@@ -190,7 +190,7 @@ func RemoveLocalAlias(
 	req *http.Request,
 	device *authtypes.Device,
 	alias string,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 
 	creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{

clientapi/routing/getevent.go

@@ -44,7 +44,7 @@ func GetEvent(
 	roomID string,
 	eventID string,
 	cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing gomatrixserverlib.KeyRing,
 ) util.JSONResponse {
@@ -52,7 +52,7 @@ func GetEvent(
 		EventIDs: []string{eventID},
 	}
 	var eventsResp api.QueryEventsByIDResponse
-	err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
+	err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed")
 		return jsonerror.InternalServerError()
@@ -88,7 +88,7 @@ func GetEvent(
 		}},
 	}
 	var stateResp api.QueryStateAfterEventsResponse
-	if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
+	if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/joinroom.go

@@ -15,434 +15,48 @@
 package routing
 
 import (
-	"fmt"
 	"net/http"
-	"strings"
-	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/producers"
-	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrix"
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
-// JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API.
-// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-join-roomidoralias
 func JoinRoomByIDOrAlias(
 	req *http.Request,
 	device *authtypes.Device,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 	roomIDOrAlias string,
-	cfg *config.Dendrite,
-	federation *gomatrixserverlib.FederationClient,
-	producer *producers.RoomserverProducer,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
-	keyRing gomatrixserverlib.KeyRing,
-	accountDB accounts.Database,
 ) util.JSONResponse {
-	var content map[string]interface{} // must be a JSON object
+	// Prepare to ask the roomserver to perform the room join.
-	if resErr := httputil.UnmarshalJSONRequest(req, &content); resErr != nil {
+	joinReq := roomserverAPI.PerformJoinRequest{
-		return *resErr
+		RoomIDOrAlias: roomIDOrAlias,
+		UserID:        device.UserID,
+	}
+	joinRes := roomserverAPI.PerformJoinResponse{}
+
+	// If content was provided in the request then incude that
+	// in the request. It'll get used as a part of the membership
+	// event content.
+	if err := httputil.UnmarshalJSONRequest(req, &joinReq.Content); err != nil {
+		return *err
 	}
 
-	evTime, err := httputil.ParseTSParam(req)
+	// Ask the roomserver to perform the join.
-	if err != nil {
+	if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+			JSON: jsonerror.Unknown(err.Error()),
 		}
 	}
 
-	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
-	}
-
-	profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed")
-		return jsonerror.InternalServerError()
-	}
-
-	content["membership"] = gomatrixserverlib.Join
-	content["displayname"] = profile.DisplayName
-	content["avatar_url"] = profile.AvatarURL
-
-	r := joinRoomReq{
-		req, evTime, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing,
-	}
-
-	if strings.HasPrefix(roomIDOrAlias, "!") {
-		return r.joinRoomByID(roomIDOrAlias)
-	}
-	if strings.HasPrefix(roomIDOrAlias, "#") {
-		return r.joinRoomByAlias(roomIDOrAlias)
-	}
 	return util.JSONResponse{
-		Code: http.StatusBadRequest,
-		JSON: jsonerror.BadJSON(
-			fmt.Sprintf("Invalid first character '%s' for room ID or alias",
-				string([]rune(roomIDOrAlias)[0])), // Wrapping with []rune makes this call UTF-8 safe
-		),
-	}
-}
-
-type joinRoomReq struct {
-	req        *http.Request
-	evTime     time.Time
-	content    map[string]interface{}
-	userID     string
-	cfg        *config.Dendrite
-	federation *gomatrixserverlib.FederationClient
-	producer   *producers.RoomserverProducer
-	queryAPI   roomserverAPI.RoomserverQueryAPI
-	aliasAPI   roomserverAPI.RoomserverAliasAPI
-	keyRing    gomatrixserverlib.KeyRing
-}
-
-// joinRoomByID joins a room by room ID
-func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse {
-	// A client should only join a room by room ID when it has an invite
-	// to the room. If the server is already in the room then we can
-	// lookup the invite and process the request as a normal state event.
-	// If the server is not in the room the we will need to look up the
-	// remote server the invite came from in order to request a join event
-	// from that server.
-	queryReq := roomserverAPI.QueryInvitesForUserRequest{
-		RoomID: roomID, TargetUserID: r.userID,
-	}
-	var queryRes roomserverAPI.QueryInvitesForUserResponse
-	if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil {
-		util.GetLogger(r.req.Context()).WithError(err).Error("r.queryAPI.QueryInvitesForUser failed")
-		return jsonerror.InternalServerError()
-	}
-
-	servers := []gomatrixserverlib.ServerName{}
-	seenInInviterIDs := map[gomatrixserverlib.ServerName]bool{}
-	for _, userID := range queryRes.InviteSenderUserIDs {
-		_, domain, err := gomatrixserverlib.SplitID('@', userID)
-		if err != nil {
-			util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-			return jsonerror.InternalServerError()
-		}
-		if !seenInInviterIDs[domain] {
-			servers = append(servers, domain)
-			seenInInviterIDs[domain] = true
-		}
-	}
-
-	// Also add the domain extracted from the roomID as a last resort to join
-	// in case the client is erroneously trying to join by ID without an invite
-	// or all previous attempts at domains extracted from the inviter IDs fail
-	// Note: It's no guarantee we'll succeed because a room isn't bound to the domain in its ID
-	_, domain, err := gomatrixserverlib.SplitID('!', roomID)
-	if err != nil {
-		util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
-	}
-	if domain != r.cfg.Matrix.ServerName && !seenInInviterIDs[domain] {
-		servers = append(servers, domain)
-	}
-
-	return r.joinRoomUsingServers(roomID, servers)
-
-}
-
-// joinRoomByAlias joins a room using a room alias.
-func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse {
-	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
-		}
-	}
-	if domain == r.cfg.Matrix.ServerName {
-		queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
-		var queryRes roomserverAPI.GetRoomIDForAliasResponse
-		if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil {
-			util.GetLogger(r.req.Context()).WithError(err).Error("r.aliasAPI.GetRoomIDForAlias failed")
-			return jsonerror.InternalServerError()
-		}
-
-		if len(queryRes.RoomID) > 0 {
-			return r.joinRoomUsingServers(queryRes.RoomID, []gomatrixserverlib.ServerName{r.cfg.Matrix.ServerName})
-		}
-		// If the response doesn't contain a non-empty string, return an error
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Room alias " + roomAlias + " not found."),
-		}
-	}
-	// If the room isn't local, use federation to join
-	return r.joinRoomByRemoteAlias(domain, roomAlias)
-}
-
-func (r joinRoomReq) joinRoomByRemoteAlias(
-	domain gomatrixserverlib.ServerName, roomAlias string,
-) util.JSONResponse {
-	resp, err := r.federation.LookupRoomAlias(r.req.Context(), domain, roomAlias)
-	if err != nil {
-		switch x := err.(type) {
-		case gomatrix.HTTPError:
-			if x.Code == http.StatusNotFound {
-				return util.JSONResponse{
-					Code: http.StatusNotFound,
-					JSON: jsonerror.NotFound("Room alias not found"),
-				}
-			}
-		}
-		util.GetLogger(r.req.Context()).WithError(err).Error("r.federation.LookupRoomAlias failed")
-		return jsonerror.InternalServerError()
-	}
-
-	return r.joinRoomUsingServers(resp.RoomID, resp.Servers)
-}
-
-func (r joinRoomReq) writeToBuilder(eb *gomatrixserverlib.EventBuilder, roomID string) error {
-	eb.Type = "m.room.member"
-
-	err := eb.SetContent(r.content)
-	if err != nil {
-		return err
-	}
-
-	err = eb.SetUnsigned(struct{}{})
-	if err != nil {
-		return err
-	}
-
-	eb.Sender = r.userID
-	eb.StateKey = &r.userID
-	eb.RoomID = roomID
-	eb.Redacts = ""
-
-	return nil
-}
-
-func (r joinRoomReq) joinRoomUsingServers(
-	roomID string, servers []gomatrixserverlib.ServerName,
-) util.JSONResponse {
-	var eb gomatrixserverlib.EventBuilder
-	err := r.writeToBuilder(&eb, roomID)
-	if err != nil {
-		util.GetLogger(r.req.Context()).WithError(err).Error("r.writeToBuilder failed")
-		return jsonerror.InternalServerError()
-	}
-
-	queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{}
-	event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes)
-	if err == nil {
-		// If we have successfully built an event at this point then we can
-		// assert that the room is a local room, as BuildEvent was able to
-		// add prev_events etc successfully.
-		if _, err = r.producer.SendEvents(
-			r.req.Context(),
-			[]gomatrixserverlib.HeaderedEvent{
-				(*event).Headered(queryRes.RoomVersion),
-			},
-			r.cfg.Matrix.ServerName,
-			nil,
-		); err != nil {
-			util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEvents failed")
-			return jsonerror.InternalServerError()
-		}
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: struct {
-				RoomID string `json:"room_id"`
-			}{roomID},
-		}
-	}
-
-	// Otherwise, if we've reached here, then we haven't been able to populate
-	// prev_events etc for the room, therefore the room is probably federated.
-
-	// TODO: This needs to be re-thought, as in the case of an invite, the room
-	// will exist in the database in roomserver_rooms but won't have any state
-	// events, therefore this below check fails.
-	if err != common.ErrRoomNoExists {
-		util.GetLogger(r.req.Context()).WithError(err).Error("common.BuildEvent failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if len(servers) == 0 {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("No candidate servers found for room"),
-		}
-	}
-
-	var lastErr error
-	for _, server := range servers {
-		var response *util.JSONResponse
-		response, lastErr = r.joinRoomUsingServer(roomID, server)
-		if lastErr != nil {
-			// There was a problem talking to one of the servers.
-			util.GetLogger(r.req.Context()).WithError(lastErr).WithField("server", server).Warn("Failed to join room using server")
-			// Try the next server.
-			if r.req.Context().Err() != nil {
-				// The request context has expired so don't bother trying any
-				// more servers - they will immediately fail due to the expired
-				// context.
-				break
-			} else {
-				// The request context hasn't expired yet so try the next server.
-				continue
-			}
-		}
-		return *response
-	}
-
-	// Every server we tried to join through resulted in an error.
-	// We return the error from the last server.
-
-	// TODO: Generate the correct HTTP status code for all different
-	// kinds of errors that could have happened.
-	// The possible errors include:
-	//   1) We can't connect to the remote servers.
-	//   2) None of the servers we could connect to think we are allowed
-	//	    to join the room.
-	//   3) The remote server returned something invalid.
-	//   4) We couldn't fetch the public keys needed to verify the
-	//      signatures on the state events.
-	//   5) ...
-	util.GetLogger(r.req.Context()).WithError(lastErr).Error("failed to join through any server")
-	return jsonerror.InternalServerError()
-}
-
-// joinRoomUsingServer tries to join a remote room using a given matrix server.
-// If there was a failure communicating with the server or the response from the
-// server was invalid this returns an error.
-// Otherwise this returns a JSONResponse.
-func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) {
-	// Ask the room server for information about room versions.
-	var request api.QueryRoomVersionCapabilitiesRequest
-	var response api.QueryRoomVersionCapabilitiesResponse
-	if err := r.queryAPI.QueryRoomVersionCapabilities(r.req.Context(), &request, &response); err != nil {
-		return nil, err
-	}
-	var supportedVersions []gomatrixserverlib.RoomVersion
-	for version := range response.AvailableRoomVersions {
-		supportedVersions = append(supportedVersions, version)
-	}
-	respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, supportedVersions)
-	if err != nil {
-		// TODO: Check if the user was not allowed to join the room.
-		return nil, fmt.Errorf("r.federation.MakeJoin: %w", err)
-	}
-
-	// Set all the fields to be what they should be, this should be a no-op
-	// but it's possible that the remote server returned us something "odd"
-	err = r.writeToBuilder(&respMakeJoin.JoinEvent, roomID)
-	if err != nil {
-		return nil, fmt.Errorf("r.writeToBuilder: %w", err)
-	}
-
-	if respMakeJoin.RoomVersion == "" {
-		respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1
-	}
-	if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion(
-				fmt.Sprintf("Room version '%s' is not supported", respMakeJoin.RoomVersion),
-			),
-		}, nil
-	}
-
-	event, err := respMakeJoin.JoinEvent.Build(
-		r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID,
-		r.cfg.Matrix.PrivateKey, respMakeJoin.RoomVersion,
-	)
-	if err != nil {
-		return nil, fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err)
-	}
-
-	respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event, respMakeJoin.RoomVersion)
-	if err != nil {
-		return nil, fmt.Errorf("r.federation.SendJoin: %w", err)
-	}
-
-	if err = r.checkSendJoinResponse(event, server, respMakeJoin, respSendJoin); err != nil {
-		return nil, err
-	}
-
-	util.GetLogger(r.req.Context()).WithFields(logrus.Fields{
-		"room_id":          roomID,
-		"num_auth_events":  len(respSendJoin.AuthEvents),
-		"num_state_events": len(respSendJoin.StateEvents),
-	}).Info("Room join signature and auth verification passed")
-
-	if err = r.producer.SendEventWithState(
-		r.req.Context(),
-		gomatrixserverlib.RespState(respSendJoin.RespState),
-		event.Headered(respMakeJoin.RoomVersion),
-	); err != nil {
-		util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState")
-	}
-
-	return &util.JSONResponse{
 		Code: http.StatusOK,
 		// TODO: Put the response struct somewhere common.
 		JSON: struct {
 			RoomID string `json:"room_id"`
-		}{roomID},
+		}{joinReq.RoomIDOrAlias},
-	}, nil
+	}
-}
-
-// checkSendJoinResponse checks that all of the signatures are correct
-// and that the join is allowed by the supplied state.
-func (r joinRoomReq) checkSendJoinResponse(
-	event gomatrixserverlib.Event,
-	server gomatrixserverlib.ServerName,
-	respMakeJoin gomatrixserverlib.RespMakeJoin,
-	respSendJoin gomatrixserverlib.RespSendJoin,
-) error {
-	// A list of events that we have retried, if they were not included in
-	// the auth events supplied in the send_join.
-	retries := map[string]bool{}
-
-retryCheck:
-	// TODO: Can we expand Check here to return a list of missing auth
-	// events rather than failing one at a time?
-	if err := respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil {
-		switch e := err.(type) {
-		case gomatrixserverlib.MissingAuthEventError:
-			// Check that we haven't already retried for this event, prevents
-			// us from ending up in endless loops
-			if !retries[e.AuthEventID] {
-				// Ask the server that we're talking to right now for the event
-				tx, txerr := r.federation.GetEvent(r.req.Context(), server, e.AuthEventID)
-				if txerr != nil {
-					return fmt.Errorf("r.federation.GetEvent: %w", txerr)
-				}
-				// For each event returned, add it to the auth events.
-				for _, pdu := range tx.PDUs {
-					ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion)
-					if everr != nil {
-						return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr)
-					}
-					respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev)
-				}
-				// Mark the event as retried and then give the check another go.
-				retries[e.AuthEventID] = true
-				goto retryCheck
-			}
-			return fmt.Errorf("respSendJoin (after retries): %w", e)
-		default:
-			return fmt.Errorf("respSendJoin: %w", err)
-		}
-	}
-	return nil
 }

clientapi/routing/membership.go

@@ -40,15 +40,17 @@ var errMissingUserID = errors.New("'user_id' must be supplied")
 
 // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
 // by building a m.room.member event then sending it to the room server
+// TODO: Can we improve the cyclo count here? Separate code paths for invites?
+// nolint:gocyclo
 func SendMembership(
 	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
 	roomID string, membership string, cfg *config.Dendrite,
-	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 	producer *producers.RoomserverProducer,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -69,7 +71,7 @@ func SendMembership(
 	}
 
 	inviteStored, jsonErrResp := checkAndProcessThreepid(
-		req, device, &body, cfg, queryAPI, accountDB, producer,
+		req, device, &body, cfg, rsAPI, accountDB, producer,
 		membership, roomID, evTime,
 	)
 	if jsonErrResp != nil {
@@ -87,7 +89,7 @@ func SendMembership(
 	}
 
 	event, err := buildMembershipEvent(
-		req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, queryAPI, asAPI,
+		req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, rsAPI, asAPI,
 	)
 	if err == errMissingUserID {
 		return util.JSONResponse{
@@ -104,23 +106,39 @@ func SendMembership(
 		return jsonerror.InternalServerError()
 	}
 
-	if _, err := producer.SendEvents(
-		req.Context(),
-		[]gomatrixserverlib.HeaderedEvent{(*event).Headered(verRes.RoomVersion)},
-		cfg.Matrix.ServerName,
-		nil,
-	); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed")
-		return jsonerror.InternalServerError()
-	}
-
 	var returnData interface{} = struct{}{}
 
+	switch membership {
+	case gomatrixserverlib.Invite:
+		// Invites need to be handled specially
+		err = producer.SendInvite(
+			req.Context(),
+			event.Headered(verRes.RoomVersion),
+			nil, // ask the roomserver to draw up invite room state for us
+			cfg.Matrix.ServerName,
+			nil,
+		)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("producer.SendInvite failed")
+			return jsonerror.InternalServerError()
+		}
+	case gomatrixserverlib.Join:
 		// The join membership requires the room id to be sent in the response
-	if membership == gomatrixserverlib.Join {
 		returnData = struct {
 			RoomID string `json:"room_id"`
 		}{roomID}
+		fallthrough
+	default:
+		_, err = producer.SendEvents(
+			req.Context(),
+			[]gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)},
+			cfg.Matrix.ServerName,
+			nil,
+		)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed")
+			return jsonerror.InternalServerError()
+		}
 	}
 
 	return util.JSONResponse{
@@ -135,7 +153,7 @@ func buildMembershipEvent(
 	device *authtypes.Device,
 	membership, roomID string,
 	cfg *config.Dendrite, evTime time.Time,
-	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*gomatrixserverlib.Event, error) {
 	stateKey, reason, err := getMembershipStateKey(body, device, membership)
 	if err != nil {
@@ -170,7 +188,7 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
-	return common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil)
+	return common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
 }
 
 // loadProfile lookups the profile of a given user from the database and returns
@@ -230,7 +248,7 @@ func checkAndProcessThreepid(
 	device *authtypes.Device,
 	body *threepid.MembershipRequest,
 	cfg *config.Dendrite,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 	accountDB accounts.Database,
 	producer *producers.RoomserverProducer,
 	membership, roomID string,
@@ -238,7 +256,7 @@ func checkAndProcessThreepid(
 ) (inviteStored bool, errRes *util.JSONResponse) {
 
 	inviteStored, err := threepid.CheckAndProcessInvite(
-		req.Context(), device, body, cfg, queryAPI, accountDB, producer,
+		req.Context(), device, body, cfg, rsAPI, accountDB, producer,
 		membership, roomID, evTime,
 	)
 	if err == threepid.ErrMissingParameter {

clientapi/routing/memberships.go

@@ -39,7 +39,7 @@ type getJoinedRoomsResponse struct {
 func GetMemberships(
 	req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool,
 	_ *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	queryReq := api.QueryMembershipsForRoomRequest{
 		JoinedOnly: joinedOnly,
@@ -47,8 +47,8 @@ func GetMemberships(
 		Sender:     device.UserID,
 	}
 	var queryRes api.QueryMembershipsForRoomResponse
-	if err := queryAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil {
+	if err := rsAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryMembershipsForRoom failed")
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed")
 		return jsonerror.InternalServerError()
 	}
 

clientapi/routing/profile.go

@@ -94,7 +94,7 @@ func GetAvatarURL(
 func SetAvatarURL(
 	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
 	userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite,
-	rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
+	rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -154,7 +154,7 @@ func SetAvatarURL(
 	}
 
 	events, err := buildMembershipEvents(
-		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+		req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
 	)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed")
@@ -208,7 +208,7 @@ func GetDisplayName(
 func SetDisplayName(
 	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
 	userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite,
-	rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
+	rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -268,7 +268,7 @@ func SetDisplayName(
 	}
 
 	events, err := buildMembershipEvents(
-		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+		req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
 	)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed")
@@ -337,14 +337,14 @@ func buildMembershipEvents(
 	ctx context.Context,
 	memberships []authtypes.Membership,
 	newProfile authtypes.Profile, userID string, cfg *config.Dendrite,
-	evTime time.Time, queryAPI api.RoomserverQueryAPI,
+	evTime time.Time, rsAPI api.RoomserverInternalAPI,
 ) ([]gomatrixserverlib.HeaderedEvent, error) {
 	evs := []gomatrixserverlib.HeaderedEvent{}
 
 	for _, membership := range memberships {
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
-		if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
+		if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
 			return []gomatrixserverlib.HeaderedEvent{}, err
 		}
 
@@ -366,7 +366,7 @@ func buildMembershipEvents(
 			return nil, err
 		}
 
-		event, err := common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil)
+		event, err := common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
 		if err != nil {
 			return nil, err
 		}

clientapi/routing/routing.go

@@ -49,8 +49,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable"
 func Setup(
 	apiMux *mux.Router, cfg *config.Dendrite,
 	producer *producers.RoomserverProducer,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	accountDB accounts.Database,
 	deviceDB devices.Database,
@@ -60,7 +59,7 @@ func Setup(
 	syncProducer *producers.SyncAPIProducer,
 	eduProducer *producers.EDUServerProducer,
 	transactionsCache *transactions.Cache,
-	federationSender federationSenderAPI.FederationSenderQueryAPI,
+	federationSender federationSenderAPI.FederationSenderInternalAPI,
 ) {
 
 	apiMux.Handle("/_matrix/client/versions",
@@ -91,7 +90,7 @@ func Setup(
 
 	r0mux.Handle("/createRoom",
 		common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI, asAPI)
+			return CreateRoom(req, device, cfg, producer, accountDB, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
@@ -101,7 +100,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return JoinRoomByIDOrAlias(
-				req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB,
+				req, device, rsAPI, vars["roomIDOrAlias"],
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -117,7 +116,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer)
+			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI, producer)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
@@ -126,7 +125,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil)
+			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, producer, nil)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
@@ -137,7 +136,7 @@ func Setup(
 			}
 			txnID := vars["txnID"]
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID,
-				nil, cfg, queryAPI, producer, transactionsCache)
+				nil, cfg, rsAPI, producer, transactionsCache)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/event/{eventID}",
@@ -146,7 +145,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing)
+			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -155,7 +154,7 @@ func Setup(
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateRequest(req.Context(), queryAPI, vars["roomID"])
+		return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
@@ -163,7 +162,7 @@ func Setup(
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], "")
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "")
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
@@ -171,7 +170,7 @@ func Setup(
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], vars["stateKey"])
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
@@ -186,7 +185,7 @@ func Setup(
 			if strings.HasSuffix(eventType, "/") {
 				eventType = eventType[:len(eventType)-1]
 			}
-			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil)
+			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, producer, nil)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
@@ -197,7 +196,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			stateKey := vars["stateKey"]
-			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil)
+			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, producer, nil)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
@@ -219,7 +218,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return DirectoryRoom(req, vars["roomAlias"], federation, cfg, aliasAPI, federationSender)
+			return DirectoryRoom(req, vars["roomAlias"], federation, cfg, rsAPI, federationSender)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -229,7 +228,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetLocalAlias(req, device, vars["roomAlias"], cfg, aliasAPI)
+			return SetLocalAlias(req, device, vars["roomAlias"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
@@ -239,7 +238,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI)
+			return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
 
@@ -353,7 +352,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI)
+			return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
@@ -375,7 +374,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI)
+			return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
@@ -488,7 +487,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI)
+			return GetMemberships(req, device, vars["roomID"], false, cfg, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -498,7 +497,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI)
+			return GetMemberships(req, device, vars["roomID"], true, cfg, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -602,7 +601,7 @@ func Setup(
 
 	r0mux.Handle("/capabilities",
 		common.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			return GetCapabilities(req, queryAPI)
+			return GetCapabilities(req, rsAPI)
 		}),
 	).Methods(http.MethodGet)
 }

clientapi/routing/sendevent.go

@@ -45,13 +45,13 @@ func SendEvent(
 	device *authtypes.Device,
 	roomID, eventType string, txnID, stateKey *string,
 	cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	producer *producers.RoomserverProducer,
 	txnCache *transactions.Cache,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -65,7 +65,7 @@ func SendEvent(
 		}
 	}
 
-	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, queryAPI)
+	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI)
 	if resErr != nil {
 		return *resErr
 	}
@@ -115,7 +115,7 @@ func generateSendEvent(
 	device *authtypes.Device,
 	roomID, eventType string, stateKey *string,
 	cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	// parse the incoming http request
 	userID := device.UserID
@@ -148,7 +148,7 @@ func generateSendEvent(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, queryAPI, &queryRes)
+	e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes)
 	if err == common.ErrRoomNoExists {
 		return nil, &util.JSONResponse{
 			Code: http.StatusNotFound,

clientapi/routing/state.go

@@ -40,7 +40,7 @@ type stateEventInStateResp struct {
 // TODO: Check if the user is in the room. If not, check if the room's history
 // is publicly visible. Current behaviour is returning an empty array if the
 // user cannot see the room's history.
-func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string) util.JSONResponse {
+func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 	stateReq := api.QueryLatestEventsAndStateRequest{
@@ -48,7 +48,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI
 	}
 	stateRes := api.QueryLatestEventsAndStateResponse{}
 
-	if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
+	if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
 		return jsonerror.InternalServerError()
 	}
@@ -98,7 +98,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI
 // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current
 // state to see if there is an event with that type and state key, if there
 // is then (by default) we return the content, otherwise a 404.
-func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string, evType, stateKey string) util.JSONResponse {
+func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string, evType, stateKey string) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 	util.GetLogger(ctx).WithFields(log.Fields{
@@ -118,7 +118,7 @@ func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQuer
 	}
 	stateRes := api.QueryLatestEventsAndStateResponse{}
 
-	if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
+	if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/threepid/invites.go

@@ -87,7 +87,7 @@ var (
 func CheckAndProcessInvite(
 	ctx context.Context,
 	device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI, db accounts.Database,
+	rsAPI api.RoomserverInternalAPI, db accounts.Database,
 	producer *producers.RoomserverProducer, membership string, roomID string,
 	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
@@ -112,7 +112,7 @@ func CheckAndProcessInvite(
 		// "m.room.third_party_invite" have to be emitted from the data in
 		// storeInviteRes.
 		err = emit3PIDInviteEvent(
-			ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer, evTime,
+			ctx, body, storeInviteRes, device, roomID, cfg, rsAPI, producer, evTime,
 		)
 		inviteStoredOnIDServer = err == nil
 
@@ -331,7 +331,7 @@ func emit3PIDInviteEvent(
 	ctx context.Context,
 	body *MembershipRequest, res *idServerStoreInviteResponse,
 	device *authtypes.Device, roomID string, cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer,
+	rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer,
 	evTime time.Time,
 ) error {
 	builder := &gomatrixserverlib.EventBuilder{
@@ -354,7 +354,7 @@ func emit3PIDInviteEvent(
 	}
 
 	queryRes := api.QueryLatestEventsAndStateResponse{}
-	event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, &queryRes)
+	event, err := common.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes)
 	if err != nil {
 		return err
 	}

cmd/create-account/main.go

@@ -63,7 +63,7 @@ func main() {
 
 	serverName := gomatrixserverlib.ServerName(*serverNameStr)
 
-	accountDB, err := accounts.NewDatabase(*database, serverName)
+	accountDB, err := accounts.NewDatabase(*database, nil, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)
@@ -78,7 +78,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	deviceDB, err := devices.NewDatabase(*database, serverName)
+	deviceDB, err := devices.NewDatabase(*database, nil, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)

cmd/dendrite-appservice-server/main.go

@@ -28,11 +28,11 @@ func main() {
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
 	federation := base.CreateFederationClient()
-	alias, _, query := base.CreateHTTPRoomserverAPIs()
+	rsAPI := base.CreateHTTPRoomserverAPIs()
 	cache := transactions.New()
 
 	appservice.SetupAppServiceAPIComponent(
-		base, accountDB, deviceDB, federation, alias, query, cache,
+		base, accountDB, deviceDB, federation, rsAPI, cache,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI))

cmd/dendrite-client-api-server/main.go

@@ -36,13 +36,14 @@ func main() {
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
 	asQuery := base.CreateHTTPAppServiceAPIs()
-	alias, input, query := base.CreateHTTPRoomserverAPIs()
+	rsAPI := base.CreateHTTPRoomserverAPIs()
-	fedSenderAPI := base.CreateHTTPFederationSenderAPIs()
+	fsAPI := base.CreateHTTPFederationSenderAPIs()
+	rsAPI.SetFederationSenderAPI(fsAPI)
 	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB, federation, &keyRing,
-		alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
+		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))

cmd/dendrite-demo-libp2p/main.go

@@ -56,6 +56,7 @@ func createKeyDB(
 ) keydb.Database {
 	db, err := keydb.NewDatabase(
 		string(base.Base.Cfg.Database.ServerKey),
+		base.Base.Cfg.DbProperties(),
 		base.Base.Cfg.Matrix.ServerName,
 		base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
 		base.Base.Cfg.Matrix.KeyID,
@@ -148,27 +149,34 @@ func main() {
 	federation := createFederationClient(base)
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	alias, input, query := roomserver.SetupRoomServerComponent(&base.Base)
+	rsAPI := roomserver.SetupRoomServerComponent(
-	eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New())
+		&base.Base, keyRing, federation,
-	asQuery := appservice.SetupAppServiceAPIComponent(
-		&base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(),
 	)
-	fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query)
+	eduInputAPI := eduserver.SetupEDUServerComponent(
+		&base.Base, cache.New(),
+	)
+	asAPI := appservice.SetupAppServiceAPIComponent(
+		&base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
+	)
+	fsAPI := federationsender.SetupFederationSenderComponent(
+		&base.Base, federation, rsAPI, &keyRing,
+	)
+	rsAPI.SetFederationSenderAPI(fsAPI)
 
 	clientapi.SetupClientAPIComponent(
 		&base.Base, deviceDB, accountDB,
-		federation, &keyRing, alias, input, query,
+		federation, &keyRing, rsAPI,
-		eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
+		eduInputAPI, asAPI, transactions.New(), fsAPI,
 	)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-	federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
+	federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB)
 	publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, query, federation, nil) // Check this later
+	publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later
-	syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, query, federation, &cfg)
+	syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.Base.APIMux)
 

cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go

@@ -45,7 +45,7 @@ type PublicRoomsServerDatabase struct {
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
 func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil)
 	if err != nil {
 		return nil, err
 	}

cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go

@@ -48,7 +48,7 @@ type PublicRoomsServerDatabase struct {
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
 func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil)
 	if err != nil {
 		return nil, err
 	}

cmd/dendrite-federation-api-server/main.go

@@ -32,17 +32,18 @@ func main() {
 	deviceDB := base.CreateDeviceDB()
 	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
-	federationSender := base.CreateHTTPFederationSenderAPIs()
+	fsAPI := base.CreateHTTPFederationSenderAPIs()
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	alias, input, query := base.CreateHTTPRoomserverAPIs()
+	rsAPI := base.CreateHTTPRoomserverAPIs()
-	asQuery := base.CreateHTTPAppServiceAPIs()
+	asAPI := base.CreateHTTPAppServiceAPIs()
+	rsAPI.SetFederationSenderAPI(fsAPI)
 	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 
 	federationapi.SetupFederationAPIComponent(
 		base, accountDB, deviceDB, federation, &keyRing,
-		alias, input, query, asQuery, federationSender, eduProducer,
+		rsAPI, asAPI, fsAPI, eduProducer,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))

cmd/dendrite-federation-sender-server/main.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/common/keydb"
 	"github.com/matrix-org/dendrite/federationsender"
 )
 
@@ -25,12 +26,13 @@ func main() {
 	defer base.Close() // nolint: errcheck
 
 	federation := base.CreateFederationClient()
-
+	keyDB := base.CreateKeyDB()
-	_, _, query := base.CreateHTTPRoomserverAPIs()
+	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
-
+	rsAPI := base.CreateHTTPRoomserverAPIs()
-	federationsender.SetupFederationSenderComponent(
+	fsAPI := federationsender.SetupFederationSenderComponent(
-		base, federation, query,
+		base, federation, rsAPI, &keyRing,
 	)
+	rsAPI.SetFederationSenderAPI(fsAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender))
 

cmd/dendrite-monolith-server/main.go

@@ -57,27 +57,34 @@ func main() {
 	federation := base.CreateFederationClient()
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	alias, input, query := roomserver.SetupRoomServerComponent(base)
+	rsAPI := roomserver.SetupRoomServerComponent(
-	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
+		base, keyRing, federation,
-	asQuery := appservice.SetupAppServiceAPIComponent(
-		base, accountDB, deviceDB, federation, alias, query, transactions.New(),
 	)
-	fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query)
+	eduInputAPI := eduserver.SetupEDUServerComponent(
+		base, cache.New(),
+	)
+	asAPI := appservice.SetupAppServiceAPIComponent(
+		base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
+	)
+	fsAPI := federationsender.SetupFederationSenderComponent(
+		base, federation, rsAPI, &keyRing,
+	)
+	rsAPI.SetFederationSenderAPI(fsAPI)
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB,
-		federation, &keyRing, alias, input, query,
+		federation, &keyRing, rsAPI,
-		eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
+		eduInputAPI, asAPI, transactions.New(), fsAPI,
 	)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
+	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, nil)
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil)
-	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
+	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)
 

cmd/dendrite-public-rooms-api-server/main.go

@@ -28,12 +28,15 @@ func main() {
 
 	deviceDB := base.CreateDeviceDB()
 
-	_, _, query := base.CreateHTTPRoomserverAPIs()
+	fsAPI := base.CreateHTTPFederationSenderAPIs()
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
+	rsAPI := base.CreateHTTPRoomserverAPIs()
+	rsAPI.SetFederationSenderAPI(fsAPI)
+
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, nil, nil)
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, nil, nil)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI))
 

cmd/dendrite-room-server/main.go

@@ -18,6 +18,7 @@ import (
 	_ "net/http/pprof"
 
 	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/common/keydb"
 	"github.com/matrix-org/dendrite/roomserver"
 )
 
@@ -25,8 +26,13 @@ func main() {
 	cfg := basecomponent.ParseFlags()
 	base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI")
 	defer base.Close() // nolint: errcheck
+	keyDB := base.CreateKeyDB()
+	federation := base.CreateFederationClient()
+	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	roomserver.SetupRoomServerComponent(base)
+	fsAPI := base.CreateHTTPFederationSenderAPIs()
+	rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation)
+	rsAPI.SetFederationSenderAPI(fsAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer))
 

cmd/dendrite-sync-api-server/main.go

@@ -28,9 +28,9 @@ func main() {
 	accountDB := base.CreateAccountsDB()
 	federation := base.CreateFederationClient()
 
-	_, _, query := base.CreateHTTPRoomserverAPIs()
+	rsAPI := base.CreateHTTPRoomserverAPIs()
 
-	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
+	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI))
 

cmd/dendritejs/main.go

@@ -123,27 +123,28 @@ func main() {
 	}
 	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node)
 
-	alias, input, query := roomserver.SetupRoomServerComponent(base)
+	rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation)
 	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
 	asQuery := appservice.SetupAppServiceAPIComponent(
-		base, accountDB, deviceDB, federation, alias, query, transactions.New(),
+		base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
 	)
-	fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query)
+	fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing)
+	rsAPI.SetFederationSenderAPI(fedSenderAPI)
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB,
-		federation, &keyRing, alias, input, query,
+		federation, &keyRing, rsAPI,
 		eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
 	)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
+	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
 	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, p2pPublicRoomProvider)
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider)
-	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
+	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)
 

cmd/kafka-producer/main.go

@@ -21,7 +21,7 @@ import (
 	"os"
 	"strings"
 
-	sarama "gopkg.in/Shopify/sarama.v1"
+	sarama "github.com/Shopify/sarama"
 )
 
 const usage = `Usage: %s

cmd/roomserver-integration-tests/main.go

@@ -209,7 +209,7 @@ func writeToRoomServer(input []string, roomserverURL string) error {
 			return err
 		}
 	}
-	x, err := api.NewRoomserverInputAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP})
+	x, err := api.NewRoomserverInternalAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}, nil)
 	if err != nil {
 		return err
 	}
@@ -225,7 +225,7 @@ func writeToRoomServer(input []string, roomserverURL string) error {
 // Once those messages have been written it runs the checkQueries function passing
 // a api.RoomserverQueryAPI client. The caller can use this function to check the
 // behaviour of the query API.
-func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) {
+func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverInternalAPI)) {
 	dir, err := ioutil.TempDir("", "room-server-test")
 	if err != nil {
 		panic(err)
@@ -276,7 +276,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 	cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)}
 
 	gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() {
-		queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache)
+		queryAPI, _ := api.NewRoomserverInternalAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache)
 		checkQueries(queryAPI)
 	})
 	if err != nil {
@@ -410,7 +410,7 @@ func main() {
 		}}`,
 	}
 
-	testRoomserver(input, want, func(q api.RoomserverQueryAPI) {
+	testRoomserver(input, want, func(q api.RoomserverInternalAPI) {
 		var response api.QueryLatestEventsAndStateResponse
 		if err := q.QueryLatestEventsAndState(
 			context.Background(),

common/basecomponent/base.go

@@ -33,8 +33,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/common"
 
+	"github.com/Shopify/sarama"
 	"github.com/gorilla/mux"
-	sarama "gopkg.in/Shopify/sarama.v1"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/common/config"
@@ -119,24 +119,12 @@ func (b *BaseDendrite) CreateHTTPAppServiceAPIs() appserviceAPI.AppServiceQueryA
 
 // CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI for hitting
 // the roomserver over HTTP.
-func (b *BaseDendrite) CreateHTTPRoomserverAPIs() (
+func (b *BaseDendrite) CreateHTTPRoomserverAPIs() roomserverAPI.RoomserverInternalAPI {
-	roomserverAPI.RoomserverAliasAPI,
+	rsAPI, err := roomserverAPI.NewRoomserverInternalAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache)
-	roomserverAPI.RoomserverInputAPI,
-	roomserverAPI.RoomserverQueryAPI,
-) {
-	alias, err := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient)
 	if err != nil {
-		logrus.WithError(err).Panic("NewRoomserverAliasAPIHTTP failed")
+		logrus.WithError(err).Panic("NewRoomserverInternalAPIHTTP failed", b.httpClient)
 	}
-	input, err := roomserverAPI.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient)
+	return rsAPI
-	if err != nil {
-		logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient)
-	}
-	query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache)
-	if err != nil {
-		logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient)
-	}
-	return alias, input, query
 }
 
 // CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU
@@ -149,12 +137,12 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI
 	return e
 }
 
-// CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting
+// CreateHTTPFederationSenderAPIs returns FederationSenderInternalAPI for hitting
 // the federation sender over HTTP
-func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI {
+func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderInternalAPI {
-	f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient)
+	f, err := federationSenderAPI.NewFederationSenderInternalAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient)
 	if err != nil {
-		logrus.WithError(err).Panic("NewFederationSenderQueryAPIHTTP failed", b.httpClient)
+		logrus.WithError(err).Panic("NewFederationSenderInternalAPIHTTP failed", b.httpClient)
 	}
 	return f
 }
@@ -162,7 +150,7 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede
 // CreateDeviceDB creates a new instance of the device database. Should only be
 // called once per component.
 func (b *BaseDendrite) CreateDeviceDB() devices.Database {
-	db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.Matrix.ServerName)
+	db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to devices db")
 	}
@@ -173,7 +161,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database {
 // CreateAccountsDB creates a new instance of the accounts database. Should only
 // be called once per component.
 func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
-	db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.Matrix.ServerName)
+	db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to accounts db")
 	}
@@ -186,6 +174,7 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
 func (b *BaseDendrite) CreateKeyDB() keydb.Database {
 	db, err := keydb.NewDatabase(
 		string(b.Cfg.Database.ServerKey),
+		b.Cfg.DbProperties(),
 		b.Cfg.Matrix.ServerName,
 		b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
 		b.Cfg.Matrix.KeyID,
@@ -256,7 +245,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
 
 	uri, err := url.Parse(string(cfg.Database.Naffka))
 	if err != nil || uri.Scheme == "file" {
-		db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka))
+		db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka), nil)
 		if err != nil {
 			logrus.WithError(err).Panic("Failed to open naffka database")
 		}
@@ -266,7 +255,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
 			logrus.WithError(err).Panic("Failed to setup naffka database")
 		}
 	} else {
-		db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka))
+		db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil)
 		if err != nil {
 			logrus.WithError(err).Panic("Failed to open naffka database")
 		}

common/config/config.go

@@ -188,6 +188,12 @@ type Dendrite struct {
 		PublicRoomsAPI DataSource `yaml:"public_rooms_api"`
 		// The Naffka database is used internally by the naffka library, if used.
 		Naffka DataSource `yaml:"naffka,omitempty"`
+		// Maximum open connections to the DB (0 = use default, negative means unlimited)
+		MaxOpenConns int `yaml:"max_open_conns"`
+		// Maximum idle connections to the DB (0 = use default, negative means unlimited)
+		MaxIdleConns int `yaml:"max_idle_conns"`
+		// maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited)
+		ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"`
 	} `yaml:"database"`
 
 	// TURN Server Config
@@ -484,6 +490,15 @@ func (config *Dendrite) SetDefaults() {
 		defaultMaxFileSizeBytes := FileSizeBytes(10485760)
 		config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes
 	}
+
+	if config.Database.MaxIdleConns == 0 {
+		config.Database.MaxIdleConns = 2
+	}
+
+	if config.Database.MaxOpenConns == 0 {
+		config.Database.MaxOpenConns = 100
+	}
+
 }
 
 // Error returns a string detailing how many errors were contained within a
@@ -746,6 +761,33 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err
 	)
 }
 
+// MaxIdleConns returns maximum idle connections to the DB
+func (config Dendrite) MaxIdleConns() int {
+	return config.Database.MaxIdleConns
+}
+
+// MaxOpenConns returns maximum open connections to the DB
+func (config Dendrite) MaxOpenConns() int {
+	return config.Database.MaxOpenConns
+}
+
+// ConnMaxLifetime returns maximum amount of time a connection may be reused
+func (config Dendrite) ConnMaxLifetime() time.Duration {
+	return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second
+}
+
+// DbProperties functions return properties used by database/sql/DB
+type DbProperties interface {
+	MaxIdleConns() int
+	MaxOpenConns() int
+	ConnMaxLifetime() time.Duration
+}
+
+// DbProperties returns cfg as a DbProperties interface
+func (config Dendrite) DbProperties() DbProperties {
+	return config
+}
+
 // logrusLogger is a small wrapper that implements jaeger.Logger using logrus.
 type logrusLogger struct {
 	l *logrus.Logger

common/consumers.go

@@ -18,7 +18,7 @@ import (
 	"context"
 	"fmt"
 
-	sarama "gopkg.in/Shopify/sarama.v1"
+	"github.com/Shopify/sarama"
 )
 
 // A PartitionOffset is the offset into a partition of the input log.

common/events.go

@@ -39,13 +39,13 @@ var ErrRoomNoExists = errors.New("Room does not exist")
 func BuildEvent(
 	ctx context.Context,
 	builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time,
-	queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse,
+	rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse,
 ) (*gomatrixserverlib.Event, error) {
 	if queryRes == nil {
 		queryRes = &api.QueryLatestEventsAndStateResponse{}
 	}
 
-	err := AddPrevEventsToEvent(ctx, builder, queryAPI, queryRes)
+	err := AddPrevEventsToEvent(ctx, builder, rsAPI, queryRes)
 	if err != nil {
 		// This can pass through a ErrRoomNoExists to the caller
 		return nil, err
@@ -66,7 +66,7 @@ func BuildEvent(
 func AddPrevEventsToEvent(
 	ctx context.Context,
 	builder *gomatrixserverlib.EventBuilder,
-	queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse,
+	rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse,
 ) error {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
 	if err != nil {
@@ -82,8 +82,8 @@ func AddPrevEventsToEvent(
 		RoomID:       builder.RoomID,
 		StateToFetch: eventsNeeded.Tuples(),
 	}
-	if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil {
+	if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil {
-		return fmt.Errorf("queryAPI.QueryLatestEventsAndState: %w", err)
+		return fmt.Errorf("rsAPI.QueryLatestEventsAndState: %w", err)
 	}
 
 	if !queryRes.RoomExists {

common/keydb/keydb.go

@@ -21,6 +21,7 @@ import (
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/keydb/postgres"
 	"github.com/matrix-org/dendrite/common/keydb/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -29,20 +30,21 @@ import (
 // NewDatabase opens a database connection.
 func NewDatabase(
 	dataSourceName string,
+	dbProperties common.DbProperties,
 	serverName gomatrixserverlib.ServerName,
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
 	default:
-		return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
 	}
 }

common/keydb/keydb_wasm.go

@@ -20,6 +20,7 @@ import (
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/keydb/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -27,6 +28,7 @@ import (
 // NewDatabase opens a database connection.
 func NewDatabase(
 	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
 	serverName gomatrixserverlib.ServerName,
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,

common/keydb/postgres/keydb.go

@@ -21,6 +21,7 @@ import (
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -37,11 +38,12 @@ type Database struct {
 // Returns an error if there was a problem talking to the database.
 func NewDatabase(
 	dataSourceName string,
+	dbProperties common.DbProperties,
 	serverName gomatrixserverlib.ServerName,
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (*Database, error) {
-	db, err := sqlutil.Open("postgres", dataSourceName)
+	db, err := sqlutil.Open("postgres", dataSourceName, dbProperties)
 	if err != nil {
 		return nil, err
 	}

common/keydb/sqlite3/keydb.go

@@ -44,7 +44,7 @@ func NewDatabase(
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (*Database, error) {
-	db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName)
+	db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil)
 	if err != nil {
 		return nil, err
 	}

common/routing.go

@@ -24,7 +24,7 @@ import (
 func URLDecodeMapValues(vmap map[string]string) (map[string]string, error) {
 	decoded := make(map[string]string, len(vmap))
 	for key, value := range vmap {
-		decodedVal, err := url.QueryUnescape(value)
+		decodedVal, err := url.PathUnescape(value)
 		if err != nil {
 			return make(map[string]string), err
 		}

common/sql.go

@@ -18,6 +18,7 @@ import (
 	"database/sql"
 	"fmt"
 	"runtime"
+	"time"
 )
 
 // A Transaction is something that can be committed or rolledback.
@@ -99,3 +100,10 @@ func SQLiteDriverName() string {
 	}
 	return "sqlite3"
 }
+
+// DbProperties functions return properties used by database/sql/DB
+type DbProperties interface {
+	MaxIdleConns() int
+	MaxOpenConns() int
+	ConnMaxLifetime() time.Duration
+}

dendrite-config.yaml

@@ -116,6 +116,9 @@ database:
     federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable"
     appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable"
     public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable"
+    max_open_conns: 100
+    max_idle_conns: 2
+    conn_max_lifetime: -1
     # If using naffka you need to specify a naffka database
     # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable"
 

eduserver/input/input.go

@@ -18,12 +18,12 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/Shopify/sarama"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
-	"gopkg.in/Shopify/sarama.v1"
 )
 
 // EDUServerInputAPI implements api.EDUServerInputAPI

federationapi/federationapi.go

@@ -36,18 +36,16 @@ func SetupFederationAPIComponent(
 	deviceDB devices.Database,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing *gomatrixserverlib.KeyRing,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	inputAPI roomserverAPI.RoomserverInputAPI,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
-	federationSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 	eduProducer *producers.EDUServerProducer,
 ) {
-	roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI)
+	roomserverProducer := producers.NewRoomserverProducer(rsAPI)
 
 	routing.Setup(
-		base.APIMux, base.Cfg, queryAPI, aliasAPI, asAPI,
+		base.APIMux, base.Cfg, rsAPI, asAPI, roomserverProducer,
-		roomserverProducer, eduProducer, federationSenderAPI, *keyRing,
+		eduProducer, federationSenderAPI, *keyRing,
 		federation, accountsDB, deviceDB,
 	)
 }

federationapi/routing/backfill.go

@@ -33,7 +33,7 @@ import (
 func Backfill(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
@@ -69,6 +69,7 @@ func Backfill(
 
 	// Populate the request.
 	req := api.QueryBackfillRequest{
+		RoomID:            roomID,
 		EarliestEventsIDs: eIDs,
 		ServerName:        request.Origin(),
 	}
@@ -81,7 +82,7 @@ func Backfill(
 	}
 
 	// Query the roomserver.
-	if err = query.QueryBackfill(httpReq.Context(), &req, &res); err != nil {
+	if err = rsAPI.QueryBackfill(httpReq.Context(), &req, &res); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed")
 		return jsonerror.InternalServerError()
 	}
@@ -97,7 +98,10 @@ func Backfill(
 	}
 
 	var eventJSONs []json.RawMessage
-	for _, e := range gomatrixserverlib.ReverseTopologicalOrdering(evs) {
+	for _, e := range gomatrixserverlib.ReverseTopologicalOrdering(
+		evs,
+		gomatrixserverlib.TopologicalOrderByPrevEvents,
+	) {
 		eventJSONs = append(eventJSONs, e.JSON())
 	}
 

federationapi/routing/eventauth.go

@@ -25,13 +25,13 @@ import (
 func GetEventAuth(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 	eventID string,
 ) util.JSONResponse {
 	// TODO: Optimisation: we shouldn't be querying all the room state
 	// that is in state.StateEvents - we just ignore it.
-	state, err := getState(ctx, request, query, roomID, eventID)
+	state, err := getState(ctx, request, rsAPI, roomID, eventID)
 	if err != nil {
 		return *err
 	}

federationapi/routing/events.go

@@ -16,7 +16,9 @@ package routing
 
 import (
 	"context"
+	"encoding/json"
 	"net/http"
+	"time"
 
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -27,15 +29,22 @@ import (
 func GetEvent(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	eventID string,
+	origin gomatrixserverlib.ServerName,
 ) util.JSONResponse {
-	event, err := getEvent(ctx, request, query, eventID)
+	event, err := getEvent(ctx, request, rsAPI, eventID)
 	if err != nil {
 		return *err
 	}
 
-	return util.JSONResponse{Code: http.StatusOK, JSON: event}
+	return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.Transaction{
+		Origin:         origin,
+		OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
+		PDUs: []json.RawMessage{
+			event.JSON(),
+		},
+	}}
 }
 
 // getEvent returns the requested event,
@@ -43,11 +52,11 @@ func GetEvent(
 func getEvent(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	eventID string,
 ) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	var authResponse api.QueryServerAllowedToSeeEventResponse
-	err := query.QueryServerAllowedToSeeEvent(
+	err := rsAPI.QueryServerAllowedToSeeEvent(
 		ctx,
 		&api.QueryServerAllowedToSeeEventRequest{
 			EventID:    eventID,
@@ -66,7 +75,7 @@ func getEvent(
 	}
 
 	var eventsResponse api.QueryEventsByIDResponse
-	err = query.QueryEventsByID(
+	err = rsAPI.QueryEventsByID(
 		ctx,
 		&api.QueryEventsByIDRequest{EventIDs: []string{eventID}},
 		&eventsResponse,

federationapi/routing/invite.go

@@ -16,11 +16,13 @@ package routing
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/common/config"
+	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -44,6 +46,16 @@ func Invite(
 	}
 	event := inviteReq.Event()
 
+	// Check that we can accept invites for this room version.
+	if _, err := roomserverVersion.SupportedRoomVersion(inviteReq.RoomVersion()); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnsupportedRoomVersion(
+				fmt.Sprintf("Room version %q is not supported by this server.", inviteReq.RoomVersion()),
+			),
+		}
+	}
+
 	// Check that the room ID is correct.
 	if event.RoomID() != roomID {
 		return util.JSONResponse{
@@ -90,6 +102,8 @@ func Invite(
 		httpReq.Context(),
 		signedEvent.Headered(inviteReq.RoomVersion()),
 		inviteReq.InviteRoomState(),
+		event.Origin(),
+		nil,
 	); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed")
 		return jsonerror.InternalServerError()
@@ -99,6 +113,6 @@ func Invite(
 	// the other servers in the room that we have been invited.
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: gomatrixserverlib.RespInvite{Event: signedEvent},
+		JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent},
 	}
 }

federationapi/routing/join.go

@@ -33,13 +33,13 @@ func MakeJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID, userID string,
 	remoteVersions []gomatrixserverlib.RoomVersion,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.InternalServerError(),
@@ -61,9 +61,7 @@ func MakeJoin(
 	if !remoteSupportsVersion {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion(
+			JSON: jsonerror.IncompatibleRoomVersion(verRes.RoomVersion),
-				fmt.Sprintf("Joining server does not support room version %s", verRes.RoomVersion),
-			),
 		}
 	}
 
@@ -97,7 +95,7 @@ func MakeJoin(
 	queryRes := api.QueryLatestEventsAndStateResponse{
 		RoomVersion: verRes.RoomVersion,
 	}
-	event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes)
+	event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
 	if err == common.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -132,19 +130,22 @@ func MakeJoin(
 }
 
 // SendJoin implements the /send_join API
+// The make-join send-join dance makes much more sense as a single
+// flow so the cyclomatic complexity is high:
+// nolint:gocyclo
 func SendJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	producer *producers.RoomserverProducer,
 	keys gomatrixserverlib.KeyRing,
 	roomID, eventID string,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryRoomVersionForRoom failed")
+		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryRoomVersionForRoom failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.InternalServerError(),
@@ -159,6 +160,16 @@ func SendJoin(
 		}
 	}
 
+	// Check that a state key is provided.
+	if event.StateKey() == nil || (event.StateKey() != nil && *event.StateKey() == "") {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(
+				fmt.Sprintf("No state key was provided in the join event."),
+			),
+		}
+	}
+
 	// Check that the room ID is correct.
 	if event.RoomID() != roomID {
 		return util.JSONResponse{
@@ -216,14 +227,14 @@ func SendJoin(
 	// Fetch the state and auth chain. We do this before we send the events
 	// on, in case this fails.
 	var stateAndAuthChainResponse api.QueryStateAndAuthChainResponse
-	err = query.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{
+	err = rsAPI.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{
 		PrevEventIDs: event.PrevEventIDs(),
 		AuthEventIDs: event.AuthEventIDs(),
 		RoomID:       roomID,
 		ResolveState: true,
 	}, &stateAndAuthChainResponse)
 	if err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryStateAndAuthChain failed")
+		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryStateAndAuthChain failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -234,9 +245,22 @@ func SendJoin(
 		}
 	}
 
+	// Check if the user is already in the room. If they're already in then
+	// there isn't much point in sending another join event into the room.
+	alreadyJoined := false
+	for _, se := range stateAndAuthChainResponse.StateEvents {
+		if membership, merr := se.Membership(); merr == nil {
+			if se.StateKey() != nil && *se.StateKey() == *event.StateKey() {
+				alreadyJoined = (membership == "join")
+				break
+			}
+		}
+	}
+
 	// Send the events to the room server.
 	// We are responsible for notifying other servers that the user has joined
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
+	if !alreadyJoined {
 		_, err = producer.SendEvents(
 			httpReq.Context(),
 			[]gomatrixserverlib.HeaderedEvent{
@@ -249,12 +273,16 @@ func SendJoin(
 			util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed")
 			return jsonerror.InternalServerError()
 		}
+	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: map[string]interface{}{
+		JSON: gomatrixserverlib.RespSendJoin{
-			"state":      gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents),
+			RespState: gomatrixserverlib.RespState{
-			"auth_chain": gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents),
+				StateEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents),
+				AuthEvents:  gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents),
+			},
+			Origin: cfg.Matrix.ServerName,
 		},
 	}
 }

federationapi/routing/leave.go

@@ -30,7 +30,7 @@ func MakeLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID, userID string,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
@@ -61,7 +61,7 @@ func MakeLeave(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes)
+	event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
 	if err == common.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -102,7 +102,7 @@ func SendLeave(
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := producer.QueryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
+	if err := producer.RsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),

federationapi/routing/missingevents.go

@@ -34,7 +34,7 @@ type getMissingEventRequest struct {
 func GetMissingEvents(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 ) util.JSONResponse {
 	var gme getMissingEventRequest
@@ -46,7 +46,7 @@ func GetMissingEvents(
 	}
 
 	var eventsResponse api.QueryMissingEventsResponse
-	if err := query.QueryMissingEvents(
+	if err := rsAPI.QueryMissingEvents(
 		httpReq.Context(), &api.QueryMissingEventsRequest{
 			EarliestEvents: gme.EarliestEvents,
 			LatestEvents:   gme.LatestEvents,

federationapi/routing/query.go

@@ -32,8 +32,8 @@ func RoomAliasToID(
 	httpReq *http.Request,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.Dendrite,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	senderAPI federationSenderAPI.FederationSenderQueryAPI,
+	senderAPI federationSenderAPI.FederationSenderInternalAPI,
 ) util.JSONResponse {
 	roomAlias := httpReq.FormValue("room_alias")
 	if roomAlias == "" {
@@ -55,7 +55,7 @@ func RoomAliasToID(
 	if domain == cfg.Matrix.ServerName {
 		queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
 		var queryRes roomserverAPI.GetRoomIDForAliasResponse
-		if err = aliasAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil {
+		if err = rsAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil {
 			util.GetLogger(httpReq.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed")
 			return jsonerror.InternalServerError()
 		}

federationapi/routing/routing.go

@@ -44,12 +44,11 @@ const (
 func Setup(
 	apiMux *mux.Router,
 	cfg *config.Dendrite,
-	query roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	producer *producers.RoomserverProducer,
 	eduProducer *producers.EDUServerProducer,
-	federationSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 	keys gomatrixserverlib.KeyRing,
 	federation *gomatrixserverlib.FederationClient,
 	accountDB accounts.Database,
@@ -80,7 +79,7 @@ func Setup(
 			}
 			return Send(
 				httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]),
-				cfg, query, producer, eduProducer, keys, federation,
+				cfg, rsAPI, producer, eduProducer, keys, federation,
 			)
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
@@ -101,7 +100,7 @@ func Setup(
 
 	v1fedmux.Handle("/3pid/onbind", common.MakeExternalAPI("3pid_onbind",
 		func(req *http.Request) util.JSONResponse {
-			return CreateInvitesFrom3PIDInvites(req, query, asAPI, cfg, producer, federation, accountDB)
+			return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, producer, federation, accountDB)
 		},
 	)).Methods(http.MethodPost, http.MethodOptions)
 
@@ -113,7 +112,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return ExchangeThirdPartyInvite(
-				httpReq, request, vars["roomID"], query, cfg, federation, producer,
+				httpReq, request, vars["roomID"], rsAPI, cfg, federation, producer,
 			)
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
@@ -126,7 +125,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return GetEvent(
-				httpReq.Context(), request, query, vars["eventID"],
+				httpReq.Context(), request, rsAPI, vars["eventID"], cfg.Matrix.ServerName,
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -139,7 +138,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return GetState(
-				httpReq.Context(), request, query, vars["roomID"],
+				httpReq.Context(), request, rsAPI, vars["roomID"],
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -152,7 +151,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return GetStateIDs(
-				httpReq.Context(), request, query, vars["roomID"],
+				httpReq.Context(), request, rsAPI, vars["roomID"],
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -162,7 +161,7 @@ func Setup(
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars := mux.Vars(httpReq)
 			return GetEventAuth(
-				httpReq.Context(), request, query, vars["roomID"], vars["eventID"],
+				httpReq.Context(), request, rsAPI, vars["roomID"], vars["eventID"],
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -171,7 +170,7 @@ func Setup(
 		"federation_query_room_alias", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			return RoomAliasToID(
-				httpReq, federation, cfg, aliasAPI, federationSenderAPI,
+				httpReq, federation, cfg, rsAPI, federationSenderAPI,
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -222,7 +221,7 @@ func Setup(
 				remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1)
 			}
 			return MakeJoin(
-				httpReq, request, cfg, query, roomID, eventID, remoteVersions,
+				httpReq, request, cfg, rsAPI, roomID, eventID, remoteVersions,
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -237,7 +236,7 @@ func Setup(
 			roomID := vars["roomID"]
 			eventID := vars["eventID"]
 			return SendJoin(
-				httpReq, request, cfg, query, producer, keys, roomID, eventID,
+				httpReq, request, cfg, rsAPI, producer, keys, roomID, eventID,
 			)
 		},
 	)).Methods(http.MethodPut)
@@ -252,7 +251,7 @@ func Setup(
 			roomID := vars["roomID"]
 			eventID := vars["eventID"]
 			return MakeLeave(
-				httpReq, request, cfg, query, roomID, eventID,
+				httpReq, request, cfg, rsAPI, roomID, eventID,
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -286,7 +285,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetMissingEvents(httpReq, request, query, vars["roomID"])
+			return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"])
 		},
 	)).Methods(http.MethodPost)
 
@@ -297,7 +296,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return Backfill(httpReq, request, query, vars["roomID"], cfg)
+			return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg)
 		},
 	)).Methods(http.MethodGet)
 }

federationapi/routing/send.go

@@ -34,7 +34,7 @@ func Send(
 	request *gomatrixserverlib.FederationRequest,
 	txnID gomatrixserverlib.TransactionID,
 	cfg *config.Dendrite,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	producer *producers.RoomserverProducer,
 	eduProducer *producers.EDUServerProducer,
 	keys gomatrixserverlib.KeyRing,
@@ -42,7 +42,7 @@ func Send(
 ) util.JSONResponse {
 	t := txnReq{
 		context:     httpReq.Context(),
-		query:       query,
+		rsAPI:       rsAPI,
 		producer:    producer,
 		eduProducer: eduProducer,
 		keys:        keys,
@@ -99,7 +99,7 @@ func Send(
 type txnReq struct {
 	gomatrixserverlib.Transaction
 	context     context.Context
-	query       api.RoomserverQueryAPI
+	rsAPI       api.RoomserverInternalAPI
 	producer    *producers.RoomserverProducer
 	eduProducer *producers.EDUServerProducer
 	keys        gomatrixserverlib.KeyRing
@@ -120,7 +120,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
 		}
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
-		if err := t.query.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil {
+		if err := t.rsAPI.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil {
 			util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID)
 			return nil, roomNotFoundError{verReq.RoomID}
 		}
@@ -228,7 +228,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error {
 		StateToFetch: needed.Tuples(),
 	}
 	var stateResp api.QueryStateAfterEventsResponse
-	if err := t.query.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil {
+	if err := t.rsAPI.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil {
 		return err
 	}
 

federationapi/routing/state.go

@@ -27,7 +27,7 @@ import (
 func GetState(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 ) util.JSONResponse {
 	eventID, err := parseEventIDParam(request)
@@ -35,7 +35,7 @@ func GetState(
 		return *err
 	}
 
-	state, err := getState(ctx, request, query, roomID, eventID)
+	state, err := getState(ctx, request, rsAPI, roomID, eventID)
 	if err != nil {
 		return *err
 	}
@@ -47,7 +47,7 @@ func GetState(
 func GetStateIDs(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 ) util.JSONResponse {
 	eventID, err := parseEventIDParam(request)
@@ -55,7 +55,7 @@ func GetStateIDs(
 		return *err
 	}
 
-	state, err := getState(ctx, request, query, roomID, eventID)
+	state, err := getState(ctx, request, rsAPI, roomID, eventID)
 	if err != nil {
 		return *err
 	}
@@ -94,11 +94,11 @@ func parseEventIDParam(
 func getState(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	query api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 	eventID string,
 ) (*gomatrixserverlib.RespState, *util.JSONResponse) {
-	event, resErr := getEvent(ctx, request, query, eventID)
+	event, resErr := getEvent(ctx, request, rsAPI, eventID)
 	if resErr != nil {
 		return nil, resErr
 	}
@@ -110,7 +110,7 @@ func getState(
 	authEventIDs := getIDsFromEventRef(event.AuthEvents())
 
 	var response api.QueryStateAndAuthChainResponse
-	err := query.QueryStateAndAuthChain(
+	err := rsAPI.QueryStateAndAuthChain(
 		ctx,
 		&api.QueryStateAndAuthChainRequest{
 			RoomID:       roomID,

federationapi/routing/threepid.go

@@ -58,7 +58,7 @@ var (
 
 // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind
 func CreateInvitesFrom3PIDInvites(
-	req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI,
+	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite,
 	producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient,
 	accountDB accounts.Database,
@@ -72,7 +72,7 @@ func CreateInvitesFrom3PIDInvites(
 	for _, inv := range body.Invites {
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
-		if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+		if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -80,7 +80,7 @@ func CreateInvitesFrom3PIDInvites(
 		}
 
 		event, err := createInviteFrom3PIDInvite(
-			req.Context(), queryAPI, asAPI, cfg, inv, federation, accountDB,
+			req.Context(), rsAPI, asAPI, cfg, inv, federation, accountDB,
 		)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed")
@@ -108,7 +108,7 @@ func ExchangeThirdPartyInvite(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 	federation *gomatrixserverlib.FederationClient,
 	producer *producers.RoomserverProducer,
@@ -148,7 +148,7 @@ func ExchangeThirdPartyInvite(
 
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err = queryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
+	if err = rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -156,7 +156,7 @@ func ExchangeThirdPartyInvite(
 	}
 
 	// Auth and build the event from what the remote server sent us
-	event, err := buildMembershipEvent(httpReq.Context(), &builder, queryAPI, cfg)
+	event, err := buildMembershipEvent(httpReq.Context(), &builder, rsAPI, cfg)
 	if err == errNotInRoom {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -199,14 +199,14 @@ func ExchangeThirdPartyInvite(
 // Returns an error if there was a problem building the event or fetching the
 // necessary data to do so.
 func createInviteFrom3PIDInvite(
-	ctx context.Context, queryAPI roomserverAPI.RoomserverQueryAPI,
+	ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite,
 	inv invite, federation *gomatrixserverlib.FederationClient,
 	accountDB accounts.Database,
 ) (*gomatrixserverlib.Event, error) {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
 		return nil, err
 	}
 
@@ -245,7 +245,7 @@ func createInviteFrom3PIDInvite(
 		return nil, err
 	}
 
-	event, err := buildMembershipEvent(ctx, builder, queryAPI, cfg)
+	event, err := buildMembershipEvent(ctx, builder, rsAPI, cfg)
 	if err == errNotInRoom {
 		return nil, sendToRemoteServer(ctx, inv, federation, cfg, *builder)
 	}
@@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite(
 // Returns an error if something failed during the process.
 func buildMembershipEvent(
 	ctx context.Context,
-	builder *gomatrixserverlib.EventBuilder, queryAPI roomserverAPI.RoomserverQueryAPI,
+	builder *gomatrixserverlib.EventBuilder, rsAPI roomserverAPI.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 ) (*gomatrixserverlib.Event, error) {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
@@ -281,7 +281,7 @@ func buildMembershipEvent(
 		StateToFetch: eventsNeeded.Tuples(),
 	}
 	var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
-	if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil {
+	if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil {
 		return nil, err
 	}
 

federationsender/api/api.go

@@ -0,0 +1,59 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+)
+
+// FederationSenderInternalAPI is used to query information from the federation sender.
+type FederationSenderInternalAPI interface {
+	// PerformDirectoryLookup looks up a remote room ID from a room alias.
+	PerformDirectoryLookup(
+		ctx context.Context,
+		request *PerformDirectoryLookupRequest,
+		response *PerformDirectoryLookupResponse,
+	) error
+	// Query the joined hosts and the membership events accounting for their participation in a room.
+	// Note that if a server has multiple users in the room, it will have multiple entries in the returned slice.
+	// See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version.
+	QueryJoinedHostsInRoom(
+		ctx context.Context,
+		request *QueryJoinedHostsInRoomRequest,
+		response *QueryJoinedHostsInRoomResponse,
+	) error
+	// Query the server names of the joined hosts in a room.
+	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
+	// containing only the server names (without information for membership events).
+	QueryJoinedHostServerNamesInRoom(
+		ctx context.Context,
+		request *QueryJoinedHostServerNamesInRoomRequest,
+		response *QueryJoinedHostServerNamesInRoomResponse,
+	) error
+	// Handle an instruction to make_join & send_join with a remote server.
+	PerformJoin(
+		ctx context.Context,
+		request *PerformJoinRequest,
+		response *PerformJoinResponse,
+	) error
+	// Handle an instruction to make_leave & send_leave with a remote server.
+	PerformLeave(
+		ctx context.Context,
+		request *PerformLeaveRequest,
+		response *PerformLeaveResponse,
+	) error
+}
+
+// NewFederationSenderInternalAPIHTTP creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API.
+// If httpClient is nil an error is returned
+func NewFederationSenderInternalAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderInternalAPI, error) {
+	if httpClient == nil {
+		return nil, errors.New("NewFederationSenderInternalAPIHTTP: httpClient is <nil>")
+	}
+	return &httpFederationSenderInternalAPI{federationSenderURL, httpClient}, nil
+}
+
+type httpFederationSenderInternalAPI struct {
+	federationSenderURL string
+	httpClient          *http.Client
+}

federationsender/api/perform.go

@@ -0,0 +1,87 @@
+package api
+
+import (
+	"context"
+
+	commonHTTP "github.com/matrix-org/dendrite/common/http"
+	"github.com/matrix-org/dendrite/federationsender/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/opentracing/opentracing-go"
+)
+
+const (
+	// FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API.
+	FederationSenderPerformDirectoryLookupRequestPath = "/api/federationsender/performDirectoryLookup"
+
+	// FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API.
+	FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest"
+
+	// FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API.
+	FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest"
+)
+
+type PerformDirectoryLookupRequest struct {
+	RoomAlias  string                       `json:"room_alias"`
+	ServerName gomatrixserverlib.ServerName `json:"server_name"`
+}
+
+type PerformDirectoryLookupResponse struct {
+	RoomID      string                         `json:"room_id"`
+	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
+}
+
+// Handle an instruction to make_join & send_join with a remote server.
+func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup(
+	ctx context.Context,
+	request *PerformDirectoryLookupRequest,
+	response *PerformDirectoryLookupResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDirectoryLookup")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
+type PerformJoinRequest struct {
+	RoomID      string                 `json:"room_id"`
+	UserID      string                 `json:"user_id"`
+	ServerNames types.ServerNames      `json:"server_names"`
+	Content     map[string]interface{} `json:"content"`
+}
+
+type PerformJoinResponse struct {
+}
+
+// Handle an instruction to make_join & send_join with a remote server.
+func (h *httpFederationSenderInternalAPI) PerformJoin(
+	ctx context.Context,
+	request *PerformJoinRequest,
+	response *PerformJoinResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
+type PerformLeaveRequest struct {
+	RoomID string `json:"room_id"`
+}
+
+type PerformLeaveResponse struct {
+}
+
+// Handle an instruction to make_leave & send_leave with a remote server.
+func (h *httpFederationSenderInternalAPI) PerformLeave(
+	ctx context.Context,
+	request *PerformLeaveRequest,
+	response *PerformLeaveResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

federationsender/api/query.go

@@ -2,16 +2,20 @@ package api
 
 import (
 	"context"
-	"errors"
-	"net/http"
 
 	commonHTTP "github.com/matrix-org/dendrite/common/http"
+	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/gomatrixserverlib"
 
-	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/opentracing/opentracing-go"
 )
 
+// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API.
+const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom"
+
+// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API.
+const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom"
+
 // QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom
 type QueryJoinedHostsInRoomRequest struct {
 	RoomID string `json:"room_id"`
@@ -22,6 +26,19 @@ type QueryJoinedHostsInRoomResponse struct {
 	JoinedHosts []types.JoinedHost `json:"joined_hosts"`
 }
 
+// QueryJoinedHostsInRoom implements FederationSenderInternalAPI
+func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom(
+	ctx context.Context,
+	request *QueryJoinedHostsInRoomRequest,
+	response *QueryJoinedHostsInRoomResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
 // QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames
 type QueryJoinedHostServerNamesInRoomRequest struct {
 	RoomID string `json:"room_id"`
@@ -32,61 +49,8 @@ type QueryJoinedHostServerNamesInRoomResponse struct {
 	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
 }
 
-// FederationSenderQueryAPI is used to query information from the federation sender.
+// QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI
-type FederationSenderQueryAPI interface {
+func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom(
-	// Query the joined hosts and the membership events accounting for their participation in a room.
-	// Note that if a server has multiple users in the room, it will have multiple entries in the returned slice.
-	// See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version.
-	QueryJoinedHostsInRoom(
-		ctx context.Context,
-		request *QueryJoinedHostsInRoomRequest,
-		response *QueryJoinedHostsInRoomResponse,
-	) error
-	// Query the server names of the joined hosts in a room.
-	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
-	// containing only the server names (without information for membership events).
-	QueryJoinedHostServerNamesInRoom(
-		ctx context.Context,
-		request *QueryJoinedHostServerNamesInRoomRequest,
-		response *QueryJoinedHostServerNamesInRoomResponse,
-	) error
-}
-
-// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API.
-const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom"
-
-// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API.
-const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom"
-
-// NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API.
-// If httpClient is nil an error is returned
-func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderQueryAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewFederationSenderQueryAPIHTTP: httpClient is <nil>")
-	}
-	return &httpFederationSenderQueryAPI{federationSenderURL, httpClient}, nil
-}
-
-type httpFederationSenderQueryAPI struct {
-	federationSenderURL string
-	httpClient          *http.Client
-}
-
-// QueryJoinedHostsInRoom implements FederationSenderQueryAPI
-func (h *httpFederationSenderQueryAPI) QueryJoinedHostsInRoom(
-	ctx context.Context,
-	request *QueryJoinedHostsInRoomRequest,
-	response *QueryJoinedHostsInRoomResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom")
-	defer span.Finish()
-
-	apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath
-	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}
-
-// QueryJoinedHostServerNamesInRoom implements FederationSenderQueryAPI
-func (h *httpFederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom(
 	ctx context.Context,
 	request *QueryJoinedHostServerNamesInRoomRequest,
 	response *QueryJoinedHostServerNamesInRoomResponse,

federationsender/consumers/eduserver.go

@@ -16,6 +16,7 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/Shopify/sarama"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/eduserver/api"
@@ -23,7 +24,6 @@ import (
 	"github.com/matrix-org/dendrite/federationsender/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/Shopify/sarama.v1"
 )
 
 // OutputTypingEventConsumer consumes events that originate in EDU server.

federationsender/consumers/roomserver.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/Shopify/sarama"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/federationsender/queue"
@@ -27,16 +28,16 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	log "github.com/sirupsen/logrus"
-	sarama "gopkg.in/Shopify/sarama.v1"
+	"github.com/tidwall/gjson"
 )
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
 	cfg        *config.Dendrite
-	roomServerConsumer *common.ContinualConsumer
+	rsAPI      api.RoomserverInternalAPI
+	rsConsumer *common.ContinualConsumer
 	db         storage.Database
 	queues     *queue.OutgoingQueues
-	query              api.RoomserverQueryAPI
 }
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
@@ -45,7 +46,7 @@ func NewOutputRoomEventConsumer(
 	kafkaConsumer sarama.Consumer,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) *OutputRoomEventConsumer {
 	consumer := common.ContinualConsumer{
 		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
@@ -54,10 +55,10 @@ func NewOutputRoomEventConsumer(
 	}
 	s := &OutputRoomEventConsumer{
 		cfg:        cfg,
-		roomServerConsumer: &consumer,
+		rsConsumer: &consumer,
 		db:         store,
 		queues:     queues,
-		query:              queryAPI,
+		rsAPI:      rsAPI,
 	}
 	consumer.ProcessMessage = s.onMessage
 
@@ -66,7 +67,7 @@ func NewOutputRoomEventConsumer(
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	return s.roomServerConsumer.Start()
+	return s.rsConsumer.Start()
 }
 
 // onMessage is called when the federation server receives a new event from the room server output log.
@@ -187,49 +188,12 @@ func (s *OutputRoomEventConsumer) processInvite(oie api.OutputNewInviteEvent) er
 		return nil
 	}
 
-	// When sending a v2 invite, the inviting server should try and include
+	// Try to extract the room invite state. The roomserver will have stashed
-	// a "stripped down" version of the room state. This is pretty much just
+	// this for us in invite_room_state if it didn't already exist.
-	// enough information for the remote side to show something useful to the
-	// user, like the room name, aliases etc.
 	strippedState := []gomatrixserverlib.InviteV2StrippedState{}
-	stateWanted := []string{
+	if inviteRoomState := gjson.GetBytes(oie.Event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() {
-		gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias,
+		if err := json.Unmarshal([]byte(inviteRoomState.Raw), &strippedState); err != nil {
-		gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules,
+			log.WithError(err).Warn("failed to extract invite_room_state from event unsigned")
-	}
-
-	// For each of the state keys that we want to try and send, ask the
-	// roomserver if we have a state event for that room that matches the
-	// state key.
-	for _, wanted := range stateWanted {
-		queryReq := api.QueryLatestEventsAndStateRequest{
-			RoomID: oie.Event.RoomID(),
-			StateToFetch: []gomatrixserverlib.StateKeyTuple{
-				gomatrixserverlib.StateKeyTuple{
-					EventType: wanted,
-					StateKey:  "",
-				},
-			},
-		}
-		// If this fails then we just move onto the next event - we don't
-		// actually know at this point whether the room even has that type
-		// of state.
-		queryRes := api.QueryLatestEventsAndStateResponse{}
-		if err := s.query.QueryLatestEventsAndState(context.TODO(), &queryReq, &queryRes); err != nil {
-			log.WithFields(log.Fields{
-				"room_id":    queryReq.RoomID,
-				"event_type": wanted,
-			}).WithError(err).Info("couldn't find state to strip")
-			continue
-		}
-		// Append the stripped down copy of the state to our list.
-		for _, headeredEvent := range queryRes.StateEvents {
-			event := headeredEvent.Unwrap()
-			strippedState = append(strippedState, gomatrixserverlib.NewInviteV2StrippedState(&event))
-
-			log.WithFields(log.Fields{
-				"room_id":    queryReq.RoomID,
-				"event_type": event.Type(),
-			}).Info("adding stripped state")
 		}
 	}
 
@@ -405,7 +369,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 	// from the roomserver using the query API.
 	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
 	var eventResp api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
+	if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
 		return nil, err
 	}
 

federationsender/federationsender.go

@@ -20,7 +20,8 @@ import (
 	"github.com/matrix-org/dendrite/common/basecomponent"
 	"github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/federationsender/consumers"
-	"github.com/matrix-org/dendrite/federationsender/query"
+	"github.com/matrix-org/dendrite/federationsender/internal"
+	"github.com/matrix-org/dendrite/federationsender/producers"
 	"github.com/matrix-org/dendrite/federationsender/queue"
 	"github.com/matrix-org/dendrite/federationsender/storage"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
@@ -33,18 +34,21 @@ import (
 func SetupFederationSenderComponent(
 	base *basecomponent.BaseDendrite,
 	federation *gomatrixserverlib.FederationClient,
-	rsQueryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-) api.FederationSenderQueryAPI {
+	keyRing *gomatrixserverlib.KeyRing,
-	federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender))
+) api.FederationSenderInternalAPI {
+	federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panic("failed to connect to federation sender db")
 	}
 
-	queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation)
+	roomserverProducer := producers.NewRoomserverProducer(rsAPI, base.Cfg.Matrix.ServerName)
+
+	queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation, roomserverProducer)
 
 	rsConsumer := consumers.NewOutputRoomEventConsumer(
 		base.Cfg, base.KafkaConsumer, queues,
-		federationSenderDB, rsQueryAPI,
+		federationSenderDB, rsAPI,
 	)
 	if err = rsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start room server consumer")
@@ -57,10 +61,10 @@ func SetupFederationSenderComponent(
 		logrus.WithError(err).Panic("failed to start typing server consumer")
 	}
 
-	queryAPI := query.FederationSenderQueryAPI{
+	queryAPI := internal.NewFederationSenderInternalAPI(
-		DB: federationSenderDB,
+		federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing,
-	}
+	)
 	queryAPI.SetupHTTP(http.DefaultServeMux)
 
-	return &queryAPI
+	return queryAPI
 }

federationsender/internal/api.go

@@ -0,0 +1,97 @@
+package internal
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/federationsender/producers"
+	"github.com/matrix-org/dendrite/federationsender/storage"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI
+type FederationSenderInternalAPI struct {
+	api.FederationSenderInternalAPI
+	db         storage.Database
+	cfg        *config.Dendrite
+	producer   *producers.RoomserverProducer
+	federation *gomatrixserverlib.FederationClient
+	keyRing    *gomatrixserverlib.KeyRing
+}
+
+func NewFederationSenderInternalAPI(
+	db storage.Database, cfg *config.Dendrite,
+	producer *producers.RoomserverProducer,
+	federation *gomatrixserverlib.FederationClient,
+	keyRing *gomatrixserverlib.KeyRing,
+) *FederationSenderInternalAPI {
+	return &FederationSenderInternalAPI{
+		db:         db,
+		cfg:        cfg,
+		producer:   producer,
+		federation: federation,
+		keyRing:    keyRing,
+	}
+}
+
+// SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux.
+func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) {
+	servMux.Handle(
+		api.FederationSenderQueryJoinedHostsInRoomPath,
+		common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse {
+			var request api.QueryJoinedHostsInRoomRequest
+			var response api.QueryJoinedHostsInRoomResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.FederationSenderQueryJoinedHostServerNamesInRoomPath,
+		common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
+			var request api.QueryJoinedHostServerNamesInRoomRequest
+			var response api.QueryJoinedHostServerNamesInRoomResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(api.FederationSenderPerformJoinRequestPath,
+		common.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse {
+			var request api.PerformJoinRequest
+			var response api.PerformJoinResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := f.PerformJoin(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(api.FederationSenderPerformLeaveRequestPath,
+		common.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse {
+			var request api.PerformLeaveRequest
+			var response api.PerformLeaveResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := f.PerformLeave(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+}

federationsender/internal/perform.go

@@ -0,0 +1,157 @@
+package internal
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/federationsender/internal/perform"
+	"github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+)
+
+// PerformLeaveRequest implements api.FederationSenderInternalAPI
+func (r *FederationSenderInternalAPI) PerformDirectoryLookup(
+	ctx context.Context,
+	request *api.PerformDirectoryLookupRequest,
+	response *api.PerformDirectoryLookupResponse,
+) (err error) {
+	dir, err := r.federation.LookupRoomAlias(
+		ctx,
+		request.ServerName,
+		request.RoomAlias,
+	)
+	if err != nil {
+		return err
+	}
+	response.RoomID = dir.RoomID
+	response.ServerNames = dir.Servers
+	return nil
+}
+
+// PerformJoinRequest implements api.FederationSenderInternalAPI
+func (r *FederationSenderInternalAPI) PerformJoin(
+	ctx context.Context,
+	request *api.PerformJoinRequest,
+	response *api.PerformJoinResponse,
+) (err error) {
+	// Look up the supported room versions.
+	var supportedVersions []gomatrixserverlib.RoomVersion
+	for version := range version.SupportedRoomVersions() {
+		supportedVersions = append(supportedVersions, version)
+	}
+
+	// Deduplicate the server names we were provided.
+	util.Unique(request.ServerNames)
+
+	// Try each server that we were provided until we land on one that
+	// successfully completes the make-join send-join dance.
+	for _, serverName := range request.ServerNames {
+		// Try to perform a make_join using the information supplied in the
+		// request.
+		respMakeJoin, err := r.federation.MakeJoin(
+			ctx,
+			serverName,
+			request.RoomID,
+			request.UserID,
+			supportedVersions,
+		)
+		if err != nil {
+			// TODO: Check if the user was not allowed to join the room.
+			return fmt.Errorf("r.federation.MakeJoin: %w", err)
+		}
+
+		// Set all the fields to be what they should be, this should be a no-op
+		// but it's possible that the remote server returned us something "odd"
+		respMakeJoin.JoinEvent.Type = gomatrixserverlib.MRoomMember
+		respMakeJoin.JoinEvent.Sender = request.UserID
+		respMakeJoin.JoinEvent.StateKey = &request.UserID
+		respMakeJoin.JoinEvent.RoomID = request.RoomID
+		respMakeJoin.JoinEvent.Redacts = ""
+		if request.Content == nil {
+			request.Content = map[string]interface{}{}
+		}
+		request.Content["membership"] = "join"
+		if err = respMakeJoin.JoinEvent.SetContent(request.Content); err != nil {
+			return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err)
+		}
+		if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil {
+			return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err)
+		}
+
+		// Work out if we support the room version that has been supplied in
+		// the make_join response.
+		if respMakeJoin.RoomVersion == "" {
+			respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1
+		}
+		if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil {
+			return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err)
+		}
+
+		// Build the join event.
+		event, err := respMakeJoin.JoinEvent.Build(
+			time.Now(),
+			r.cfg.Matrix.ServerName,
+			r.cfg.Matrix.KeyID,
+			r.cfg.Matrix.PrivateKey,
+			respMakeJoin.RoomVersion,
+		)
+		if err != nil {
+			return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err)
+		}
+
+		// Try to perform a send_join using the newly built event.
+		respSendJoin, err := r.federation.SendJoin(
+			ctx,
+			serverName,
+			event,
+			respMakeJoin.RoomVersion,
+		)
+		if err != nil {
+			logrus.WithError(err).Warnf("r.federation.SendJoin failed")
+			continue
+		}
+
+		// Check that the send_join response was valid.
+		joinCtx := perform.JoinContext(r.federation, r.keyRing)
+		if err = joinCtx.CheckSendJoinResponse(
+			ctx, event, serverName, respMakeJoin, respSendJoin,
+		); err != nil {
+			logrus.WithError(err).Warnf("joinCtx.CheckSendJoinResponse failed")
+			continue
+		}
+
+		// If we successfully performed a send_join above then the other
+		// server now thinks we're a part of the room. Send the newly
+		// returned state to the roomserver to update our local view.
+		if err = r.producer.SendEventWithState(
+			ctx,
+			respSendJoin.ToRespState(),
+			event.Headered(respMakeJoin.RoomVersion),
+		); err != nil {
+			logrus.WithError(err).Warnf("r.producer.SendEventWithState failed")
+			continue
+		}
+
+		// We're all good.
+		return nil
+	}
+
+	// If we reach here then we didn't complete a join for some reason.
+	return fmt.Errorf(
+		"failed to join user %q to room %q through %d server(s)",
+		request.UserID, request.RoomID, len(request.ServerNames),
+	)
+}
+
+// PerformLeaveRequest implements api.FederationSenderInternalAPI
+func (r *FederationSenderInternalAPI) PerformLeave(
+	ctx context.Context,
+	request *api.PerformLeaveRequest,
+	response *api.PerformLeaveResponse,
+) (err error) {
+	return nil
+}

federationsender/internal/perform/join.go

@@ -0,0 +1,70 @@
+package perform
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// This file contains helpers for the PerformJoin function.
+
+type joinContext struct {
+	federation *gomatrixserverlib.FederationClient
+	keyRing    *gomatrixserverlib.KeyRing
+}
+
+// Returns a new join context.
+func JoinContext(f *gomatrixserverlib.FederationClient, k *gomatrixserverlib.KeyRing) *joinContext {
+	return &joinContext{
+		federation: f,
+		keyRing:    k,
+	}
+}
+
+// checkSendJoinResponse checks that all of the signatures are correct
+// and that the join is allowed by the supplied state.
+func (r joinContext) CheckSendJoinResponse(
+	ctx context.Context,
+	event gomatrixserverlib.Event,
+	server gomatrixserverlib.ServerName,
+	respMakeJoin gomatrixserverlib.RespMakeJoin,
+	respSendJoin gomatrixserverlib.RespSendJoin,
+) error {
+	// A list of events that we have retried, if they were not included in
+	// the auth events supplied in the send_join.
+	retries := map[string]bool{}
+
+retryCheck:
+	// TODO: Can we expand Check here to return a list of missing auth
+	// events rather than failing one at a time?
+	if err := respSendJoin.Check(ctx, r.keyRing, event); err != nil {
+		switch e := err.(type) {
+		case gomatrixserverlib.MissingAuthEventError:
+			// Check that we haven't already retried for this event, prevents
+			// us from ending up in endless loops
+			if !retries[e.AuthEventID] {
+				// Ask the server that we're talking to right now for the event
+				tx, txerr := r.federation.GetEvent(ctx, server, e.AuthEventID)
+				if txerr != nil {
+					return fmt.Errorf("r.federation.GetEvent: %w", txerr)
+				}
+				// For each event returned, add it to the auth events.
+				for _, pdu := range tx.PDUs {
+					ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion)
+					if everr != nil {
+						return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr)
+					}
+					respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev)
+				}
+				// Mark the event as retried and then give the check another go.
+				retries[e.AuthEventID] = true
+				goto retryCheck
+			}
+			return fmt.Errorf("respSendJoin (after retries): %w", e)
+		default:
+			return fmt.Errorf("respSendJoin: %w", err)
+		}
+	}
+	return nil
+}

federationsender/internal/query.go

@@ -0,0 +1,39 @@
+package internal
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// QueryJoinedHostsInRoom implements api.FederationSenderInternalAPI
+func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom(
+	ctx context.Context,
+	request *api.QueryJoinedHostsInRoomRequest,
+	response *api.QueryJoinedHostsInRoomResponse,
+) (err error) {
+	response.JoinedHosts, err = f.db.GetJoinedHosts(ctx, request.RoomID)
+	return
+}
+
+// QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI
+func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom(
+	ctx context.Context,
+	request *api.QueryJoinedHostServerNamesInRoomRequest,
+	response *api.QueryJoinedHostServerNamesInRoomResponse,
+) (err error) {
+	joinedHosts, err := f.db.GetJoinedHosts(ctx, request.RoomID)
+	if err != nil {
+		return
+	}
+
+	response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(joinedHosts))
+	for _, host := range joinedHosts {
+		response.ServerNames = append(response.ServerNames, host.ServerName)
+	}
+
+	// TODO: remove duplicates?
+
+	return
+}

federationsender/producers/roomserver.go

@@ -0,0 +1,102 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package producers
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// RoomserverProducer produces events for the roomserver to consume.
+type RoomserverProducer struct {
+	InputAPI   api.RoomserverInternalAPI
+	serverName gomatrixserverlib.ServerName
+}
+
+// NewRoomserverProducer creates a new RoomserverProducer
+func NewRoomserverProducer(
+	rsAPI api.RoomserverInternalAPI, serverName gomatrixserverlib.ServerName,
+) *RoomserverProducer {
+	return &RoomserverProducer{
+		InputAPI:   rsAPI,
+		serverName: serverName,
+	}
+}
+
+// SendInviteResponse drops an invite response back into the roomserver so that users
+// already in the room will be notified of the new invite. The invite response is signed
+// by the remote side.
+func (c *RoomserverProducer) SendInviteResponse(
+	ctx context.Context, res gomatrixserverlib.RespInviteV2, roomVersion gomatrixserverlib.RoomVersion,
+) (string, error) {
+	ev := res.Event.Headered(roomVersion)
+	ire := api.InputRoomEvent{
+		Kind:          api.KindNew,
+		Event:         ev,
+		AuthEventIDs:  ev.AuthEventIDs(),
+		SendAsServer:  string(c.serverName),
+		TransactionID: nil,
+	}
+	return c.SendInputRoomEvents(ctx, []api.InputRoomEvent{ire})
+}
+
+// SendEventWithState writes an event with KindNew to the roomserver input log
+// with the state at the event as KindOutlier before it.
+func (c *RoomserverProducer) SendEventWithState(
+	ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent,
+) error {
+	outliers, err := state.Events()
+	if err != nil {
+		return err
+	}
+
+	var ires []api.InputRoomEvent
+	for _, outlier := range outliers {
+		ires = append(ires, api.InputRoomEvent{
+			Kind:         api.KindOutlier,
+			Event:        outlier.Headered(event.RoomVersion),
+			AuthEventIDs: outlier.AuthEventIDs(),
+		})
+	}
+
+	stateEventIDs := make([]string, len(state.StateEvents))
+	for i := range state.StateEvents {
+		stateEventIDs[i] = state.StateEvents[i].EventID()
+	}
+
+	ires = append(ires, api.InputRoomEvent{
+		Kind:          api.KindNew,
+		Event:         event,
+		AuthEventIDs:  event.AuthEventIDs(),
+		HasState:      true,
+		StateEventIDs: stateEventIDs,
+	})
+
+	_, err = c.SendInputRoomEvents(ctx, ires)
+	return err
+}
+
+// SendInputRoomEvents writes the given input room events to the roomserver input API.
+func (c *RoomserverProducer) SendInputRoomEvents(
+	ctx context.Context, ires []api.InputRoomEvent,
+) (eventID string, err error) {
+	request := api.InputRoomEventsRequest{InputRoomEvents: ires}
+	var response api.InputRoomEventsResponse
+	err = c.InputAPI.InputRoomEvents(ctx, &request, &response)
+	eventID = response.EventID
+	return
+}

federationsender/query/query.go

@@ -1,88 +0,0 @@
-package query
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-
-	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/dendrite/federationsender/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-// FederationSenderQueryDatabase has the APIs needed to implement the query API.
-type FederationSenderQueryDatabase interface {
-	GetJoinedHosts(
-		ctx context.Context, roomID string,
-	) ([]types.JoinedHost, error)
-}
-
-// FederationSenderQueryAPI is an implementation of api.FederationSenderQueryAPI
-type FederationSenderQueryAPI struct {
-	DB FederationSenderQueryDatabase
-}
-
-// QueryJoinedHostsInRoom implements api.FederationSenderQueryAPI
-func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom(
-	ctx context.Context,
-	request *api.QueryJoinedHostsInRoomRequest,
-	response *api.QueryJoinedHostsInRoomResponse,
-) (err error) {
-	response.JoinedHosts, err = f.DB.GetJoinedHosts(ctx, request.RoomID)
-	return
-}
-
-// QueryJoinedHostServerNamesInRoom implements api.FederationSenderQueryAPI
-func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom(
-	ctx context.Context,
-	request *api.QueryJoinedHostServerNamesInRoomRequest,
-	response *api.QueryJoinedHostServerNamesInRoomResponse,
-) (err error) {
-	joinedHosts, err := f.DB.GetJoinedHosts(ctx, request.RoomID)
-	if err != nil {
-		return
-	}
-
-	response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(joinedHosts))
-	for _, host := range joinedHosts {
-		response.ServerNames = append(response.ServerNames, host.ServerName)
-	}
-
-	// TODO: remove duplicates?
-
-	return
-}
-
-// SetupHTTP adds the FederationSenderQueryAPI handlers to the http.ServeMux.
-func (f *FederationSenderQueryAPI) SetupHTTP(servMux *http.ServeMux) {
-	servMux.Handle(
-		api.FederationSenderQueryJoinedHostsInRoomPath,
-		common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryJoinedHostsInRoomRequest
-			var response api.QueryJoinedHostsInRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.FederationSenderQueryJoinedHostServerNamesInRoomPath,
-		common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryJoinedHostServerNamesInRoomRequest
-			var response api.QueryJoinedHostServerNamesInRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-}

federationsender/queue/destinationqueue.go

@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationsender/producers"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	log "github.com/sirupsen/logrus"
@@ -32,6 +33,7 @@ import (
 // ensures that only one request is in flight to a given destination
 // at a time.
 type destinationQueue struct {
+	rsProducer  *producers.RoomserverProducer
 	client      *gomatrixserverlib.FederationClient
 	origin      gomatrixserverlib.ServerName
 	destination gomatrixserverlib.ServerName
@@ -165,18 +167,38 @@ func (oq *destinationQueue) nextInvites() bool {
 	}
 
 	for _, inviteReq := range oq.pendingInvites {
-		ev := inviteReq.Event()
+		ev, roomVersion := inviteReq.Event(), inviteReq.RoomVersion()
 
-		if _, err := oq.client.SendInviteV2(
+		log.WithFields(log.Fields{
+			"event_id":     ev.EventID(),
+			"room_version": roomVersion,
+			"destination":  oq.destination,
+		}).Info("sending invite")
+
+		inviteRes, err := oq.client.SendInviteV2(
 			context.TODO(),
 			oq.destination,
 			*inviteReq,
-		); err != nil {
+		)
+		if err != nil {
 			log.WithFields(log.Fields{
 				"event_id":    ev.EventID(),
 				"state_key":   ev.StateKey(),
 				"destination": oq.destination,
 			}).WithError(err).Error("failed to send invite")
+			continue
+		}
+
+		if _, err = oq.rsProducer.SendInviteResponse(
+			context.TODO(),
+			inviteRes,
+			roomVersion,
+		); err != nil {
+			log.WithFields(log.Fields{
+				"event_id":    ev.EventID(),
+				"state_key":   ev.StateKey(),
+				"destination": oq.destination,
+			}).WithError(err).Error("failed to return signed invite to roomserver")
 		}
 	}
 

federationsender/queue/queue.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/matrix-org/dendrite/federationsender/producers"
 	"github.com/matrix-org/gomatrixserverlib"
 	log "github.com/sirupsen/logrus"
 )
@@ -25,6 +26,7 @@ import (
 // OutgoingQueues is a collection of queues for sending transactions to other
 // matrix servers
 type OutgoingQueues struct {
+	rsProducer *producers.RoomserverProducer
 	origin     gomatrixserverlib.ServerName
 	client     *gomatrixserverlib.FederationClient
 	// The queuesMutex protects queues
@@ -33,8 +35,13 @@ type OutgoingQueues struct {
 }
 
 // NewOutgoingQueues makes a new OutgoingQueues
-func NewOutgoingQueues(origin gomatrixserverlib.ServerName, client *gomatrixserverlib.FederationClient) *OutgoingQueues {
+func NewOutgoingQueues(
+	origin gomatrixserverlib.ServerName,
+	client *gomatrixserverlib.FederationClient,
+	rsProducer *producers.RoomserverProducer,
+) *OutgoingQueues {
 	return &OutgoingQueues{
+		rsProducer: rsProducer,
 		origin:     origin,
 		client:     client,
 		queues:     map[gomatrixserverlib.ServerName]*destinationQueue{},
@@ -67,6 +74,7 @@ func (oqs *OutgoingQueues) SendEvent(
 		oq := oqs.queues[destination]
 		if oq == nil {
 			oq = &destinationQueue{
+				rsProducer:  oqs.rsProducer,
 				origin:      oqs.origin,
 				destination: destination,
 				client:      oqs.client,
@@ -111,6 +119,7 @@ func (oqs *OutgoingQueues) SendInvite(
 	oq := oqs.queues[destination]
 	if oq == nil {
 		oq = &destinationQueue{
+			rsProducer:  oqs.rsProducer,
 			origin:      oqs.origin,
 			destination: destination,
 			client:      oqs.client,
@@ -151,6 +160,7 @@ func (oqs *OutgoingQueues) SendEDU(
 		oq := oqs.queues[destination]
 		if oq == nil {
 			oq = &destinationQueue{
+				rsProducer:  oqs.rsProducer,
 				origin:      oqs.origin,
 				destination: destination,
 				client:      oqs.client,

federationsender/storage/postgres/storage.go

@@ -33,10 +33,10 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

federationsender/storage/sqlite3/storage.go

@@ -38,7 +38,7 @@ type Database struct {
 func NewDatabase(dataSourceName string) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

federationsender/storage/storage.go

@@ -19,22 +19,23 @@ package storage
 import (
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/federationsender/storage/postgres"
 	"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
 )
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 	switch uri.Scheme {
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName)
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	default:
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 }

federationsender/storage/storage_wasm.go

@@ -18,11 +18,15 @@ import (
 	"fmt"
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
 )
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (Database, error) {
+func NewDatabase(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

federationsender/types/types.go

@@ -28,6 +28,12 @@ type JoinedHost struct {
 	ServerName gomatrixserverlib.ServerName
 }
 
+type ServerNames []gomatrixserverlib.ServerName
+
+func (s ServerNames) Len() int           { return len(s) }
+func (s ServerNames) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s ServerNames) Less(i, j int) bool { return s[i] < s[j] }
+
 // A EventIDMismatchError indicates that we have got out of sync with the
 // room server.
 type EventIDMismatchError struct {

go.mod

@@ -1,9 +1,9 @@
 module github.com/matrix-org/dendrite
 
 require (
+	github.com/Shopify/sarama v1.26.1
 	github.com/gorilla/mux v1.7.3
 	github.com/hashicorp/golang-lru v0.5.4
-	github.com/kr/pretty v0.2.0 // indirect
 	github.com/lib/pq v1.2.0
 	github.com/libp2p/go-libp2p v0.6.0
 	github.com/libp2p/go-libp2p-circuit v0.1.4
@@ -17,26 +17,21 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec
-	github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1
+	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
-	github.com/mattn/go-sqlite3 v2.0.3+incompatible
+	github.com/mattn/go-sqlite3 v2.0.2+incompatible
 	github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5
 	github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6
 	github.com/opentracing/opentracing-go v1.1.0
-	github.com/pierrec/lz4 v2.5.0+incompatible // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.4.1
-	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/sirupsen/logrus v1.4.2
-	github.com/tidwall/gjson v1.6.0 // indirect
+	github.com/tidwall/gjson v1.6.0
-	github.com/tidwall/pretty v1.0.1 // indirect
+	github.com/uber/jaeger-client-go v2.15.0+incompatible
-	github.com/uber/jaeger-client-go v2.22.1+incompatible
+	github.com/uber/jaeger-lib v1.5.0
-	github.com/uber/jaeger-lib v2.2.0+incompatible
+	go.uber.org/atomic v1.4.0
-	go.uber.org/atomic v1.6.0
 	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
-	golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 // indirect
-	gopkg.in/Shopify/sarama.v1 v1.20.1
 	gopkg.in/h2non/bimg.v1 v1.0.18
 	gopkg.in/yaml.v2 v2.2.8
 )

go.sum

@@ -1,11 +1,12 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE=
 github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
+github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA=
+github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
 github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
@@ -61,19 +62,18 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
-github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
-github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
 github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -119,6 +119,8 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
+github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
@@ -140,7 +142,6 @@ github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67Fexh
 github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
 github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
 github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
-github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU=
 github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
 github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
@@ -158,7 +159,6 @@ github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50=
 github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
-github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
 github.com/ipfs/go-log v1.0.2 h1:s19ZwJxH8rPWzypjcDpqPLIyV7BnbLqvpli3iZoqYK0=
 github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
@@ -178,6 +178,8 @@ github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod
 github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
 github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10=
 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
+github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
+github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
@@ -188,9 +190,10 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
-github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
+github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
@@ -364,10 +367,12 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7 h1:4vE84tE3r7BitCt2HQvT231JrhMjDfjDVDqVoiVPv0w=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec h1:9MvZSZzBKvCWqM5KXMGZ1PBDrSLcxs5zfc561UPgcYA=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk=
 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A=
+github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
+github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8=
 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
@@ -380,8 +385,6 @@ github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U=
 github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
-github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -475,8 +478,6 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg=
 github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrec/lz4 v2.5.0+incompatible h1:MbdIZ43A//duwOjQqK3nP+up+65yraNFyX3Vp6Rwues=
-github.com/pierrec/lz4 v2.5.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -505,10 +506,6 @@ github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLk
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
-github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
-github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
@@ -550,17 +547,12 @@ github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4=
 github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
 github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk=
 github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
-github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM=
-github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo=
 github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
-github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
-github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
-github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46FQZjprUcc=
 github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE=
 github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA=
 github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA=
@@ -571,8 +563,9 @@ github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.
 github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow=
 github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg=
 github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -580,9 +573,8 @@ go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010=
 go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
-go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
@@ -605,16 +597,13 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI=
 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -629,6 +618,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -673,11 +663,6 @@ golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 h1:bMm0eoDiGkM5VfIyKjxDvoflW5GLp7+VCo+60n8F+TE=
-golang.org/x/tools v0.0.0-20200402223321-bcf690261a44/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
@@ -707,6 +692,16 @@ gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4=
 gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So=
 gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM=
 gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
+gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
+gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
+gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg=
+gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
+gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
+gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
 gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI=
 gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o=
 gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8=

internal/sqlutil/trace.go

@@ -21,9 +21,11 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/ngrok/sqlmw"
 	"github.com/sirupsen/logrus"
 )
@@ -76,12 +78,27 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [
 // Open opens a database specified by its database driver name and a driver-specific data source name,
 // usually consisting of at least a database name and connection information. Includes tracing driver
 // if DENDRITE_TRACE_SQL=1
-func Open(driverName, dsn string) (*sql.DB, error) {
+func Open(driverName, dsn string, dbProperties common.DbProperties) (*sql.DB, error) {
 	if tracingEnabled {
 		// install the wrapped driver
 		driverName += "-trace"
 	}
-	return sql.Open(driverName, dsn)
+	db, err := sql.Open(driverName, dsn)
+	if err != nil {
+		return nil, err
+	}
+	if driverName != common.SQLiteDriverName() && dbProperties != nil {
+		logrus.WithFields(logrus.Fields{
+			"MaxOpenConns":    dbProperties.MaxOpenConns(),
+			"MaxIdleConns":    dbProperties.MaxIdleConns(),
+			"ConnMaxLifetime": dbProperties.ConnMaxLifetime(),
+			"dataSourceName":  regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
+		}).Debug("Setting DB connection limits")
+		db.SetMaxOpenConns(dbProperties.MaxOpenConns())
+		db.SetMaxIdleConns(dbProperties.MaxIdleConns())
+		db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime())
+	}
+	return db, nil
 }
 
 func init() {

mediaapi/mediaapi.go

@@ -29,7 +29,7 @@ func SetupMediaAPIComponent(
 	base *basecomponent.BaseDendrite,
 	deviceDB devices.Database,
 ) {
-	mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI))
+	mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to media db")
 	}

mediaapi/storage/postgres/storage.go

@@ -21,6 +21,7 @@ import (
 
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -33,10 +34,10 @@ type Database struct {
 }
 
 // Open opens a postgres database.
-func Open(dataSourceName string) (*Database, error) {
+func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {

mediaapi/storage/sqlite3/storage.go

@@ -37,7 +37,7 @@ type Database struct {
 func Open(dataSourceName string) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {