Merge branch 'master' into neilalexander/gobind

appservice/api/query.go

@@ -22,10 +22,9 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/matrix-org/dendrite/internal"
 )
 
 // RoomAliasExistsRequest is a request to an application service
@@ -110,7 +109,7 @@ func RetrieveUserProfile(
 
 	// If no user exists, return
 	if !userResp.UserIDExists {
-		return nil, internal.ErrProfileNoExists
+		return nil, eventutil.ErrProfileNoExists
 	}
 
 	// Try to query the user from the local database again

appservice/appservice.go

@@ -16,7 +16,6 @@ package appservice
 
 import (
 	"context"
-	"errors"
 	"net/http"
 	"sync"
 	"time"
@@ -29,12 +28,10 @@ import (
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/appservice/workers"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
 )
 
@@ -46,9 +43,8 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQuer
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *basecomponent.BaseDendrite,
+	base *setup.BaseDendrite,
-	accountsDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceQueryAPI {
 	// Create a connection to the appservice postgres DB
@@ -70,7 +66,7 @@ func NewInternalAPI(
 		workerStates[i] = ws
 
 		// Create bot account for this AS if it doesn't already exist
-		if err = generateAppServiceAccount(accountsDB, deviceDB, appservice); err != nil {
+		if err = generateAppServiceAccount(userAPI, appservice); err != nil {
 			logrus.WithFields(logrus.Fields{
 				"appservice": appservice.ID,
 			}).WithError(err).Panicf("failed to generate bot account for appservice")
@@ -86,12 +82,16 @@ func NewInternalAPI(
 		Cfg: base.Cfg,
 	}
 
-	consumer := consumers.NewOutputRoomEventConsumer(
+	// Only consume if we actually have ASes to track, else we'll just chew cycles needlessly.
-		base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB,
+	// We can't add ASes at runtime so this is safe to do.
-		rsAPI, workerStates,
+	if len(workerStates) > 0 {
-	)
+		consumer := consumers.NewOutputRoomEventConsumer(
-	if err := consumer.Start(); err != nil {
+			base.Cfg, base.KafkaConsumer, appserviceDB,
-		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
+			rsAPI, workerStates,
+		)
+		if err := consumer.Start(); err != nil {
+			logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
+		}
 	}
 
 	// Create application service transaction workers
@@ -105,22 +105,25 @@ func NewInternalAPI(
 // `sender_localpart` field of each application service if it doesn't
 // exist already
 func generateAppServiceAccount(
-	accountsDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 	as config.ApplicationService,
 ) error {
-	ctx := context.Background()
+	var accRes userapi.PerformAccountCreationResponse
-
+	err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{
-	// Create an account for the application service
+		AccountType:  userapi.AccountTypeUser,
-	_, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID)
+		Localpart:    as.SenderLocalpart,
+		AppServiceID: as.ID,
+		OnConflict:   userapi.ConflictUpdate,
+	}, &accRes)
 	if err != nil {
-		if errors.Is(err, internal.ErrUserExists) { // This account already exists
-			return nil
-		}
 		return err
 	}
-
+	var devRes userapi.PerformDeviceCreationResponse
-	// Create a dummy device with a dummy token for the application service
+	err = userAPI.PerformDeviceCreation(context.Background(), &userapi.PerformDeviceCreationRequest{
-	_, err = deviceDB.CreateDevice(ctx, as.SenderLocalpart, nil, as.ASToken, &as.SenderLocalpart)
+		Localpart:         as.SenderLocalpart,
+		AccessToken:       as.ASToken,
+		DeviceID:          &as.SenderLocalpart,
+		DeviceDisplayName: &as.SenderLocalpart,
+	}, &devRes)
 	return err
 }

appservice/consumers/roomserver.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -33,7 +32,6 @@ import (
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
 	roomServerConsumer *internal.ContinualConsumer
-	db                 accounts.Database
 	asDB               storage.Database
 	rsAPI              api.RoomserverInternalAPI
 	serverName         string
@@ -45,7 +43,6 @@ type OutputRoomEventConsumer struct {
 func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
-	store accounts.Database,
 	appserviceDB storage.Database,
 	rsAPI api.RoomserverInternalAPI,
 	workerStates []types.ApplicationServiceWorkerState,
@@ -53,11 +50,10 @@ func NewOutputRoomEventConsumer(
 	consumer := internal.ContinualConsumer{
 		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
 		Consumer:       kafkaConsumer,
-		PartitionStore: store,
+		PartitionStore: appserviceDB,
 	}
 	s := &OutputRoomEventConsumer{
 		roomServerConsumer: &consumer,
-		db:                 store,
 		asDB:               appserviceDB,
 		rsAPI:              rsAPI,
 		serverName:         string(cfg.Matrix.ServerName),
@@ -91,60 +87,13 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 		return nil
 	}
 
-	ev := output.NewRoomEvent.Event
+	events := []gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
-	log.WithFields(log.Fields{
+	events = append(events, output.NewRoomEvent.AddStateEvents...)
-		"event_id": ev.EventID(),
-		"room_id":  ev.RoomID(),
-		"type":     ev.Type(),
-	}).Info("appservice received an event from roomserver")
-
-	missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev)
-	if err != nil {
-		return err
-	}
-	events := append(missingEvents, ev)
 
 	// Send event to any relevant application services
 	return s.filterRoomserverEvents(context.TODO(), events)
 }
 
-// lookupMissingStateEvents looks up the state events that are added by a new event,
-// and returns any not already present.
-func (s *OutputRoomEventConsumer) lookupMissingStateEvents(
-	addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent,
-) ([]gomatrixserverlib.HeaderedEvent, error) {
-	// Fast path if there aren't any new state events.
-	if len(addsStateEventIDs) == 0 {
-		return []gomatrixserverlib.HeaderedEvent{}, nil
-	}
-
-	// Fast path if the only state event added is the event itself.
-	if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() {
-		return []gomatrixserverlib.HeaderedEvent{}, nil
-	}
-
-	result := []gomatrixserverlib.HeaderedEvent{}
-	missing := []string{}
-	for _, id := range addsStateEventIDs {
-		if id != event.EventID() {
-			// If the event isn't the current one, add it to the list of events
-			// to retrieve from the roomserver
-			missing = append(missing, id)
-		}
-	}
-
-	// Request the missing events from the roomserver
-	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
-	var eventResp api.QueryEventsByIDResponse
-	if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
-		return nil, err
-	}
-
-	result = append(result, eventResp.Events...)
-
-	return result, nil
-}
-
 // filterRoomserverEvents takes in events and decides whether any of them need
 // to be passed on to an external application service. It does this by checking
 // each namespace of each registered application service, and if there is a

appservice/inthttp/client.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/appservice/api"
-	internalHTTP "github.com/matrix-org/dendrite/internal/http"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/opentracing/opentracing-go"
 )
 
@@ -46,7 +46,7 @@ func (h *httpAppServiceQueryAPI) RoomAliasExists(
 	defer span.Finish()
 
 	apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // UserIDExists implements AppServiceQueryAPI
@@ -59,5 +59,5 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
 	defer span.Finish()
 
 	apiURL := h.appserviceURL + AppServiceUserIDExistsPath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }

appservice/inthttp/server.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/util"
 )
 
@@ -14,7 +14,7 @@ import (
 func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		AppServiceRoomAliasExistsPath,
-		internal.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
 			var request api.RoomAliasExistsRequest
 			var response api.RoomAliasExistsResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -28,7 +28,7 @@ func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) {
 	)
 	internalAPIMux.Handle(
 		AppServiceUserIDExistsPath,
-		internal.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse {
 			var request api.UserIDExistsRequest
 			var response api.UserIDExistsResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {

appservice/storage/interface.go

@@ -17,10 +17,12 @@ package storage
 import (
 	"context"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
+	internal.PartitionStorer
 	StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.HeaderedEvent) error
 	GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error)
 	CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error)

appservice/storage/postgres/storage.go

@@ -21,20 +21,20 @@ import (
 
 	// Import postgres database driver
 	_ "github.com/lib/pq"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // Database stores events intended to be later sent to application services
 type Database struct {
+	sqlutil.PartitionOffsetStatements
 	events eventsStatements
 	txnID  txnStatements
 	db     *sql.DB
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
 	var result Database
 	var err error
 	if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
@@ -43,6 +43,9 @@ func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (*Da
 	if err = result.prepare(); err != nil {
 		return nil, err
 	}
+	if err = result.PartitionOffsetStatements.Prepare(result.db, "appservice"); err != nil {
+		return nil, err
+	}
 	return &result, nil
 }
 

appservice/storage/sqlite3/storage.go

@@ -20,7 +20,6 @@ import (
 	"database/sql"
 
 	// Import SQLite database driver
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	_ "github.com/mattn/go-sqlite3"
@@ -28,6 +27,7 @@ import (
 
 // Database stores events intended to be later sent to application services
 type Database struct {
+	sqlutil.PartitionOffsetStatements
 	events eventsStatements
 	txnID  txnStatements
 	db     *sql.DB
@@ -41,12 +41,15 @@ func NewDatabase(dataSourceName string) (*Database, error) {
 	if err != nil {
 		return nil, err
 	}
-	if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil {
+	if result.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {
 		return nil, err
 	}
+	if err = result.PartitionOffsetStatements.Prepare(result.db, "appservice"); err != nil {
+		return nil, err
+	}
 	return &result, nil
 }
 

appservice/storage/storage.go

@@ -21,12 +21,12 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage/postgres"
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
 // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets DB connection parameters
-func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) {
+func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return postgres.NewDatabase(dataSourceName, dbProperties)

appservice/storage/storage_wasm.go

@@ -19,12 +19,12 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
 func NewDatabase(
 	dataSourceName string,
-	dbProperties internal.DbProperties, // nolint:unparam
+	dbProperties sqlutil.DbProperties, // nolint:unparam
 ) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {

are-we-synapse-yet.list

@@ -97,8 +97,8 @@ rst PUT power_levels should not explode if the old power levels were empty
 rst Both GET and PUT work
 rct POST /rooms/:room_id/receipt can create receipts 
 red POST /rooms/:room_id/read_markers can create read marker
-med POST /media/v1/upload can create an upload 
+med POST /media/r0/upload can create an upload 
-med GET /media/v1/download can fetch the value again 
+med GET /media/r0/download can fetch the value again 
 cap GET /capabilities is present and well formed for registered user 
 cap GET /r0/capabilities is not public
 reg Register with a recaptcha 

are-we-synapse-yet.py

@@ -33,6 +33,7 @@ import sys
 
 test_mappings = {
     "nsp": "Non-Spec API",
+    "unk": "Unknown API (no group specified)",
     "f": "Federation", # flag to mark test involves federation
 
     "federation_apis": {
@@ -214,7 +215,8 @@ def main(results_tap_path, verbose):
             # }
         },
         "nonspec": {
-            "nsp": {}
+            "nsp": {},
+            "unk": {}
         },
     }
     with open(results_tap_path, "r") as f:
@@ -225,7 +227,7 @@ def main(results_tap_path, verbose):
             name = test_result["name"]
             group_id = test_name_to_group_id.get(name)
             if not group_id:
-                raise Exception("The test '%s' doesn't have a group" % (name,))
+                summary["nonspec"]["unk"][name] = test_result["ok"]
             if group_id == "nsp":
                 summary["nonspec"]["nsp"][name] = test_result["ok"]
             elif group_id in test_mappings["federation_apis"]:

clientapi/auth/auth.go

@@ -18,17 +18,13 @@ package auth
 import (
 	"context"
 	"crypto/rand"
-	"database/sql"
 	"encoding/base64"
 	"fmt"
 	"net/http"
 	"strings"
 
-	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/util"
 )
 
@@ -39,21 +35,13 @@ var tokenByteLength = 32
 // DeviceDatabase represents a device database.
 type DeviceDatabase interface {
 	// Look up the device matching the given access token.
-	GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error)
+	GetDeviceByAccessToken(ctx context.Context, token string) (*api.Device, error)
 }
 
 // AccountDatabase represents an account database.
 type AccountDatabase interface {
 	// Look up the account matching the given localpart.
-	GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error)
+	GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error)
-}
-
-// Data contains information required to authenticate a request.
-type Data struct {
-	AccountDB AccountDatabase
-	DeviceDB  DeviceDatabase
-	// AppServices is the list of all registered AS
-	AppServices []config.ApplicationService
 }
 
 // VerifyUserFromRequest authenticates the HTTP request,
@@ -62,8 +50,8 @@ type Data struct {
 // Note: For an AS user, AS dummy device is returned.
 // On failure returns an JSON error response which can be sent to the client.
 func VerifyUserFromRequest(
-	req *http.Request, data Data,
+	req *http.Request, userAPI api.UserInternalAPI,
-) (*authtypes.Device, *util.JSONResponse) {
+) (*api.Device, *util.JSONResponse) {
 	// Try to find the Application Service user
 	token, err := ExtractAccessToken(req)
 	if err != nil {
@@ -72,105 +60,31 @@ func VerifyUserFromRequest(
 			JSON: jsonerror.MissingToken(err.Error()),
 		}
 	}
-
+	var res api.QueryAccessTokenResponse
-	// Search for app service with given access_token
+	err = userAPI.QueryAccessToken(req.Context(), &api.QueryAccessTokenRequest{
-	var appService *config.ApplicationService
+		AccessToken:      token,
-	for _, as := range data.AppServices {
+		AppServiceUserID: req.URL.Query().Get("user_id"),
-		if as.ASToken == token {
+	}, &res)
-			appService = &as
+	if err != nil {
-			break
+		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccessToken failed")
-		}
+		jsonErr := jsonerror.InternalServerError()
+		return nil, &jsonErr
 	}
-
+	if res.Err != nil {
-	if appService != nil {
+		if forbidden, ok := res.Err.(*api.ErrorForbidden); ok {
-		// Create a dummy device for AS user
-		dev := authtypes.Device{
-			// Use AS dummy device ID
-			ID: types.AppServiceDeviceID,
-			// AS dummy device has AS's token.
-			AccessToken: token,
-		}
-
-		userID := req.URL.Query().Get("user_id")
-		localpart, err := userutil.ParseUsernameParam(userID, nil)
-		if err != nil {
-			return nil, &util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidUsername(err.Error()),
-			}
-		}
-
-		if localpart != "" { // AS is masquerading as another user
-			// Verify that the user is registered
-			account, err := data.AccountDB.GetAccountByLocalpart(req.Context(), localpart)
-			// Verify that account exists & appServiceID matches
-			if err == nil && account.AppServiceID == appService.ID {
-				// Set the userID of dummy device
-				dev.UserID = userID
-				return &dev, nil
-			}
-
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("Application service has not registered this user"),
+				JSON: jsonerror.Forbidden(forbidden.Message),
 			}
 		}
-
-		// AS is not masquerading as any user, so use AS's sender_localpart
-		dev.UserID = appService.SenderLocalpart
-		return &dev, nil
 	}
-
+	if res.Device == nil {
-	// Try to find local user from device database
+		return nil, &util.JSONResponse{
-	dev, devErr := verifyAccessToken(req, data.DeviceDB)
-	if devErr == nil {
-		return dev, verifyUserParameters(req)
-	}
-
-	return nil, &util.JSONResponse{
-		Code: http.StatusUnauthorized,
-		JSON: jsonerror.UnknownToken("Unrecognized access token"), // nolint: misspell
-	}
-}
-
-// verifyUserParameters ensures that a request coming from a regular user is not
-// using any query parameters reserved for an application service
-func verifyUserParameters(req *http.Request) *util.JSONResponse {
-	if req.URL.Query().Get("ts") != "" {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown("parameter 'ts' not allowed without valid parameter 'access_token'"),
-		}
-	}
-	return nil
-}
-
-// verifyAccessToken verifies that an access token was supplied in the given HTTP request
-// and returns the device it corresponds to. Returns resErr (an error response which can be
-// sent to the client) if the token is invalid or there was a problem querying the database.
-func verifyAccessToken(req *http.Request, deviceDB DeviceDatabase) (device *authtypes.Device, resErr *util.JSONResponse) {
-	token, err := ExtractAccessToken(req)
-	if err != nil {
-		resErr = &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.MissingToken(err.Error()),
+			JSON: jsonerror.UnknownToken("Unknown token"),
-		}
-		return
-	}
-	device, err = deviceDB.GetDeviceByAccessToken(req.Context(), token)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			resErr = &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: jsonerror.UnknownToken("Unknown token"),
-			}
-		} else {
-			util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDeviceByAccessToken failed")
-			jsonErr := jsonerror.InternalServerError()
-			resErr = &jsonErr
 		}
 	}
-	return
+	return res.Device, nil
 }
 
 // GenerateAccessToken creates a new access token. Returns an error if failed to generate

clientapi/auth/authtypes/device.go

@@ -1,30 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package authtypes
-
-// Device represents a client's device (mobile, web, etc)
-type Device struct {
-	ID     string
-	UserID string
-	// The access_token granted to this device.
-	// This uniquely identifies the device from all other devices and clients.
-	AccessToken string
-	// The unique ID of the session identified by the access token.
-	// Can be used as a secure substitution in places where data needs to be
-	// associated with access tokens.
-	SessionID int64
-	// TODO: display name, last used timestamp, keys, etc
-	DisplayName string
-}

clientapi/clientapi.go

@@ -18,8 +18,6 @@ import (
 	"github.com/Shopify/sarama"
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/consumers"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/routing"
@@ -28,6 +26,9 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
@@ -41,12 +42,12 @@ func AddPublicRoutes(
 	deviceDB devices.Database,
 	accountsDB accounts.Database,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing *gomatrixserverlib.KeyRing,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	eduInputAPI eduServerAPI.EDUServerInputAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	transactionsCache *transactions.Cache,
 	fsAPI federationSenderAPI.FederationSenderInternalAPI,
+	userAPI userapi.UserInternalAPI,
 ) {
 	syncProducer := &producers.SyncAPIProducer{
 		Producer: producer,
@@ -62,7 +63,7 @@ func AddPublicRoutes(
 
 	routing.Setup(
 		router, cfg, eduInputAPI, rsAPI, asAPI,
-		accountsDB, deviceDB, federation, *keyRing,
+		accountsDB, deviceDB, userAPI, federation,
 		syncProducer, transactionsCache, fsAPI,
 	)
 }

clientapi/consumers/roomserver.go

@@ -18,10 +18,10 @@ import (
 	"context"
 	"encoding/json"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/Shopify/sarama"
@@ -84,63 +84,9 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 		return nil
 	}
 
-	ev := output.NewRoomEvent.Event
+	return s.db.UpdateMemberships(
-	log.WithFields(log.Fields{
+		context.TODO(),
-		"event_id": ev.EventID(),
+		gomatrixserverlib.UnwrapEventHeaders(output.NewRoomEvent.AddsState()),
-		"room_id":  ev.RoomID(),
+		output.NewRoomEvent.RemovesStateEventIDs,
-		"type":     ev.Type(),
+	)
-	}).Info("received event from roomserver")
-
-	events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event)
-	if err != nil {
-		return err
-	}
-
-	return s.db.UpdateMemberships(context.TODO(), events, output.NewRoomEvent.RemovesStateEventIDs)
-}
-
-// lookupStateEvents looks up the state events that are added by a new event.
-func (s *OutputRoomEventConsumer) lookupStateEvents(
-	addsStateEventIDs []string, event gomatrixserverlib.Event,
-) ([]gomatrixserverlib.Event, error) {
-	// Fast path if there aren't any new state events.
-	if len(addsStateEventIDs) == 0 {
-		// If the event is a membership update (e.g. for a profile update), it won't
-		// show up in AddsStateEventIDs, so we need to add it manually
-		if event.Type() == "m.room.member" {
-			return []gomatrixserverlib.Event{event}, nil
-		}
-		return nil, nil
-	}
-
-	// Fast path if the only state event added is the event itself.
-	if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() {
-		return []gomatrixserverlib.Event{event}, nil
-	}
-
-	result := []gomatrixserverlib.Event{}
-	missing := []string{}
-	for _, id := range addsStateEventIDs {
-		// Append the current event in the results if its ID is in the events list
-		if id == event.EventID() {
-			result = append(result, event)
-		} else {
-			// If the event isn't the current one, add it to the list of events
-			// to retrieve from the roomserver
-			missing = append(missing, id)
-		}
-	}
-
-	// Request the missing events from the roomserver
-	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
-	var eventResp api.QueryEventsByIDResponse
-	if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
-		return nil, err
-	}
-
-	for _, headeredEvent := range eventResp.Events {
-		result = append(result, headeredEvent.Event)
-	}
-
-	return result, nil
 }

clientapi/producers/syncapi.go

@@ -17,9 +17,9 @@ package producers
 import (
 	"encoding/json"
 
-	"github.com/matrix-org/dendrite/internal"
-
 	"github.com/Shopify/sarama"
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	log "github.com/sirupsen/logrus"
 )
 
 // SyncAPIProducer produces events for the sync API server to consume
@@ -32,7 +32,7 @@ type SyncAPIProducer struct {
 func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string) error {
 	var m sarama.ProducerMessage
 
-	data := internal.AccountData{
+	data := eventutil.AccountData{
 		RoomID: roomID,
 		Type:   dataType,
 	}
@@ -44,6 +44,11 @@ func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string
 	m.Topic = string(p.Topic)
 	m.Key = sarama.StringEncoder(userID)
 	m.Value = sarama.ByteEncoder(value)
+	log.WithFields(log.Fields{
+		"user_id":   userID,
+		"room_id":   roomID,
+		"data_type": dataType,
+	}).Infof("Producing to topic '%s'", p.Topic)
 
 	_, _, err = p.Producer.SendMessage(&m)
 	return err

clientapi/routing/account_data.go

@@ -19,10 +19,10 @@ import (
 	"io/ioutil"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/util"
@@ -30,7 +30,7 @@ import (
 
 // GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type}
 func GetAccountData(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *api.Device,
 	userID string, roomID string, dataType string,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -51,7 +51,7 @@ func GetAccountData(
 	); err == nil {
 		return util.JSONResponse{
 			Code: http.StatusOK,
-			JSON: data,
+			JSON: data.Content,
 		}
 	}
 
@@ -63,7 +63,7 @@ func GetAccountData(
 
 // SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type}
 func SaveAccountData(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *api.Device,
 	userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
 ) util.JSONResponse {
 	if userID != device.UserID {

clientapi/routing/createroom.go

@@ -24,14 +24,14 @@ import (
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/dendrite/userapi/api"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	log "github.com/sirupsen/logrus"
@@ -98,7 +98,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 
 	// Validate creation_content fields defined in the spec by marshalling the
 	// creation_content map into bytes and then unmarshalling the bytes into
-	// internal.CreateContent.
+	// eventutil.CreateContent.
 
 	creationContentBytes, err := json.Marshal(r.CreationContent)
 	if err != nil {
@@ -135,7 +135,7 @@ type fledglingEvent struct {
 
 // CreateRoom implements /createRoom
 func CreateRoom(
-	req *http.Request, device *authtypes.Device,
+	req *http.Request, device *api.Device,
 	cfg *config.Dendrite,
 	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
@@ -149,7 +149,7 @@ func CreateRoom(
 // createRoom implements /createRoom
 // nolint: gocyclo
 func createRoom(
-	req *http.Request, device *authtypes.Device,
+	req *http.Request, device *api.Device,
 	cfg *config.Dendrite, roomID string,
 	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
@@ -279,25 +279,25 @@ func createRoom(
 	eventsToMake := []fledglingEvent{
 		{"m.room.create", "", r.CreationContent},
 		{"m.room.member", userID, membershipContent},
-		{"m.room.power_levels", "", internal.InitialPowerLevelsContent(userID)},
+		{"m.room.power_levels", "", eventutil.InitialPowerLevelsContent(userID)},
 		{"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}},
-		{"m.room.history_visibility", "", internal.HistoryVisibilityContent{HistoryVisibility: historyVisibility}},
+		{"m.room.history_visibility", "", eventutil.HistoryVisibilityContent{HistoryVisibility: historyVisibility}},
 	}
 	if roomAlias != "" {
 		// TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room.
 		// This means we might fail creating the alias but say the canonical alias is something that doesn't exist.
 		// m.room.aliases is handled when we call roomserver.SetRoomAlias
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", internal.CanonicalAlias{Alias: roomAlias}})
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", eventutil.CanonicalAlias{Alias: roomAlias}})
 	}
 	if r.GuestCanJoin {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", internal.GuestAccessContent{GuestAccess: "can_join"}})
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", eventutil.GuestAccessContent{GuestAccess: "can_join"}})
 	}
 	eventsToMake = append(eventsToMake, r.InitialState...)
 	if r.Name != "" {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", internal.NameContent{Name: r.Name}})
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", eventutil.NameContent{Name: r.Name}})
 	}
 	if r.Topic != "" {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", internal.TopicContent{Topic: r.Topic}})
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", eventutil.TopicContent{Topic: r.Topic}})
 	}
 	// TODO: invite events
 	// TODO: 3pid invite events

clientapi/routing/device.go

@@ -19,9 +19,9 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -47,7 +47,7 @@ type devicesDeleteJSON struct {
 
 // GetDeviceByID handles /devices/{deviceID}
 func GetDeviceByID(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -78,7 +78,7 @@ func GetDeviceByID(
 
 // GetDevicesByLocalpart handles /devices
 func GetDevicesByLocalpart(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
@@ -110,7 +110,7 @@ func GetDevicesByLocalpart(
 
 // UpdateDeviceByID handles PUT on /devices/{deviceID}
 func UpdateDeviceByID(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -160,7 +160,7 @@ func UpdateDeviceByID(
 
 // DeleteDeviceById handles DELETE requests to /devices/{deviceId}
 func DeleteDeviceById(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -185,7 +185,7 @@ func DeleteDeviceById(
 
 // DeleteDevices handles POST requests to /delete_devices
 func DeleteDevices(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {

clientapi/routing/directory.go

@@ -18,12 +18,12 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -112,7 +112,7 @@ func DirectoryRoom(
 // TODO: Check if the user has the power level to set an alias
 func SetLocalAlias(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	alias string,
 	cfg *config.Dendrite,
 	aliasAPI roomserverAPI.RoomserverInternalAPI,
@@ -188,7 +188,7 @@ func SetLocalAlias(
 // RemoveLocalAlias implements DELETE /directory/room/{roomAlias}
 func RemoveLocalAlias(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	alias string,
 	aliasAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {

clientapi/routing/filter.go

@@ -17,17 +17,17 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 // GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId}
 func GetFilter(
-	req *http.Request, device *authtypes.Device, accountDB accounts.Database, userID string, filterID string,
+	req *http.Request, device *api.Device, accountDB accounts.Database, userID string, filterID string,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -64,7 +64,7 @@ type filterResponse struct {
 
 //PutFilter implements POST /_matrix/client/r0/user/{userId}/filter
 func PutFilter(
-	req *http.Request, device *authtypes.Device, accountDB accounts.Database, userID string,
+	req *http.Request, device *api.Device, accountDB accounts.Database, userID string,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{

clientapi/routing/getevent.go

@@ -17,22 +17,21 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 type getEventRequest struct {
 	req            *http.Request
-	device         *authtypes.Device
+	device         *userapi.Device
 	roomID         string
 	eventID        string
 	cfg            *config.Dendrite
 	federation     *gomatrixserverlib.FederationClient
-	keyRing        gomatrixserverlib.KeyRing
 	requestedEvent gomatrixserverlib.Event
 }
 
@@ -40,13 +39,12 @@ type getEventRequest struct {
 // https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid
 func GetEvent(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	roomID string,
 	eventID string,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing gomatrixserverlib.KeyRing,
 ) util.JSONResponse {
 	eventsReq := api.QueryEventsByIDRequest{
 		EventIDs: []string{eventID},
@@ -75,7 +73,6 @@ func GetEvent(
 		eventID:        eventID,
 		cfg:            cfg,
 		federation:     federation,
-		keyRing:        keyRing,
 		requestedEvent: requestedEvent,
 	}
 

clientapi/routing/joinroom.go

@@ -17,18 +17,18 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 func JoinRoomByIDOrAlias(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	accountDB accounts.Database,
 	roomIDOrAlias string,
@@ -43,9 +43,7 @@ func JoinRoomByIDOrAlias(
 	// If content was provided in the request then incude that
 	// in the request. It'll get used as a part of the membership
 	// event content.
-	if err := httputil.UnmarshalJSONRequest(req, &joinReq.Content); err != nil {
+	_ = httputil.UnmarshalJSONRequest(req, &joinReq.Content)
-		return *err
-	}
 
 	// Work out our localpart for the client profile request.
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)

clientapi/routing/leaveroom.go

@@ -17,15 +17,15 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
 func LeaveRoomByID(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	roomID string,
 ) util.JSONResponse {

clientapi/routing/login.go

@@ -22,13 +22,13 @@ import (
 	"context"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -49,9 +49,8 @@ type loginIdentifier struct {
 
 type loginWithPasswordRequest struct {
 	Identifier loginIdentifier `json:"identifier"`
-	Medium     string          `json:"medium"`   // third-party only
+	User       string          `json:"user"` // deprecated in favour of identifier
-	Password   string          `json:"password"` // m.login.password only
+	Password   string          `json:"password"`
-	Token      string          `json:"token"`    // m.login.token only
 	// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
 	// Thus a pointer is needed to differentiate between the two
 	InitialDisplayName *string `json:"initial_device_display_name"`
@@ -88,9 +87,10 @@ func Login(
 			JSON: passwordLogin(),
 		}
 	} else if req.Method == http.MethodPost {
-		var temp interface{}
+		var r passwordRequest
-		var acc *authtypes.Account
+		var acc *api.Account
-		resErr := httputil.UnmarshalJSONRequest(req, &temp)
+		var errJSON *util.JSONResponse
+		resErr := httputil.UnmarshalJSONRequest(req, &r)
 		if resErr != nil {
 			return *resErr
 		}
@@ -113,43 +113,23 @@ func Login(
 						JSON: jsonerror.BadJSON("'user' must be supplied."),
 					}
 				}
-
+			}
-				util.GetLogger(req.Context()).WithField("user", r.Identifier.User).Info("Processing login request")
+			acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.Identifier.User)
-
+			if errJSON != nil {
-				localpart, err := userutil.ParseUsernameParam(r.Identifier.User, &cfg.Matrix.ServerName)
+				return *errJSON
-				if err != nil {
-					return util.JSONResponse{
-						Code: http.StatusBadRequest,
-						JSON: jsonerror.InvalidUsername(err.Error()),
-					}
-				}
-
-				acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password)
-				if err != nil {
-					// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
-					// but that would leak the existence of the user.
-					return util.JSONResponse{
-						Code: http.StatusForbidden,
-						JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"),
-					}
-				}
-
-			case "m.login.token":
-				return util.JSONResponse{
-					Code: http.StatusBadRequest,
-					JSON: jsonerror.Unknown("Token login is not supported"),
-				}
-
-			default:
-				return util.JSONResponse{
-					Code: http.StatusBadRequest,
-					JSON: jsonerror.Unknown("login identifier '" + r.Identifier.Type + "' not supported"),
-				}
 			}
 		default:
-			return util.JSONResponse{
+			// TODO: The below behaviour is deprecated but without it Riot iOS won't log in
-				Code: http.StatusBadRequest,
+			if r.User != "" {
-				JSON: jsonerror.Unknown(fmt.Sprintf("Login type %q not supported", r.Type)),
+				acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.User)
+				if errJSON != nil {
+					return *errJSON
+				}
+			} else {
+				return util.JSONResponse{
+					Code: http.StatusBadRequest,
+					JSON: jsonerror.BadJSON("login identifier '" + r.Identifier.Type + "' not supported"),
+				}
 			}
 		}
 
@@ -188,11 +168,40 @@ func getDevice(
 	ctx context.Context,
 	r loginWithPasswordRequest,
 	deviceDB devices.Database,
-	acc *authtypes.Account,
+	acc *api.Account,
 	token string,
-) (dev *authtypes.Device, err error) {
+) (dev *api.Device, err error) {
 	dev, err = deviceDB.CreateDevice(
 		ctx, acc.Localpart, r.DeviceID, token, r.InitialDisplayName,
 	)
 	return
 }
+
+func (r *passwordRequest) processUsernamePasswordLoginRequest(
+	req *http.Request, accountDB accounts.Database,
+	cfg *config.Dendrite, username string,
+) (acc *api.Account, errJSON *util.JSONResponse) {
+	util.GetLogger(req.Context()).WithField("user", username).Info("Processing login request")
+
+	localpart, err := userutil.ParseUsernameParam(username, &cfg.Matrix.ServerName)
+	if err != nil {
+		errJSON = &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidUsername(err.Error()),
+		}
+		return
+	}
+
+	acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password)
+	if err != nil {
+		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
+		// but that would leak the existence of the user.
+		errJSON = &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"),
+		}
+		return
+	}
+
+	return
+}

clientapi/routing/logout.go

@@ -17,16 +17,16 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 // Logout handles POST /logout
 func Logout(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
@@ -47,7 +47,7 @@ func Logout(
 
 // LogoutAll handles POST /logout/all
 func LogoutAll(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {

clientapi/routing/membership.go

@@ -22,14 +22,15 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/util"
@@ -42,7 +43,7 @@ var errMissingUserID = errors.New("'user_id' must be supplied")
 // TODO: Can we improve the cyclo count here? Separate code paths for invites?
 // nolint:gocyclo
 func SendMembership(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
 	roomID string, membership string, cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -95,7 +96,7 @@ func SendMembership(
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.BadJSON(err.Error()),
 		}
-	} else if err == internal.ErrRoomNoExists {
+	} else if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound(err.Error()),
@@ -149,7 +150,7 @@ func SendMembership(
 func buildMembershipEvent(
 	ctx context.Context,
 	body threepid.MembershipRequest, accountDB accounts.Database,
-	device *authtypes.Device,
+	device *userapi.Device,
 	membership, roomID string, isDirect bool,
 	cfg *config.Dendrite, evTime time.Time,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
@@ -188,7 +189,7 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
-	return internal.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
+	return eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
 }
 
 // loadProfile lookups the profile of a given user from the database and returns
@@ -223,7 +224,7 @@ func loadProfile(
 // In the latter case, if there was an issue retrieving the user ID from the request body,
 // returns a JSONResponse with a corresponding error code and message.
 func getMembershipStateKey(
-	body threepid.MembershipRequest, device *authtypes.Device, membership string,
+	body threepid.MembershipRequest, device *userapi.Device, membership string,
 ) (stateKey string, reason string, err error) {
 	if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite {
 		// If we're in this case, the state key is contained in the request body,
@@ -245,7 +246,7 @@ func getMembershipStateKey(
 
 func checkAndProcessThreepid(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	body *threepid.MembershipRequest,
 	cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
@@ -268,7 +269,7 @@ func checkAndProcessThreepid(
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.NotTrusted(body.IDServer),
 		}
-	} else if err == internal.ErrRoomNoExists {
+	} else if err == eventutil.ErrRoomNoExists {
 		return inviteStored, &util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound(err.Error()),

clientapi/routing/memberships.go

@@ -15,14 +15,15 @@
 package routing
 
 import (
+	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -35,9 +36,19 @@ type getJoinedRoomsResponse struct {
 	JoinedRooms []string `json:"joined_rooms"`
 }
 
+// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
+type getJoinedMembersResponse struct {
+	Joined map[string]joinedMember `json:"joined"`
+}
+
+type joinedMember struct {
+	DisplayName string `json:"display_name"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
 // GetMemberships implements GET /rooms/{roomId}/members
 func GetMemberships(
-	req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool,
+	req *http.Request, device *userapi.Device, roomID string, joinedOnly bool,
 	_ *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
@@ -59,6 +70,22 @@ func GetMemberships(
 		}
 	}
 
+	if joinedOnly {
+		var res getJoinedMembersResponse
+		res.Joined = make(map[string]joinedMember)
+		for _, ev := range queryRes.JoinEvents {
+			var content joinedMember
+			if err := json.Unmarshal(ev.Content, &content); err != nil {
+				util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
+				return jsonerror.InternalServerError()
+			}
+			res.Joined[ev.Sender] = content
+		}
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: res,
+		}
+	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: getMembershipResponse{queryRes.JoinEvents},
@@ -67,7 +94,7 @@ func GetMemberships(
 
 func GetJoinedRooms(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	accountsDB accounts.Database,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)

clientapi/routing/profile.go

@@ -21,12 +21,13 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/gomatrix"
@@ -42,7 +43,7 @@ func GetProfile(
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
 	if err != nil {
-		if err == internal.ErrProfileNoExists {
+		if err == eventutil.ErrProfileNoExists {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
 				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
@@ -55,7 +56,7 @@ func GetProfile(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: internal.ProfileResponse{
+		JSON: eventutil.ProfileResponse{
 			AvatarURL:   profile.AvatarURL,
 			DisplayName: profile.DisplayName,
 		},
@@ -70,7 +71,7 @@ func GetAvatarURL(
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
 	if err != nil {
-		if err == internal.ErrProfileNoExists {
+		if err == eventutil.ErrProfileNoExists {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
 				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
@@ -83,7 +84,7 @@ func GetAvatarURL(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: internal.AvatarURL{
+		JSON: eventutil.AvatarURL{
 			AvatarURL: profile.AvatarURL,
 		},
 	}
@@ -92,7 +93,7 @@ func GetAvatarURL(
 // SetAvatarURL implements PUT /profile/{userID}/avatar_url
 // nolint:gocyclo
 func SetAvatarURL(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
 	userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -102,7 +103,7 @@ func SetAvatarURL(
 		}
 	}
 
-	var r internal.AvatarURL
+	var r eventutil.AvatarURL
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
@@ -184,7 +185,7 @@ func GetDisplayName(
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
 	if err != nil {
-		if err == internal.ErrProfileNoExists {
+		if err == eventutil.ErrProfileNoExists {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
 				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
@@ -197,7 +198,7 @@ func GetDisplayName(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: internal.DisplayName{
+		JSON: eventutil.DisplayName{
 			DisplayName: profile.DisplayName,
 		},
 	}
@@ -206,7 +207,7 @@ func GetDisplayName(
 // SetDisplayName implements PUT /profile/{userID}/displayname
 // nolint:gocyclo
 func SetDisplayName(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
 	userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -216,7 +217,7 @@ func SetDisplayName(
 		}
 	}
 
-	var r internal.DisplayName
+	var r eventutil.DisplayName
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
@@ -293,7 +294,7 @@ func SetDisplayName(
 // getProfile gets the full profile of a user by querying the database or a
 // remote homeserver.
 // Returns an error when something goes wrong or specifically
-// internal.ErrProfileNoExists when the profile doesn't exist.
+// eventutil.ErrProfileNoExists when the profile doesn't exist.
 func getProfile(
 	ctx context.Context, accountDB accounts.Database, cfg *config.Dendrite,
 	userID string,
@@ -310,7 +311,7 @@ func getProfile(
 		if fedErr != nil {
 			if x, ok := fedErr.(gomatrix.HTTPError); ok {
 				if x.Code == http.StatusNotFound {
-					return nil, internal.ErrProfileNoExists
+					return nil, eventutil.ErrProfileNoExists
 				}
 			}
 
@@ -365,7 +366,7 @@ func buildMembershipEvents(
 			return nil, err
 		}
 
-		event, err := internal.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
+		event, err := eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
 		if err != nil {
 			return nil, err
 		}

clientapi/routing/register.go

@@ -33,15 +33,15 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
-	"github.com/matrix-org/dendrite/internal"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/tokens"
 	"github.com/matrix-org/util"
@@ -136,7 +136,7 @@ type registerRequest struct {
 	DeviceID           *string `json:"device_id"`
 
 	// Prevent this user from logging in
-	InhibitLogin internal.WeakBoolean `json:"inhibit_login"`
+	InhibitLogin eventutil.WeakBoolean `json:"inhibit_login"`
 
 	// Application Services place Type in the root of their registration
 	// request, whereas clients place it in the authDict struct.
@@ -440,8 +440,8 @@ func validateApplicationService(
 // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
 func Register(
 	req *http.Request,
+	userAPI userapi.UserInternalAPI,
 	accountDB accounts.Database,
-	deviceDB devices.Database,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var r registerRequest
@@ -450,7 +450,7 @@ func Register(
 		return *resErr
 	}
 	if req.URL.Query().Get("kind") == "guest" {
-		return handleGuestRegistration(req, r, cfg, accountDB, deviceDB)
+		return handleGuestRegistration(req, r, cfg, userAPI)
 	}
 
 	// Retrieve or generate the sessionID
@@ -506,21 +506,19 @@ func Register(
 		"session_id": r.Auth.Session,
 	}).Info("Processing registration request")
 
-	resp := handleRegistrationFlow(req, r, sessionID, cfg, accountDB, deviceDB)
+	return handleRegistrationFlow(req, r, sessionID, cfg, userAPI)
-	j, _ := json.MarshalIndent(resp, "", "  ")
-	fmt.Println("ERROR!")
-	fmt.Println(string(j))
-	return resp
 }
 
 func handleGuestRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 ) util.JSONResponse {
-	acc, err := accountDB.CreateGuestAccount(req.Context())
+	var res userapi.PerformAccountCreationResponse
+	err := userAPI.PerformAccountCreation(req.Context(), &userapi.PerformAccountCreationRequest{
+		AccountType: userapi.AccountTypeGuest,
+	}, &res)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -529,8 +527,8 @@ func handleGuestRegistration(
 	}
 	token, err := tokens.GenerateLoginToken(tokens.TokenOptions{
 		ServerPrivateKey: cfg.Matrix.PrivateKey.Seed(),
-		ServerName:       string(acc.ServerName),
+		ServerName:       string(res.Account.ServerName),
-		UserID:           acc.UserID,
+		UserID:           res.Account.UserID,
 	})
 
 	if err != nil {
@@ -540,7 +538,12 @@ func handleGuestRegistration(
 		}
 	}
 	//we don't allow guests to specify their own device_id
-	dev, err := deviceDB.CreateDevice(req.Context(), acc.Localpart, nil, token, r.InitialDisplayName)
+	var devRes userapi.PerformDeviceCreationResponse
+	err = userAPI.PerformDeviceCreation(req.Context(), &userapi.PerformDeviceCreationRequest{
+		Localpart:         res.Account.Localpart,
+		DeviceDisplayName: r.InitialDisplayName,
+		AccessToken:       token,
+	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -550,10 +553,10 @@ func handleGuestRegistration(
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: registerResponse{
-			UserID:      dev.UserID,
+			UserID:      devRes.Device.UserID,
-			AccessToken: dev.AccessToken,
+			AccessToken: devRes.Device.AccessToken,
-			HomeServer:  acc.ServerName,
+			HomeServer:  res.Account.ServerName,
-			DeviceID:    dev.ID,
+			DeviceID:    devRes.Device.ID,
 		},
 	}
 }
@@ -566,8 +569,7 @@ func handleRegistrationFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 ) util.JSONResponse {
 	// TODO: Shared secret registration (create new user scripts)
 	// TODO: Enable registration config flag
@@ -618,7 +620,7 @@ func handleRegistrationFlow(
 		// by whether the request contains an access token.
 		if err == nil {
 			return handleApplicationServiceRegistration(
-				accessToken, err, req, r, cfg, accountDB, deviceDB,
+				accessToken, err, req, r, cfg, userAPI,
 			)
 		}
 
@@ -629,7 +631,7 @@ func handleRegistrationFlow(
 		// don't need a condition on that call since the registration is clearly
 		// stated as being AS-related.
 		return handleApplicationServiceRegistration(
-			accessToken, err, req, r, cfg, accountDB, deviceDB,
+			accessToken, err, req, r, cfg, userAPI,
 		)
 
 	case authtypes.LoginTypeDummy:
@@ -648,7 +650,7 @@ func handleRegistrationFlow(
 	// A response with current registration flow and remaining available methods
 	// will be returned if a flow has not been successfully completed yet
 	return checkAndCompleteFlow(sessions.GetCompletedStages(sessionID),
-		req, r, sessionID, cfg, accountDB, deviceDB)
+		req, r, sessionID, cfg, userAPI)
 }
 
 // handleApplicationServiceRegistration handles the registration of an
@@ -665,8 +667,7 @@ func handleApplicationServiceRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 ) util.JSONResponse {
 	// Check if we previously had issues extracting the access token from the
 	// request.
@@ -690,7 +691,7 @@ func handleApplicationServiceRegistration(
 	// Don't need to worry about appending to registration stages as
 	// application service registration is entirely separate.
 	return completeRegistration(
-		req.Context(), accountDB, deviceDB, r.Username, "", appserviceID,
+		req.Context(), userAPI, r.Username, "", appserviceID,
 		r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 	)
 }
@@ -704,13 +705,12 @@ func checkAndCompleteFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 ) util.JSONResponse {
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
 		return completeRegistration(
-			req.Context(), accountDB, deviceDB, r.Username, r.Password, "",
+			req.Context(), userAPI, r.Username, r.Password, "",
 			r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 		)
 	}
@@ -727,8 +727,7 @@ func checkAndCompleteFlow(
 // LegacyRegister process register requests from the legacy v1 API
 func LegacyRegister(
 	req *http.Request,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var r legacyRegisterRequest
@@ -763,10 +762,10 @@ func LegacyRegister(
 			return util.MessageResponse(http.StatusForbidden, "HMAC incorrect")
 		}
 
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
+		return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil)
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
+		return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil)
 	default:
 		return util.JSONResponse{
 			Code: http.StatusNotImplemented,
@@ -812,10 +811,9 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u
 // not all
 func completeRegistration(
 	ctx context.Context,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 	username, password, appserviceID string,
-	inhibitLogin internal.WeakBoolean,
+	inhibitLogin eventutil.WeakBoolean,
 	displayName, deviceID *string,
 ) util.JSONResponse {
 	if username == "" {
@@ -832,9 +830,16 @@ func completeRegistration(
 		}
 	}
 
-	acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID)
+	var accRes userapi.PerformAccountCreationResponse
+	err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{
+		AppServiceID: appserviceID,
+		Localpart:    username,
+		Password:     password,
+		AccountType:  userapi.AccountTypeUser,
+		OnConflict:   userapi.ConflictAbort,
+	}, &accRes)
 	if err != nil {
-		if errors.Is(err, internal.ErrUserExists) { // user already exists
+		if _, ok := err.(*userapi.ErrorConflict); ok { // user already exists
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: jsonerror.UserInUse("Desired user ID is already taken."),
@@ -855,8 +860,8 @@ func completeRegistration(
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: registerResponse{
-				UserID:     userutil.MakeUserID(username, acc.ServerName),
+				UserID:     userutil.MakeUserID(username, accRes.Account.ServerName),
-				HomeServer: acc.ServerName,
+				HomeServer: accRes.Account.ServerName,
 			},
 		}
 	}
@@ -869,7 +874,13 @@ func completeRegistration(
 		}
 	}
 
-	dev, err := deviceDB.CreateDevice(ctx, username, deviceID, token, displayName)
+	var devRes userapi.PerformDeviceCreationResponse
+	err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
+		Localpart:         username,
+		AccessToken:       token,
+		DeviceDisplayName: displayName,
+		DeviceID:          deviceID,
+	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -880,10 +891,10 @@ func completeRegistration(
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: registerResponse{
-			UserID:      dev.UserID,
+			UserID:      devRes.Device.UserID,
-			AccessToken: dev.AccessToken,
+			AccessToken: devRes.Device.AccessToken,
-			HomeServer:  acc.ServerName,
+			HomeServer:  accRes.Account.ServerName,
-			DeviceID:    dev.ID,
+			DeviceID:    devRes.Device.ID,
 		},
 	}
 }

clientapi/routing/room_tagging.go

@@ -20,11 +20,11 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -41,7 +41,7 @@ func newTag() gomatrix.TagContent {
 func GetTags(
 	req *http.Request,
 	accountDB accounts.Database,
-	device *authtypes.Device,
+	device *api.Device,
 	userID string,
 	roomID string,
 	syncProducer *producers.SyncAPIProducer,
@@ -79,7 +79,7 @@ func GetTags(
 func PutTag(
 	req *http.Request,
 	accountDB accounts.Database,
-	device *authtypes.Device,
+	device *api.Device,
 	userID string,
 	roomID string,
 	tag string,
@@ -139,7 +139,7 @@ func PutTag(
 func DeleteTag(
 	req *http.Request,
 	accountDB accounts.Database,
-	device *authtypes.Device,
+	device *api.Device,
 	userID string,
 	roomID string,
 	tag string,

clientapi/routing/routing.go

@@ -21,18 +21,17 @@ import (
 
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -54,15 +53,15 @@ func Setup(
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	accountDB accounts.Database,
 	deviceDB devices.Database,
+	userAPI api.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing gomatrixserverlib.KeyRing,
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
 	federationSender federationSenderAPI.FederationSenderInternalAPI,
 ) {
 
 	publicAPIMux.Handle("/client/versions",
-		internal.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusOK,
 				JSON: struct {
@@ -81,20 +80,14 @@ func Setup(
 	v1mux := publicAPIMux.PathPrefix(pathPrefixV1).Subrouter()
 	unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter()
 
-	authData := auth.Data{
-		AccountDB:   accountDB,
-		DeviceDB:    deviceDB,
-		AppServices: cfg.Derived.ApplicationServices,
-	}
-
 	r0mux.Handle("/createRoom",
-		internal.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
-		internal.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -104,13 +97,13 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/joined_rooms",
-		internal.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetJoinedRooms(req, device, accountDB)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/leave",
-		internal.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -120,8 +113,8 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}",
-		internal.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -129,8 +122,8 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
-		internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -138,8 +131,8 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
-		internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -149,42 +142,44 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/event/{eventID}",
-		internal.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing)
+			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-		vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
 		return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type}", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state/{type}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-		vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "")
+		eventFormat := req.URL.Query().Get("format") == "event"
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "", eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-		vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"])
+		eventFormat := req.URL.Query().Get("format") == "event"
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"], eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
-		internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -199,8 +194,8 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
-		internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -209,21 +204,21 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/register", internal.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
+	r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
-		return Register(req, accountDB, deviceDB, cfg)
+		return Register(req, userAPI, accountDB, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
-	v1mux.Handle("/register", internal.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
+	v1mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
-		return LegacyRegister(req, accountDB, deviceDB, cfg)
+		return LegacyRegister(req, userAPI, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/register/available", internal.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
+	r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
 		return RegisterAvailable(req, cfg, accountDB)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		internal.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -232,8 +227,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		internal.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -242,8 +237,8 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		internal.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -252,20 +247,20 @@ func Setup(
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/logout",
-		internal.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return Logout(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/logout/all",
-		internal.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return LogoutAll(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
-		internal.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -274,8 +269,8 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/sendToDevice/{eventType}/{txnID}",
-		internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -288,8 +283,8 @@ func Setup(
 	// rather than r0. It's an exact duplicate of the above handler.
 	// TODO: Remove this if/when sytest is fixed!
 	unstableMux.Handle("/sendToDevice/{eventType}/{txnID}",
-		internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -299,7 +294,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/account/whoami",
-		internal.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return Whoami(req, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -307,20 +302,20 @@ func Setup(
 	// Stub endpoints required by Riot
 
 	r0mux.Handle("/login",
-		internal.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
 			return Login(req, accountDB, deviceDB, cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/auth/{authType}/fallback/web",
-		internal.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse {
+		httputil.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse {
 			vars := mux.Vars(req)
 			return AuthFallback(w, req, vars["authType"], cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/pushrules/",
-		internal.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse {
 			// TODO: Implement push rules API
 			res := json.RawMessage(`{
 					"global": {
@@ -339,8 +334,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/filter",
-		internal.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -349,8 +344,8 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/filter/{filterId}",
-		internal.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -361,8 +356,8 @@ func Setup(
 	// Riot user settings
 
 	r0mux.Handle("/profile/{userID}",
-		internal.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -371,8 +366,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
-		internal.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -381,8 +376,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
-		internal.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -393,8 +388,8 @@ func Setup(
 	// PUT requests, so we need to allow this method
 
 	r0mux.Handle("/profile/{userID}/displayname",
-		internal.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -403,8 +398,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/displayname",
-		internal.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -415,32 +410,32 @@ func Setup(
 	// PUT requests, so we need to allow this method
 
 	r0mux.Handle("/account/3pid",
-		internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetAssociated3PIDs(req, accountDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/account/3pid",
-		internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return CheckAndSave3PIDAssociation(req, accountDB, device, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	unstableMux.Handle("/account/3pid/delete",
-		internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return Forget3PID(req, accountDB)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken",
-		internal.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse {
 			return RequestEmailToken(req, accountDB, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	// Riot logs get flooded unless this is handled
 	r0mux.Handle("/presence/{userID}/status",
-		internal.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
 			// TODO: Set presence (probably the responsibility of a presence server not clientapi)
 			return util.JSONResponse{
 				Code: http.StatusOK,
@@ -450,13 +445,13 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/voip/turnServer",
-		internal.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return RequestTurnServer(req, device, cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/thirdparty/protocols",
-		internal.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
 			// TODO: Return the third party protcols
 			return util.JSONResponse{
 				Code: http.StatusOK,
@@ -466,7 +461,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/initialSync",
-		internal.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse {
 			// TODO: Allow people to peek into rooms.
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
@@ -476,8 +471,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
-		internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -486,8 +481,8 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
-		internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -496,8 +491,8 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
-		internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -506,8 +501,8 @@ func Setup(
 	).Methods(http.MethodGet)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
-		internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -516,8 +511,8 @@ func Setup(
 	).Methods(http.MethodGet)
 
 	r0mux.Handle("/rooms/{roomID}/members",
-		internal.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -526,8 +521,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/joined_members",
-		internal.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -536,21 +531,21 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/read_markers",
-		internal.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse {
 			// TODO: return the read_markers.
 			return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/devices",
-		internal.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetDevicesByLocalpart(req, deviceDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		internal.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -559,8 +554,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		internal.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -569,8 +564,8 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		internal.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -579,14 +574,14 @@ func Setup(
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/delete_devices",
-		internal.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return DeleteDevices(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	// Stub implementations for sytest
 	r0mux.Handle("/events",
-		internal.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
 				"chunk": []interface{}{},
 				"start": "",
@@ -596,7 +591,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/initialSync",
-		internal.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
 				"end": "",
 			}}
@@ -604,8 +599,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags",
-		internal.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -614,8 +609,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
-		internal.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -624,8 +619,8 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
-		internal.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -634,7 +629,7 @@ func Setup(
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/capabilities",
-		internal.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetCapabilities(req, rsAPI)
 		}),
 	).Methods(http.MethodGet)

clientapi/routing/sendevent.go

@@ -17,13 +17,13 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
@@ -41,7 +41,7 @@ type sendEventResponse struct {
 //   /rooms/{roomID}/state/{eventType}/{stateKey}
 func SendEvent(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	roomID, eventType string, txnID, stateKey *string,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
@@ -110,7 +110,7 @@ func SendEvent(
 
 func generateSendEvent(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	roomID, eventType string, stateKey *string,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
@@ -146,8 +146,8 @@ func generateSendEvent(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := internal.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes)
+	e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes)
-	if err == internal.ErrRoomNoExists {
+	if err == eventutil.ErrRoomNoExists {
 		return nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound("Room does not exist"),
@@ -158,7 +158,7 @@ func generateSendEvent(
 			JSON: jsonerror.BadJSON(e.Error()),
 		}
 	} else if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("internal.BuildEvent failed")
+		util.GetLogger(req.Context()).WithError(err).Error("eventutil.BuildEvent failed")
 		resErr := jsonerror.InternalServerError()
 		return nil, &resErr
 	}

clientapi/routing/sendtodevice.go

@@ -16,18 +16,18 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/internal/transactions"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
 // SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}
 // sends the device events to the EDU Server
 func SendToDevice(
-	req *http.Request, device *authtypes.Device,
+	req *http.Request, device *userapi.Device,
 	eduAPI api.EDUServerInputAPI,
 	txnCache *transactions.Cache,
 	eventType string, txnID *string,

clientapi/routing/sendtyping.go

@@ -16,12 +16,12 @@ import (
 	"database/sql"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/eduserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/util"
 )
 
@@ -33,7 +33,7 @@ type typingContentJSON struct {
 // SendTyping handles PUT /rooms/{roomID}/typing/{userID}
 // sends the typing events to client API typingProducer
 func SendTyping(
-	req *http.Request, device *authtypes.Device, roomID string,
+	req *http.Request, device *userapi.Device, roomID string,
 	userID string, accountDB accounts.Database,
 	eduAPI api.EDUServerInputAPI,
 ) util.JSONResponse {

clientapi/routing/state.go

@@ -98,7 +98,8 @@ func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI
 // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current
 // state to see if there is an event with that type and state key, if there
 // is then (by default) we return the content, otherwise a 404.
-func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string, evType, stateKey string) util.JSONResponse {
+// If eventFormat=true, sends the whole event else just the content.
+func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID, evType, stateKey string, eventFormat bool) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 	util.GetLogger(ctx).WithFields(log.Fields{
@@ -134,8 +135,15 @@ func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInterna
 		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(stateRes.StateEvents[0], gomatrixserverlib.FormatAll),
 	}
 
+	var res interface{}
+	if eventFormat {
+		res = stateEvent
+	} else {
+		res = stateEvent.Content
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: stateEvent.Content,
+		JSON: res,
 	}
 }

clientapi/routing/threepid.go

@@ -18,11 +18,12 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -84,7 +85,7 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf
 
 // CheckAndSave3PIDAssociation implements POST /account/3pid
 func CheckAndSave3PIDAssociation(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *api.Device,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var body threepid.EmailAssociationCheckRequest
@@ -148,7 +149,7 @@ func CheckAndSave3PIDAssociation(
 
 // GetAssociated3PIDs implements GET /account/3pid
 func GetAssociated3PIDs(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {

clientapi/routing/voip.go

@@ -22,16 +22,16 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/util"
 )
 
 // RequestTurnServer implements:
 //     GET /voip/turnServer
-func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg *config.Dendrite) util.JSONResponse {
+func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Dendrite) util.JSONResponse {
 	turnConfig := cfg.TURN
 
 	// TODO Guest Support

clientapi/routing/whoami.go

@@ -15,7 +15,7 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
@@ -26,7 +26,7 @@ type whoamiResponse struct {
 
 // Whoami implements `/account/whoami` which enables client to query their account user id.
 // https://matrix.org/docs/spec/client_server/r0.3.0.html#get-matrix-client-r0-account-whoami
-func Whoami(req *http.Request, device *authtypes.Device) util.JSONResponse {
+func Whoami(req *http.Request, device *api.Device) util.JSONResponse {
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: whoamiResponse{UserID: device.UserID},

clientapi/threepid/invites.go

@@ -25,10 +25,11 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -85,7 +86,7 @@ var (
 // can be emitted.
 func CheckAndProcessInvite(
 	ctx context.Context,
-	device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite,
+	device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI, db accounts.Database,
 	membership string, roomID string,
 	evTime time.Time,
@@ -136,7 +137,7 @@ func CheckAndProcessInvite(
 // Returns an error if a check or a request failed.
 func queryIDServer(
 	ctx context.Context,
-	db accounts.Database, cfg *config.Dendrite, device *authtypes.Device,
+	db accounts.Database, cfg *config.Dendrite, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) {
 	if err = isTrusted(body.IDServer, cfg); err != nil {
@@ -205,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe
 // Returns an error if the request failed to send or if the response couldn't be parsed.
 func queryIDServerStoreInvite(
 	ctx context.Context,
-	db accounts.Database, cfg *config.Dendrite, device *authtypes.Device,
+	db accounts.Database, cfg *config.Dendrite, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (*idServerStoreInviteResponse, error) {
 	// Retrieve the sender's profile to get their display name
@@ -329,7 +330,7 @@ func checkIDServerSignatures(
 func emit3PIDInviteEvent(
 	ctx context.Context,
 	body *MembershipRequest, res *idServerStoreInviteResponse,
-	device *authtypes.Device, roomID string, cfg *config.Dendrite,
+	device *userapi.Device, roomID string, cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
 	evTime time.Time,
 ) error {
@@ -353,7 +354,7 @@ func emit3PIDInviteEvent(
 	}
 
 	queryRes := api.QueryLatestEventsAndStateResponse{}
-	event, err := internal.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes)
+	event, err := eventutil.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes)
 	if err != nil {
 		return err
 	}

cmd/create-account/main.go

@@ -20,8 +20,8 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 

cmd/create-room-events/main.go

@@ -47,6 +47,7 @@ var (
 	userID           = flag.String("user-id", "@userid:$SERVER_NAME", "The user ID to use as the event sender")
 	messageCount     = flag.Int("message-count", 10, "The number of m.room.messsage events to generate")
 	format           = flag.String("Format", "InputRoomEvent", "The output format to use for the messages: InputRoomEvent or Event")
+	ver              = flag.String("version", string(gomatrixserverlib.RoomVersionV1), "Room version to generate events as")
 )
 
 // By default we use a private key of 0.
@@ -109,7 +110,7 @@ func buildAndOutput() gomatrixserverlib.EventReference {
 
 	event, err := b.Build(
 		now, name, key, privateKey,
-		gomatrixserverlib.RoomVersionV1,
+		gomatrixserverlib.RoomVersion(*ver),
 	)
 	if err != nil {
 		panic(err)
@@ -127,7 +128,7 @@ func writeEvent(event gomatrixserverlib.Event) {
 	if *format == "InputRoomEvent" {
 		var ire api.InputRoomEvent
 		ire.Kind = api.KindNew
-		ire.Event = event.Headered(gomatrixserverlib.RoomVersionV1)
+		ire.Event = event.Headered(gomatrixserverlib.RoomVersion(*ver))
 		authEventIDs := []string{}
 		for _, ref := range b.AuthEvents.([]gomatrixserverlib.EventReference) {
 			authEventIDs = append(authEventIDs, ref.EventID)

cmd/dendrite-appservice-server/main.go

@@ -16,19 +16,18 @@ package main
 
 import (
 	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "AppServiceAPI", true)
+	base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true)
 
 	defer base.Close() // nolint: errcheck
-	accountDB := base.CreateAccountsDB()
+	userAPI := base.UserAPIClient()
-	deviceDB := base.CreateDeviceDB()
 	rsAPI := base.RoomserverHTTPClient()
 
-	intAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI)
+	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI))

cmd/dendrite-client-api-server/main.go

@@ -16,31 +16,29 @@ package main
 
 import (
 	"github.com/matrix-org/dendrite/clientapi"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/internal/transactions"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
 
-	base := basecomponent.NewBaseDendrite(cfg, "ClientAPI", true)
+	base := setup.NewBaseDendrite(cfg, "ClientAPI", true)
 	defer base.Close() // nolint: errcheck
 
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
 	federation := base.CreateFederationClient()
 
-	serverKeyAPI := base.ServerKeyAPIClient()
-	keyRing := serverKeyAPI.KeyRing()
-
 	asQuery := base.AppserviceHTTPClient()
 	rsAPI := base.RoomserverHTTPClient()
 	fsAPI := base.FederationSenderHTTPClient()
 	eduInputAPI := base.EDUServerClient()
+	userAPI := base.UserAPIClient()
 
 	clientapi.AddPublicRoutes(
-		base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation, keyRing,
+		base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation,
-		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI,
+		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))

cmd/dendrite-demo-libp2p/main.go

@@ -32,11 +32,12 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/serverkeyapi"
+	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/eduserver/cache"
@@ -79,6 +80,17 @@ func createFederationClient(
 	)
 }
 
+func createClient(
+	base *P2PDendrite,
+) *gomatrixserverlib.Client {
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix",
+		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
+	)
+	return gomatrixserverlib.NewClientWithTransport(tr)
+}
+
 func main() {
 	instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance")
 	instancePort := flag.Int("port", 8080, "the port that the client API will listen on")
@@ -101,6 +113,7 @@ func main() {
 	}
 
 	cfg := config.Dendrite{}
+	cfg.SetDefaults()
 	cfg.Matrix.ServerName = "p2p"
 	cfg.Matrix.PrivateKey = privKey
 	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
@@ -128,6 +141,7 @@ func main() {
 	accountDB := base.Base.CreateAccountsDB()
 	deviceDB := base.Base.CreateDeviceDB()
 	federation := createFederationClient(base)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil)
 
 	serverKeyAPI := serverkeyapi.NewInternalAPI(
 		base.Base.Cfg, federation, base.Base.Caches,
@@ -141,9 +155,9 @@ func main() {
 		&base.Base, keyRing, federation,
 	)
 	eduInputAPI := eduserver.NewInternalAPI(
-		&base.Base, cache.New(), deviceDB,
+		&base.Base, cache.New(), userAPI,
 	)
-	asAPI := appservice.NewInternalAPI(&base.Base, accountDB, deviceDB, rsAPI)
+	asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
 	fsAPI := federationsender.NewInternalAPI(
 		&base.Base, federation, rsAPI, keyRing,
 	)
@@ -157,6 +171,7 @@ func main() {
 		Config:        base.Base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
+		Client:        createClient(base),
 		FedClient:     federation,
 		KeyRing:       keyRing,
 		KafkaConsumer: base.Base.KafkaConsumer,
@@ -167,13 +182,14 @@ func main() {
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
 		ServerKeyAPI:        serverKeyAPI,
+		UserAPI:             userAPI,
 
 		PublicRoomsDB: publicRoomsDB,
 	}
 	monolith.AddAllPublicRoutes(base.Base.PublicAPIMux)
 
-	internal.SetupHTTPAPI(
+	httputil.SetupHTTPAPI(
-		http.DefaultServeMux,
+		base.Base.BaseMux,
 		base.Base.PublicAPIMux,
 		base.Base.InternalAPIMux,
 		&cfg,
@@ -184,7 +200,7 @@ func main() {
 	go func() {
 		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
 		logrus.Info("Listening on ", httpBindAddr)
-		logrus.Fatal(http.ListenAndServe(httpBindAddr, nil))
+		logrus.Fatal(http.ListenAndServe(httpBindAddr, base.Base.BaseMux))
 	}()
 	// Expose the matrix APIs also via libp2p
 	if base.LibP2P != nil {
@@ -197,7 +213,7 @@ func main() {
 			defer func() {
 				logrus.Fatal(listener.Close())
 			}()
-			logrus.Fatal(http.Serve(listener, nil))
+			logrus.Fatal(http.Serve(listener, base.Base.BaseMux))
 		}()
 	}
 

cmd/dendrite-demo-libp2p/p2pdendrite.go

@@ -22,7 +22,7 @@ import (
 
 	pstore "github.com/libp2p/go-libp2p-core/peerstore"
 	record "github.com/libp2p/go-libp2p-record"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 
 	"github.com/libp2p/go-libp2p"
 	circuit "github.com/libp2p/go-libp2p-circuit"
@@ -39,7 +39,7 @@ import (
 
 // P2PDendrite is a Peer-to-Peer variant of BaseDendrite.
 type P2PDendrite struct {
-	Base basecomponent.BaseDendrite
+	Base setup.BaseDendrite
 
 	// Store our libp2p object so that we can make outgoing connections from it
 	// later
@@ -54,7 +54,7 @@ type P2PDendrite struct {
 // The componentName is used for logging purposes, and should be a friendly name
 // of the component running, e.g. SyncAPI.
 func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
-	baseDendrite := basecomponent.NewBaseDendrite(cfg, componentName, false)
+	baseDendrite := setup.NewBaseDendrite(cfg, componentName, false)
 
 	ctx, cancel := context.WithCancel(context.Background())
 

cmd/dendrite-demo-yggdrasil/main.go

@@ -30,12 +30,12 @@ import (
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/sirupsen/logrus"
@@ -91,7 +91,7 @@ func main() {
 		panic(err)
 	}
 
-	base := basecomponent.NewBaseDendrite(cfg, "Monolith", false)
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
 	defer base.Close() // nolint: errcheck
 
 	accountDB := base.CreateAccountsDB()
@@ -101,16 +101,18 @@ func main() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil)
+
 	rsComponent := roomserver.NewInternalAPI(
 		base, keyRing, federation,
 	)
 	rsAPI := rsComponent
 
 	eduInputAPI := eduserver.NewInternalAPI(
-		base, cache.New(), deviceDB,
+		base, cache.New(), userAPI,
 	)
 
-	asAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI)
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 
 	fsAPI := federationsender.NewInternalAPI(
 		base, federation, rsAPI, keyRing,
@@ -129,6 +131,7 @@ func main() {
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
+		Client:        createClient(ygg),
 		FedClient:     federation,
 		KeyRing:       keyRing,
 		KafkaConsumer: base.KafkaConsumer,
@@ -138,20 +141,34 @@ func main() {
 		EDUInternalAPI:      eduInputAPI,
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 
 		PublicRoomsDB: publicRoomsDB,
 	}
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
 
-	internal.SetupHTTPAPI(
+	httputil.SetupHTTPAPI(
-		http.DefaultServeMux,
+		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
 		cfg,
 		base.UseHTTPAPIs,
 	)
 
+	// Build both ends of a HTTP multiplex.
+	httpServer := &http.Server{
+		Addr:         ":0",
+		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+		ReadTimeout:  15 * time.Second,
+		WriteTimeout: 45 * time.Second,
+		IdleTimeout:  60 * time.Second,
+		BaseContext: func(_ net.Listener) context.Context {
+			return context.Background()
+		},
+		Handler: base.BaseMux,
+	}
+
 	go func() {
 		logrus.Info("Listening on ", ygg.DerivedServerName())
 		logrus.Fatal(httpServer.Serve(ygg))
@@ -159,7 +176,7 @@ func main() {
 	go func() {
 		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
 		logrus.Info("Listening on ", httpBindAddr)
-		logrus.Fatal(http.ListenAndServe(httpBindAddr, nil))
+		logrus.Fatal(http.ListenAndServe(httpBindAddr, base.BaseMux))
 	}()
 
 	select {}

cmd/dendrite-demo-yggdrasil/yggconn/node.go

@@ -15,13 +15,16 @@
 package yggconn
 
 import (
+	"context"
 	"crypto/ed25519"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
@@ -48,6 +51,21 @@ type Node struct {
 	incoming  chan *yamux.Stream
 }
 
+func (n *Node) Dialer(_, address string) (net.Conn, error) {
+	tokens := strings.Split(address, ":")
+	raw, err := hex.DecodeString(tokens[0])
+	if err != nil {
+		return nil, fmt.Errorf("hex.DecodeString: %w", err)
+	}
+	converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw))
+	convhex := hex.EncodeToString(converted)
+	return n.Dial("curve25519", convhex)
+}
+
+func (n *Node) DialerContext(ctx context.Context, network, address string) (net.Conn, error) {
+	return n.Dialer(network, address)
+}
+
 // nolint:gocyclo
 func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) {
 	n := &Node{

cmd/dendrite-edu-server/main.go

@@ -17,21 +17,20 @@ import (
 
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/sirupsen/logrus"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "EDUServerAPI", true)
+	base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true)
 	defer func() {
 		if err := base.Close(); err != nil {
 			logrus.WithError(err).Warn("BaseDendrite close failed")
 		}
 	}()
-	deviceDB := base.CreateDeviceDB()
 
-	intAPI := eduserver.NewInternalAPI(base, cache.New(), deviceDB)
+	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
 	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer))

cmd/dendrite-federation-api-server/main.go

@@ -16,26 +16,24 @@ package main
 
 import (
 	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "FederationAPI", true)
+	base := setup.NewBaseDendrite(cfg, "FederationAPI", true)
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
+	userAPI := base.UserAPIClient()
-	deviceDB := base.CreateDeviceDB()
 	federation := base.CreateFederationClient()
 	serverKeyAPI := base.ServerKeyAPIClient()
 	keyRing := serverKeyAPI.KeyRing()
 	fsAPI := base.FederationSenderHTTPClient()
 	rsAPI := base.RoomserverHTTPClient()
-	asAPI := base.AppserviceHTTPClient()
 
 	federationapi.AddPublicRoutes(
-		base.PublicAPIMux, base.Cfg, accountDB, deviceDB, federation, keyRing,
+		base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing,
-		rsAPI, asAPI, fsAPI, base.EDUServerClient(),
+		rsAPI, fsAPI, base.EDUServerClient(),
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))

cmd/dendrite-federation-sender-server/main.go

@@ -16,12 +16,12 @@ package main
 
 import (
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "FederationSender", true)
+	base := setup.NewBaseDendrite(cfg, "FederationSender", true)
 	defer base.Close() // nolint: errcheck
 
 	federation := base.CreateFederationClient()

cmd/dendrite-key-server/main.go

@@ -15,19 +15,18 @@
 package main
 
 import (
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "KeyServer", true)
+	base := setup.NewBaseDendrite(cfg, "KeyServer", true)
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
+	userAPI := base.UserAPIClient()
-	deviceDB := base.CreateDeviceDB()
 
-	keyserver.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB, accountDB)
+	keyserver.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer))
 

cmd/dendrite-media-api-server/main.go

@@ -15,18 +15,20 @@
 package main
 
 import (
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/mediaapi"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "MediaAPI", true)
+	base := setup.NewBaseDendrite(cfg, "MediaAPI", true)
 	defer base.Close() // nolint: errcheck
 
-	deviceDB := base.CreateDeviceDB()
+	userAPI := base.UserAPIClient()
+	client := gomatrixserverlib.NewClient()
 
-	mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB)
+	mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI, client)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI))
 

cmd/dendrite-monolith-server/main.go

@@ -17,18 +17,21 @@ package main
 import (
 	"flag"
 	"net/http"
+	"os"
 
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/serverkeyapi"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/sirupsen/logrus"
 )
@@ -39,10 +42,11 @@ var (
 	certFile       = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
 	keyFile        = flag.String("tls-key", "", "The PEM private key to use for TLS")
 	enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)")
+	traceInternal  = os.Getenv("DENDRITE_TRACE_INTERNAL") == "1"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(true)
+	cfg := setup.ParseFlags(true)
 	if *enableHTTPAPIs {
 		// If the HTTP APIs are enabled then we need to update the Listen
 		// statements in the configuration so that we know where to find
@@ -56,7 +60,7 @@ func main() {
 		cfg.Listen.ServerKeyAPI = addr
 	}
 
-	base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs)
+	base := setup.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs)
 	defer base.Close() // nolint: errcheck
 
 	accountDB := base.CreateAccountsDB()
@@ -71,25 +75,32 @@ func main() {
 		serverKeyAPI = base.ServerKeyAPIClient()
 	}
 	keyRing := serverKeyAPI.KeyRing()
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices)
 
-	rsComponent := roomserver.NewInternalAPI(
+	rsImpl := roomserver.NewInternalAPI(
 		base, keyRing, federation,
 	)
-	rsAPI := rsComponent
+	// call functions directly on the impl unless running in HTTP mode
+	rsAPI := rsImpl
 	if base.UseHTTPAPIs {
-		roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
+		roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl)
 		rsAPI = base.RoomserverHTTPClient()
 	}
+	if traceInternal {
+		rsAPI = &api.RoomserverInternalAPITrace{
+			Impl: rsAPI,
+		}
+	}
 
 	eduInputAPI := eduserver.NewInternalAPI(
-		base, cache.New(), deviceDB,
+		base, cache.New(), userAPI,
 	)
 	if base.UseHTTPAPIs {
 		eduserver.AddInternalRoutes(base.InternalAPIMux, eduInputAPI)
 		eduInputAPI = base.EDUServerClient()
 	}
 
-	asAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI)
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	if base.UseHTTPAPIs {
 		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
 		asAPI = base.AppserviceHTTPClient()
@@ -102,7 +113,9 @@ func main() {
 		federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI)
 		fsAPI = base.FederationSenderHTTPClient()
 	}
-	rsComponent.SetFederationSenderAPI(fsAPI)
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this dependency
+	rsImpl.SetFederationSenderAPI(fsAPI)
 
 	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
 	if err != nil {
@@ -113,6 +126,7 @@ func main() {
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
+		Client:        gomatrixserverlib.NewClient(),
 		FedClient:     federation,
 		KeyRing:       keyRing,
 		KafkaConsumer: base.KafkaConsumer,
@@ -123,13 +137,14 @@ func main() {
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
 		ServerKeyAPI:        serverKeyAPI,
+		UserAPI:             userAPI,
 
 		PublicRoomsDB: publicRoomsDB,
 	}
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
 
-	internal.SetupHTTPAPI(
+	httputil.SetupHTTPAPI(
-		http.DefaultServeMux,
+		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
 		cfg,
@@ -140,7 +155,8 @@ func main() {
 	go func() {
 		serv := http.Server{
 			Addr:         *httpBindAddr,
-			WriteTimeout: basecomponent.HTTPServerTimeout,
+			WriteTimeout: setup.HTTPServerTimeout,
+			Handler:      base.BaseMux,
 		}
 
 		logrus.Info("Listening on ", serv.Addr)
@@ -151,7 +167,8 @@ func main() {
 		go func() {
 			serv := http.Server{
 				Addr:         *httpsBindAddr,
-				WriteTimeout: basecomponent.HTTPServerTimeout,
+				WriteTimeout: setup.HTTPServerTimeout,
+				Handler:      base.BaseMux,
 			}
 
 			logrus.Info("Listening on ", serv.Addr)

cmd/dendrite-public-rooms-api-server/main.go

@@ -15,18 +15,18 @@
 package main
 
 import (
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/publicroomsapi"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/sirupsen/logrus"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "PublicRoomsAPI", true)
+	base := setup.NewBaseDendrite(cfg, "PublicRoomsAPI", true)
 	defer base.Close() // nolint: errcheck
 
-	deviceDB := base.CreateDeviceDB()
+	userAPI := base.UserAPIClient()
 
 	rsAPI := base.RoomserverHTTPClient()
 
@@ -34,7 +34,7 @@ func main() {
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, deviceDB, publicRoomsDB, rsAPI, nil, nil)
+	publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, userAPI, publicRoomsDB, rsAPI, nil, nil)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI))
 

cmd/dendrite-room-server/main.go

@@ -15,13 +15,13 @@
 package main
 
 import (
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/roomserver"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI", true)
+	base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true)
 	defer base.Close() // nolint: errcheck
 	federation := base.CreateFederationClient()
 

cmd/dendrite-server-key-api-server/main.go

@@ -15,13 +15,13 @@
 package main
 
 import (
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/serverkeyapi"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "ServerKeyAPI", true)
+	base := setup.NewBaseDendrite(cfg, "ServerKeyAPI", true)
 	defer base.Close() // nolint: errcheck
 
 	federation := base.CreateFederationClient()

cmd/dendrite-sync-api-server/main.go

@@ -15,22 +15,21 @@
 package main
 
 import (
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/syncapi"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags(false)
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "SyncAPI", true)
+	base := setup.NewBaseDendrite(cfg, "SyncAPI", true)
 	defer base.Close() // nolint: errcheck
 
-	deviceDB := base.CreateDeviceDB()
+	userAPI := base.UserAPIClient()
-	accountDB := base.CreateAccountsDB()
 	federation := base.CreateFederationClient()
 
 	rsAPI := base.RoomserverHTTPClient()
 
-	syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, deviceDB, accountDB, rsAPI, federation, cfg)
+	syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, userAPI, rsAPI, federation, cfg)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI))
 

cmd/dendritejs/jsServer.go

@@ -28,7 +28,7 @@ import (
 // JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it.
 type JSServer struct {
 	// The router which will service requests
-	Mux *http.ServeMux
+	Mux http.Handler
 }
 
 // OnRequestFromJS is the function that JS will invoke when there is a new request.

cmd/dendritejs/main.go

@@ -19,19 +19,18 @@ package main
 import (
 	"crypto/ed25519"
 	"fmt"
-	"net/http"
 	"syscall/js"
 
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/userapi"
 	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -146,6 +145,11 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc
 	return fed
 }
 
+func createClient(node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.Client {
+	tr := go_http_js_libp2p.NewP2pTransport(node)
+	return gomatrixserverlib.NewClientWithTransport(tr)
+}
+
 func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) {
 	hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/"
 	node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}, "p2p")
@@ -184,12 +188,13 @@ func main() {
 	if err := cfg.Derive(); err != nil {
 		logrus.Fatalf("Failed to derive values from config: %s", err)
 	}
-	base := basecomponent.NewBaseDendrite(cfg, "Monolith", false)
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
 	defer base.Close() // nolint: errcheck
 
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
 	federation := createFederationClient(cfg, node)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil)
 
 	fetcher := &libp2pKeyFetcher{}
 	keyRing := gomatrixserverlib.KeyRing{
@@ -200,9 +205,9 @@ func main() {
 	}
 
 	rsAPI := roomserver.NewInternalAPI(base, keyRing, federation)
-	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), deviceDB)
+	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI)
 	asQuery := appservice.NewInternalAPI(
-		base, accountDB, deviceDB, rsAPI,
+		base, userAPI, rsAPI,
 	)
 	fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing)
 	rsAPI.SetFederationSenderAPI(fedSenderAPI)
@@ -217,6 +222,7 @@ func main() {
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
+		Client:        createClient(node),
 		FedClient:     federation,
 		KeyRing:       &keyRing,
 		KafkaConsumer: base.KafkaConsumer,
@@ -226,6 +232,7 @@ func main() {
 		EDUInternalAPI:      eduInputAPI,
 		FederationSenderAPI: fedSenderAPI,
 		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 
 		PublicRoomsDB:          publicRoomsDB,
@@ -233,8 +240,8 @@ func main() {
 	}
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
 
-	internal.SetupHTTPAPI(
+	httputil.SetupHTTPAPI(
-		http.DefaultServeMux,
+		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
 		cfg,
@@ -246,7 +253,7 @@ func main() {
 		go func() {
 			logrus.Info("Listening on libp2p-js host ID ", node.Id)
 			s := JSServer{
-				Mux: http.DefaultServeMux,
+				Mux: base.BaseMux,
 			}
 			s.ListenAndServe("p2p")
 		}()
@@ -256,7 +263,7 @@ func main() {
 	go func() {
 		logrus.Info("Listening for service-worker fetch traffic")
 		s := JSServer{
-			Mux: http.DefaultServeMux,
+			Mux: base.BaseMux,
 		}
 		s.ListenAndServe("fetch")
 	}()

cmd/roomserver-integration-tests/main.go

@@ -255,7 +255,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 		panic(err)
 	}
 
-	cache, err := caching.NewInMemoryLRUCache()
+	cache, err := caching.NewInMemoryLRUCache(false)
 	if err != nil {
 		panic(err)
 	}

dendrite-config.yaml

@@ -140,6 +140,7 @@ listen:
     edu_server: "localhost:7778"
     key_server: "localhost:7779"
     server_key_api: "localhost:7780"
+    user_api: "localhost:7781"
 
 # The configuration for tracing the dendrite components.
 tracing:

docs/WIRING-Current.md

@@ -0,0 +1,73 @@
+This document details how various components communicate with each other. There are two kinds of components:
+ - Public-facing: exposes CS/SS API endpoints and need to be routed to via client-api-proxy or equivalent.
+ - Internal-only: exposes internal APIs and produces Kafka events.
+
+## Internal HTTP APIs
+
+Not everything can be done using Kafka logs. For example, requesting the latest events in a room is much better suited to
+a request/response model like HTTP or RPC. Therefore, components can expose "internal APIs" which sit outside of Kafka logs.
+Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests.
+
+```
+   Tier 1            Sync        PublicRooms      FederationAPI      ClientAPI    MediaAPI
+Public Facing        | .-----1------`              | | |    | |        | | | |
+                     2 | .-------3-----------------` | |    | `--------|-|-|-|--11--------------------.
+                     | | | .--------4----------------------------------` | | |                        |
+                     | | | |         .---5-----------` |    |            | | |                        |
+                     | | | |         |  .---6----------------------------` | |                        |
+                     | | | |         |  |              |  .-----7----------` |                        |
+                     | | | |         |  |              8  | |                10                       |
+                     | | | |         |  |              |  | `---9----.       |                        |
+                     V V V V         V  V              V  V          V       V                        V
+   Tier 2           Roomserver     EDUServer         FedSender       AppService    KeyServer   ServerKeyAPI
+Internal only               |                               `------------------------12----------^   ^
+                            `------------------------------------------------------------13----------`
+
+ Client ---> Server
+```
+- 1 (PublicRooms -> Roomserver): Calculating current auth for changing visibility
+- 2 (Sync -> Roomserver): When making backfill requests
+- 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests
+- 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests
+- 5 (FedAPI -> EDUServer): Sending typing/send-to-device events
+- 6 (ClientAPI -> EDUServer): Sending typing/send-to-device events
+- 7 (ClientAPI -> FedSender): Handling directory lookups
+- 8 (FedAPI -> FedSender): Resetting backoffs when receiving traffic from a server. Querying joined hosts when handling alias lookup requests
+- 9 (FedAPI -> AppService): Working out if the client is an appservice user
+- 10 (ClientAPI -> AppService): Working out if the client is an appservice user
+- 11 (FedAPI -> ServerKeyAPI): Verifying incoming event signatures
+- 12 (FedSender -> ServerKeyAPI): Verifying event signatures of responses (e.g from send_join)
+- 13 (Roomserver -> ServerKeyAPI): Verifying event signatures of backfilled events
+
+In addition to this, all public facing components (Tier 1) talk to the `UserAPI` to verify access tokens and extract profile information where needed.
+
+## Kafka logs
+
+```
+                       .----1--------------------------------------------.
+                       V                                                 |
+   Tier 1            Sync        PublicRooms      FederationAPI      ClientAPI    MediaAPI
+Public Facing        ^   ^           ^                                 ^  
+                     |   |           |                                 |
+                     2   |           |                                 |
+                     |   `-3------------.                              |
+                     |               |  |                              |
+                     |               |  |                              |
+                     | .------4------`  |                              |
+                     | | .--------5-----|------------------------------`             
+                     | | |              |           
+   Tier 2           Roomserver     EDUServer         FedSender       AppService    KeyServer   ServerKeyAPI
+Internal only              |          |                ^ ^
+                           |          `-----6----------` |
+                           `--------------------7--------`
+
+
+Producer ----> Consumer
+```
+- 1 (ClientAPI -> Sync): For tracking account data
+- 2 (Roomserver -> Sync): For all data to send to clients
+- 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients
+- 4 (Roomserver -> PublicRooms): For tracking the current room name/topic/joined count/etc.
+- 5 (Roomserver -> ClientAPI): For tracking memberships for profile updates.
+- 6 (EDUServer -> FedSender): For sending EDUs over federation
+- 7 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts.

eduserver/eduserver.go

@@ -18,12 +18,12 @@ package eduserver
 
 import (
 	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/eduserver/input"
 	"github.com/matrix-org/dendrite/eduserver/inthttp"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
@@ -35,13 +35,13 @@ func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI)
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *basecomponent.BaseDendrite,
+	base *setup.BaseDendrite,
 	eduCache *cache.EDUCache,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 ) api.EDUServerInputAPI {
 	return &input.EDUServerInputAPI{
 		Cache:                        eduCache,
-		DeviceDB:                     deviceDB,
+		UserAPI:                      userAPI,
 		Producer:                     base.KafkaProducer,
 		OutputTypingEventTopic:       string(base.Cfg.Kafka.Topics.OutputTypingEvent),
 		OutputSendToDeviceEventTopic: string(base.Cfg.Kafka.Topics.OutputSendToDeviceEvent),

eduserver/input/input.go

@@ -22,9 +22,9 @@ import (
 	"time"
 
 	"github.com/Shopify/sarama"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/eduserver/cache"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
@@ -39,8 +39,8 @@ type EDUServerInputAPI struct {
 	OutputSendToDeviceEventTopic string
 	// kafka producer
 	Producer sarama.SyncProducer
-	// device database
+	// Internal user query API
-	DeviceDB devices.Database
+	UserAPI userapi.UserInternalAPI
 	// our server name
 	ServerName gomatrixserverlib.ServerName
 }
@@ -97,6 +97,11 @@ func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error {
 	if err != nil {
 		return err
 	}
+	logrus.WithFields(logrus.Fields{
+		"room_id": ite.RoomID,
+		"user_id": ite.UserID,
+		"typing":  ite.Typing,
+	}).Infof("Producing to topic '%s'", t.OutputTypingEventTopic)
 
 	m := &sarama.ProducerMessage{
 		Topic: string(t.OutputTypingEventTopic),
@@ -110,7 +115,7 @@ func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error {
 
 func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) error {
 	devices := []string{}
-	localpart, domain, err := gomatrixserverlib.SplitID('@', ise.UserID)
+	_, domain, err := gomatrixserverlib.SplitID('@', ise.UserID)
 	if err != nil {
 		return err
 	}
@@ -121,17 +126,25 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e
 	// wildcard as we don't know about the remote devices, so instead we leave it
 	// as-is, so that the federation sender can send it on with the wildcard intact.
 	if domain == t.ServerName && ise.DeviceID == "*" {
-		devs, err := t.DeviceDB.GetDevicesByLocalpart(context.TODO(), localpart)
+		var res userapi.QueryDevicesResponse
+		err = t.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{
+			UserID: ise.UserID,
+		}, &res)
 		if err != nil {
 			return err
 		}
-		for _, dev := range devs {
+		for _, dev := range res.Devices {
 			devices = append(devices, dev.ID)
 		}
 	} else {
 		devices = append(devices, ise.DeviceID)
 	}
 
+	logrus.WithFields(logrus.Fields{
+		"user_id":     ise.UserID,
+		"num_devices": len(devices),
+		"type":        ise.Type,
+	}).Infof("Producing to topic '%s'", t.OutputSendToDeviceEventTopic)
 	for _, device := range devices {
 		ote := &api.OutputSendToDeviceEvent{
 			UserID:            ise.UserID,
@@ -139,12 +152,6 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e
 			SendToDeviceEvent: ise.SendToDeviceEvent,
 		}
 
-		logrus.WithFields(logrus.Fields{
-			"user_id":    ise.UserID,
-			"device_id":  ise.DeviceID,
-			"event_type": ise.Type,
-		}).Info("handling send-to-device message")
-
 		eventJSON, err := json.Marshal(ote)
 		if err != nil {
 			logrus.WithError(err).Error("sendToDevice failed json.Marshal")

eduserver/inthttp/client.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/eduserver/api"
-	internalHTTP "github.com/matrix-org/dendrite/internal/http"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/opentracing/opentracing-go"
 )
 
@@ -39,7 +39,7 @@ func (h *httpEDUServerInputAPI) InputTypingEvent(
 	defer span.Finish()
 
 	apiURL := h.eduServerURL + EDUServerInputTypingEventPath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // InputSendToDeviceEvent implements EDUServerInputAPI
@@ -52,5 +52,5 @@ func (h *httpEDUServerInputAPI) InputSendToDeviceEvent(
 	defer span.Finish()
 
 	apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }

eduserver/inthttp/server.go

@@ -6,14 +6,14 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/eduserver/api"
-	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/util"
 )
 
 // AddRoutes adds the EDUServerInputAPI handlers to the http.ServeMux.
 func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(EDUServerInputTypingEventPath,
-		internal.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse {
 			var request api.InputTypingEventRequest
 			var response api.InputTypingEventResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -26,7 +26,7 @@ func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) {
 		}),
 	)
 	internalAPIMux.Handle(EDUServerInputSendToDeviceEventPath,
-		internal.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse {
 			var request api.InputSendToDeviceEventRequest
 			var response api.InputSendToDeviceEventResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {

federationapi/federationapi.go

@@ -16,13 +16,11 @@ package federationapi
 
 import (
 	"github.com/gorilla/mux"
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -32,19 +30,17 @@ import (
 func AddPublicRoutes(
 	router *mux.Router,
 	cfg *config.Dendrite,
-	accountsDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing *gomatrixserverlib.KeyRing,
+	keyRing gomatrixserverlib.JSONVerifier,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
 	federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
 ) {
 
 	routing.Setup(
-		router, cfg, rsAPI, asAPI,
+		router, cfg, rsAPI,
-		eduAPI, federationSenderAPI, *keyRing,
+		eduAPI, federationSenderAPI, keyRing,
-		federation, accountsDB, deviceDB,
+		federation, userAPI,
 	)
 }

federationapi/federationapi_test.go

@@ -0,0 +1,102 @@
+package federationapi_test
+
+import (
+	"context"
+	"crypto/ed25519"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/internal/test"
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404.
+// Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated.
+func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
+	_, privKey, _ := ed25519.GenerateKey(nil)
+	cfg := &config.Dendrite{}
+	cfg.Matrix.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
+	cfg.Matrix.ServerName = gomatrixserverlib.ServerName("localhost")
+	cfg.Matrix.PrivateKey = privKey
+	cfg.Kafka.UseNaffka = true
+	cfg.Database.Naffka = "file::memory:"
+	cfg.SetDefaults()
+	base := setup.NewBaseDendrite(cfg, "Test", false)
+	keyRing := &test.NopJSONVerifier{}
+	fsAPI := base.FederationSenderHTTPClient()
+	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
+	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
+	federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil)
+	httputil.SetupHTTPAPI(
+		base.BaseMux,
+		base.PublicAPIMux,
+		base.InternalAPIMux,
+		cfg,
+		base.UseHTTPAPIs,
+	)
+	baseURL, cancel := test.ListenAndServe(t, base.BaseMux, true)
+	defer cancel()
+	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
+
+	fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey)
+
+	testCases := []struct {
+		roomVer   gomatrixserverlib.RoomVersion
+		eventJSON string
+	}{
+		{
+			eventJSON: `{"auth_events":[["$Nzfbrhc3oaYVKzGM:localhost",{"sha256":"BCBHOgB4qxLPQkBd6th8ydFSyqjth/LF99VNjYffOQ0"}],["$EZzkD2BH1Gtm5v1D:localhost",{"sha256":"3dLUnDBs8/iC5DMw/ydKtmAqVZtzqqtHpsjsQPk7GJA"}]],"content":{"body":"Test Message"},"depth":11,"event_id":"$mGiPO3oGjQfCkIUw:localhost","hashes":{"sha256":"h+t+4DwIBC9UNyJ3jzyAQAAl4H3yQHVuHrm2S1JZizU"},"origin":"localhost","origin_server_ts":0,"prev_events":[["$tFr64vpiSHdLU0Qr:localhost",{"sha256":"+R07ZrIs4c4tjPFE+tmcYIGUfeLGFI/4e0OITb9uEcM"}]],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"LYFr/rW9m5/7UKBQMF5qWnG82He4VGsRESUgDmvkn5DrJRyS4TLL/7zl0Lymn3pa3q2yaTO74LQX/CRotqG1BA"}},"type":"m.room.message"}`,
+			roomVer:   gomatrixserverlib.RoomVersionV1,
+		},
+		// single / (handlers which do not UseEncodedPath will fail this test)
+		// EventID: $0SFh2WJbjBs3OT+E0yl95giDKo/3Zp52HsHUUk4uPyg
+		{
+			eventJSON: `{"auth_events":["$x4MKEPRSF6OGlo0qpnsP3BfSmYX5HhVlykOsQH3ECyg","$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"content":{"body":"Test Message"},"depth":8,"hashes":{"sha256":"dfK0MBn1RZZqCVJqWsn/MGY7QJHjQcwqF0unOonLCTU"},"origin":"localhost","origin_server_ts":0,"prev_events":["$1SwcZ1XY/Y8yKLjP4DzAOHN5WFBcDAZxb5vFDnW2ubA"],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"INOjuWMg+GmFkUpmzhMB0bqLNs73mSvwldY1ftYIQ/B3lD9soD2OMG3AF+wgZW/I8xqzY4DOHfbnbUeYPf67BA"}},"type":"m.room.message"}`,
+			roomVer:   gomatrixserverlib.RoomVersionV3,
+		},
+		// multiple /
+		// EventID: $OzENBCuVv/fnRAYCeQudIon/84/V5pxtEjQMTgi3emk
+		{
+			eventJSON: `{"auth_events":["$x4MKEPRSF6OGlo0qpnsP3BfSmYX5HhVlykOsQH3ECyg","$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"content":{"body":"Test Message"},"depth":2,"hashes":{"sha256":"U5+WsiJAhiEM88J8HTjuUjPImVGVzDFD3v/WS+jb2f0"},"origin":"localhost","origin_server_ts":0,"prev_events":["$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"tKS469e9+wdWPEKB/LbBJWQ8vfOOdKgTWER5IwbSAH1CxmLvkCziUsgVu85zfzDSLoUi5mU5FHLiMTC6P/qICw"}},"type":"m.room.message"}`,
+			roomVer:   gomatrixserverlib.RoomVersionV3,
+		},
+		// two slashes (handlers which clean paths before UseEncodedPath will fail this test)
+		// EventID: $EmwNBlHoSOVmCZ1cM//yv/OvxB6r4OFEIGSJea7+Amk
+		{
+			eventJSON: `{"auth_events":["$x4MKEPRSF6OGlo0qpnsP3BfSmYX5HhVlykOsQH3ECyg","$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"content":{"body":"Test Message"},"depth":3917,"hashes":{"sha256":"cNAWtlHIegrji0mMA6x1rhpYCccY8W1NsWZqSpJFhjs"},"origin":"localhost","origin_server_ts":0,"prev_events":["$4GDB0bVjkWwS3G4noUZCq5oLWzpBYpwzdMcf7gj24CI"],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"NKym6Kcy3u9mGUr21Hjfe3h7DfDilDhN5PqztT0QZ4NTZ+8Y7owseLolQVXp+TvNjecvzdDywsXXVvGiuQiWAQ"}},"type":"m.room.message"}`,
+			roomVer:   gomatrixserverlib.RoomVersionV3,
+		},
+	}
+
+	for _, tc := range testCases {
+		ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(tc.eventJSON), false, tc.roomVer)
+		if err != nil {
+			t.Errorf("failed to parse event: %s", err)
+		}
+		he := ev.Headered(tc.roomVer)
+		invReq, err := gomatrixserverlib.NewInviteV2Request(&he, nil)
+		if err != nil {
+			t.Errorf("failed to create invite v2 request: %s", err)
+			continue
+		}
+		_, err = fedCli.SendInviteV2(context.Background(), serverName, invReq)
+		if err == nil {
+			t.Errorf("expected an error, got none")
+			continue
+		}
+		gerr, ok := err.(gomatrix.HTTPError)
+		if !ok {
+			t.Errorf("failed to cast response error as gomatrix.HTTPError")
+			continue
+		}
+		t.Logf("Error: %+v", gerr)
+		if gerr.Code == 404 {
+			t.Errorf("invite event resulted in a 404")
+		}
+	}
+}

federationapi/routing/backfill.go

@@ -37,7 +37,7 @@ func Backfill(
 	roomID string,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
-	var res api.QueryBackfillResponse
+	var res api.PerformBackfillResponse
 	var eIDs []string
 	var limit string
 	var exists bool
@@ -68,7 +68,7 @@ func Backfill(
 	}
 
 	// Populate the request.
-	req := api.QueryBackfillRequest{
+	req := api.PerformBackfillRequest{
 		RoomID: roomID,
 		// we don't know who the successors are for these events, which won't
 		// be a problem because we don't use that information when servicing /backfill requests,
@@ -87,8 +87,8 @@ func Backfill(
 	}
 
 	// Query the roomserver.
-	if err = rsAPI.QueryBackfill(httpReq.Context(), &req, &res); err != nil {
+	if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed")
+		util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed")
 		return jsonerror.InternalServerError()
 	}
 

federationapi/routing/devices.go

@@ -15,9 +15,8 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/userutil"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -25,17 +24,9 @@ import (
 // GetUserDevices for the given user id
 func GetUserDevices(
 	req *http.Request,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 	userID string,
 ) util.JSONResponse {
-	localpart, err := userutil.ParseUsernameParam(userID, nil)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue("Invalid user ID"),
-		}
-	}
-
 	response := gomatrixserverlib.RespUserDevices{
 		UserID: userID,
 		// TODO: we should return an incrementing stream ID each time the device
@@ -43,13 +34,16 @@ func GetUserDevices(
 		StreamID: 0,
 	}
 
-	devs, err := deviceDB.GetDevicesByLocalpart(req.Context(), localpart)
+	var res userapi.QueryDevicesResponse
+	err := userAPI.QueryDevices(req.Context(), &userapi.QueryDevicesRequest{
+		UserID: userID,
+	}, &res)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDevicesByLocalPart failed")
+		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryDevices failed")
 		return jsonerror.InternalServerError()
 	}
 
-	for _, dev := range devs {
+	for _, dev := range res.Devices {
 		device := gomatrixserverlib.RespUserDevice{
 			DeviceID:    dev.ID,
 			DisplayName: dev.DisplayName,

federationapi/routing/invite.go

@@ -35,7 +35,7 @@ func Invite(
 	eventID string,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
 	inviteReq := gomatrixserverlib.InviteV2Request{}
 	if err := json.Unmarshal(request.Content(), &inviteReq); err != nil {

federationapi/routing/join.go

@@ -21,8 +21,8 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -95,8 +95,8 @@ func MakeJoin(
 	queryRes := api.QueryLatestEventsAndStateResponse{
 		RoomVersion: verRes.RoomVersion,
 	}
-	event, err := internal.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
+	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
-	if err == internal.ErrRoomNoExists {
+	if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound("Room does not exist"),
@@ -107,7 +107,7 @@ func MakeJoin(
 			JSON: jsonerror.BadJSON(e.Error()),
 		}
 	} else if err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed")
+		util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -143,7 +143,7 @@ func SendJoin(
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}

federationapi/routing/leave.go

@@ -17,8 +17,8 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -69,8 +69,8 @@ func MakeLeave(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	event, err := internal.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
+	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
-	if err == internal.ErrRoomNoExists {
+	if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound("Room does not exist"),
@@ -81,7 +81,7 @@ func MakeLeave(
 			JSON: jsonerror.BadJSON(e.Error()),
 		}
 	} else if err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed")
+		util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -113,7 +113,7 @@ func SendLeave(
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}

federationapi/routing/profile.go

@@ -18,11 +18,10 @@ import (
 	"fmt"
 	"net/http"
 
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -30,9 +29,8 @@ import (
 // GetProfile implements GET /_matrix/federation/v1/query/profile
 func GetProfile(
 	httpReq *http.Request,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
 	cfg *config.Dendrite,
-	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field")
 
@@ -60,9 +58,12 @@ func GetProfile(
 		}
 	}
 
-	profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB)
+	var profileRes userapi.QueryProfileResponse
+	err = userAPI.QueryProfile(httpReq.Context(), &userapi.QueryProfileRequest{
+		UserID: userID,
+	}, &profileRes)
 	if err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed")
+		util.GetLogger(httpReq.Context()).WithError(err).Error("userAPI.QueryProfile failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -72,21 +73,21 @@ func GetProfile(
 	if field != "" {
 		switch field {
 		case "displayname":
-			res = internal.DisplayName{
+			res = eventutil.DisplayName{
-				DisplayName: profile.DisplayName,
+				DisplayName: profileRes.DisplayName,
 			}
 		case "avatar_url":
-			res = internal.AvatarURL{
+			res = eventutil.AvatarURL{
-				AvatarURL: profile.AvatarURL,
+				AvatarURL: profileRes.AvatarURL,
 			}
 		default:
 			code = http.StatusBadRequest
 			res = jsonerror.InvalidArgumentValue("The request body did not contain an allowed value of argument 'field'. Allowed values are either: 'avatar_url', 'displayname'.")
 		}
 	} else {
-		res = internal.ProfileResponse{
+		res = eventutil.ProfileResponse{
-			AvatarURL:   profile.AvatarURL,
+			AvatarURL:   profileRes.AvatarURL,
-			DisplayName: profile.DisplayName,
+			DisplayName: profileRes.DisplayName,
 		}
 	}
 

federationapi/routing/routing.go

@@ -18,14 +18,13 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -48,23 +47,21 @@ func Setup(
 	publicAPIMux *mux.Router,
 	cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
 	fsAPI federationSenderAPI.FederationSenderInternalAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 ) {
 	v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter()
 	v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter()
 	v2fedmux := publicAPIMux.PathPrefix(pathPrefixV2Federation).Subrouter()
 
-	wakeup := &internal.FederationWakeups{
+	wakeup := &httputil.FederationWakeups{
 		FsAPI: fsAPI,
 	}
 
-	localKeys := internal.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse {
+	localKeys := httputil.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse {
 		return LocalKeys(cfg)
 	})
 
@@ -76,7 +73,7 @@ func Setup(
 	v2keysmux.Handle("/server/", localKeys).Methods(http.MethodGet)
 	v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/send/{txnID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/send/{txnID}", httputil.MakeFedAPI(
 		"federation_send", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return Send(
@@ -86,7 +83,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
 
-	v2fedmux.Handle("/invite/{roomID}/{eventID}", internal.MakeFedAPI(
+	v2fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
 		"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return Invite(
@@ -96,13 +93,13 @@ func Setup(
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
 
-	v1fedmux.Handle("/3pid/onbind", internal.MakeExternalAPI("3pid_onbind",
+	v1fedmux.Handle("/3pid/onbind", httputil.MakeExternalAPI("3pid_onbind",
 		func(req *http.Request) util.JSONResponse {
-			return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, federation, accountDB)
+			return CreateInvitesFrom3PIDInvites(req, rsAPI, cfg, federation, userAPI)
 		},
 	)).Methods(http.MethodPost, http.MethodOptions)
 
-	v1fedmux.Handle("/exchange_third_party_invite/{roomID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/exchange_third_party_invite/{roomID}", httputil.MakeFedAPI(
 		"exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return ExchangeThirdPartyInvite(
@@ -111,7 +108,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
 
-	v1fedmux.Handle("/event/{eventID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/event/{eventID}", httputil.MakeFedAPI(
 		"federation_get_event", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetEvent(
@@ -120,7 +117,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/state/{roomID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/state/{roomID}", httputil.MakeFedAPI(
 		"federation_get_state", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetState(
@@ -129,7 +126,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/state_ids/{roomID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/state_ids/{roomID}", httputil.MakeFedAPI(
 		"federation_get_state_ids", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetStateIDs(
@@ -138,7 +135,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/event_auth/{roomID}/{eventID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/event_auth/{roomID}/{eventID}", httputil.MakeFedAPI(
 		"federation_get_event_auth", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetEventAuth(
@@ -147,7 +144,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/query/directory", internal.MakeFedAPI(
+	v1fedmux.Handle("/query/directory", httputil.MakeFedAPI(
 		"federation_query_room_alias", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return RoomAliasToID(
@@ -156,25 +153,25 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/query/profile", internal.MakeFedAPI(
+	v1fedmux.Handle("/query/profile", httputil.MakeFedAPI(
 		"federation_query_profile", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetProfile(
-				httpReq, accountDB, cfg, asAPI,
+				httpReq, userAPI, cfg,
 			)
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/user/devices/{userID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/user/devices/{userID}", httputil.MakeFedAPI(
 		"federation_user_devices", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetUserDevices(
-				httpReq, deviceDB, vars["userID"],
+				httpReq, userAPI, vars["userID"],
 			)
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/make_join/{roomID}/{eventID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/make_join/{roomID}/{eventID}", httputil.MakeFedAPI(
 		"federation_make_join", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			roomID := vars["roomID"]
@@ -199,7 +196,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI(
 		"federation_send_join", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			roomID := vars["roomID"]
@@ -207,17 +204,25 @@ func Setup(
 			res := SendJoin(
 				httpReq, request, cfg, rsAPI, keys, roomID, eventID,
 			)
+			// not all responses get wrapped in [code, body]
+			var body interface{}
+			body = []interface{}{
+				res.Code, res.JSON,
+			}
+			jerr, ok := res.JSON.(*jsonerror.MatrixError)
+			if ok {
+				body = jerr
+			}
+
 			return util.JSONResponse{
 				Headers: res.Headers,
 				Code:    res.Code,
-				JSON: []interface{}{
+				JSON:    body,
-					res.Code, res.JSON,
-				},
 			}
 		},
 	)).Methods(http.MethodPut)
 
-	v2fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI(
+	v2fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI(
 		"federation_send_join", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			roomID := vars["roomID"]
@@ -228,7 +233,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPut)
 
-	v1fedmux.Handle("/make_leave/{roomID}/{eventID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/make_leave/{roomID}/{eventID}", httputil.MakeFedAPI(
 		"federation_make_leave", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			roomID := vars["roomID"]
@@ -239,7 +244,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v2fedmux.Handle("/send_leave/{roomID}/{eventID}", internal.MakeFedAPI(
+	v2fedmux.Handle("/send_leave/{roomID}/{eventID}", httputil.MakeFedAPI(
 		"federation_send_leave", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			roomID := vars["roomID"]
@@ -250,21 +255,21 @@ func Setup(
 		},
 	)).Methods(http.MethodPut)
 
-	v1fedmux.Handle("/version", internal.MakeExternalAPI(
+	v1fedmux.Handle("/version", httputil.MakeExternalAPI(
 		"federation_version",
 		func(httpReq *http.Request) util.JSONResponse {
 			return Version()
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/get_missing_events/{roomID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/get_missing_events/{roomID}", httputil.MakeFedAPI(
 		"federation_get_missing_events", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"])
 		},
 	)).Methods(http.MethodPost)
 
-	v1fedmux.Handle("/backfill/{roomID}", internal.MakeFedAPI(
+	v1fedmux.Handle("/backfill/{roomID}", httputil.MakeFedAPI(
 		"federation_backfill", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg)

federationapi/routing/send.go

@@ -37,7 +37,7 @@ func Send(
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
 	t := txnReq{

federationapi/routing/send_test.go

@@ -10,6 +10,7 @@ import (
 
 	eduAPI "github.com/matrix-org/dendrite/eduserver/api"
 	fsAPI "github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -53,15 +54,6 @@ func init() {
 	}
 }
 
-type testNopJSONVerifier struct {
-	// this verifier verifies nothing
-}
-
-func (t *testNopJSONVerifier) VerifyJSONs(ctx context.Context, requests []gomatrixserverlib.VerifyJSONRequest) ([]gomatrixserverlib.VerifyJSONResult, error) {
-	result := make([]gomatrixserverlib.VerifyJSONResult, len(requests))
-	return result, nil
-}
-
 type testEDUProducer struct {
 	// this producer keeps track of calls to InputTypingEvent
 	invocations []eduAPI.InputTypingEventRequest
@@ -128,7 +120,6 @@ func (t *testRoomserverAPI) QueryLatestEventsAndState(
 	response *api.QueryLatestEventsAndStateResponse,
 ) error {
 	r := t.queryLatestEventsAndState(request)
-	response.QueryLatestEventsAndStateRequest = *request
 	response.RoomExists = r.RoomExists
 	response.RoomVersion = testRoomVersion
 	response.LatestEvents = r.LatestEvents
@@ -144,7 +135,6 @@ func (t *testRoomserverAPI) QueryStateAfterEvents(
 	response *api.QueryStateAfterEventsResponse,
 ) error {
 	response.RoomVersion = testRoomVersion
-	response.QueryStateAfterEventsRequest = *request
 	res := t.queryStateAfterEvents(request)
 	response.PrevEventsExist = res.PrevEventsExist
 	response.RoomExists = res.RoomExists
@@ -181,15 +171,6 @@ func (t *testRoomserverAPI) QueryMembershipsForRoom(
 	return fmt.Errorf("not implemented")
 }
 
-// Query a list of invite event senders for a user in a room.
-func (t *testRoomserverAPI) QueryInvitesForUser(
-	ctx context.Context,
-	request *api.QueryInvitesForUserRequest,
-	response *api.QueryInvitesForUserResponse,
-) error {
-	return fmt.Errorf("not implemented")
-}
-
 // Query whether a server is allowed to see an event
 func (t *testRoomserverAPI) QueryServerAllowedToSeeEvent(
 	ctx context.Context,
@@ -220,10 +201,10 @@ func (t *testRoomserverAPI) QueryStateAndAuthChain(
 }
 
 // Query a given amount (or less) of events prior to a given set of events.
-func (t *testRoomserverAPI) QueryBackfill(
+func (t *testRoomserverAPI) PerformBackfill(
 	ctx context.Context,
-	request *api.QueryBackfillRequest,
+	request *api.PerformBackfillRequest,
-	response *api.QueryBackfillResponse,
+	response *api.PerformBackfillResponse,
 ) error {
 	return fmt.Errorf("not implemented")
 }
@@ -341,7 +322,7 @@ func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederat
 		context:    context.Background(),
 		rsAPI:      rsAPI,
 		eduAPI:     &testEDUProducer{},
-		keys:       &testNopJSONVerifier{},
+		keys:       &test.NopJSONVerifier{},
 		federation: fedClient,
 		haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent),
 		newEvents:  make(map[string]bool),
@@ -621,7 +602,6 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) {
 					}
 				}
 			}
-			res.QueryEventsByIDRequest = *req
 			return res
 		},
 	}

federationapi/routing/state.go

@@ -107,15 +107,13 @@ func getState(
 		return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
 	}
 
-	authEventIDs := getIDsFromEventRef(event.AuthEvents())
-
 	var response api.QueryStateAndAuthChainResponse
 	err := rsAPI.QueryStateAndAuthChain(
 		ctx,
 		&api.QueryStateAndAuthChainRequest{
 			RoomID:       roomID,
 			PrevEventIDs: []string{eventID},
-			AuthEventIDs: authEventIDs,
+			AuthEventIDs: event.AuthEventIDs(),
 		},
 		&response,
 	)
@@ -134,15 +132,6 @@ func getState(
 	}, nil
 }
 
-func getIDsFromEventRef(events []gomatrixserverlib.EventReference) []string {
-	IDs := make([]string, len(events))
-	for i := range events {
-		IDs[i] = events[i].EventID
-	}
-
-	return IDs
-}
-
 func getIDsFromEvent(events []gomatrixserverlib.Event) []string {
 	IDs := make([]string, len(events))
 	for i := range events {

federationapi/routing/threepid.go

@@ -21,13 +21,11 @@ import (
 	"net/http"
 	"time"
 
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -57,10 +55,10 @@ var (
 
 // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind
 func CreateInvitesFrom3PIDInvites(
-	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
+	req *http.Request, rsAPI api.RoomserverInternalAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite,
+	cfg *config.Dendrite,
 	federation *gomatrixserverlib.FederationClient,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
 	var body invites
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
@@ -79,7 +77,7 @@ func CreateInvitesFrom3PIDInvites(
 		}
 
 		event, err := createInviteFrom3PIDInvite(
-			req.Context(), rsAPI, asAPI, cfg, inv, federation, accountDB,
+			req.Context(), rsAPI, cfg, inv, federation, userAPI,
 		)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed")
@@ -107,7 +105,7 @@ func ExchangeThirdPartyInvite(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI api.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
@@ -197,10 +195,10 @@ func ExchangeThirdPartyInvite(
 // Returns an error if there was a problem building the event or fetching the
 // necessary data to do so.
 func createInviteFrom3PIDInvite(
-	ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI,
+	ctx context.Context, rsAPI api.RoomserverInternalAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite,
+	cfg *config.Dendrite,
 	inv invite, federation *gomatrixserverlib.FederationClient,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
 ) (*gomatrixserverlib.Event, error) {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
@@ -225,14 +223,17 @@ func createInviteFrom3PIDInvite(
 		StateKey: &inv.MXID,
 	}
 
-	profile, err := appserviceAPI.RetrieveUserProfile(ctx, inv.MXID, asAPI, accountDB)
+	var res userapi.QueryProfileResponse
+	err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{
+		UserID: inv.MXID,
+	}, &res)
 	if err != nil {
 		return nil, err
 	}
 
 	content := gomatrixserverlib.MemberContent{
-		AvatarURL:   profile.AvatarURL,
+		AvatarURL:   res.AvatarURL,
-		DisplayName: profile.DisplayName,
+		DisplayName: res.DisplayName,
 		Membership:  gomatrixserverlib.Invite,
 		ThirdPartyInvite: &gomatrixserverlib.MemberThirdPartyInvite{
 			Signed: inv.Signed,
@@ -261,7 +262,7 @@ func createInviteFrom3PIDInvite(
 // Returns an error if something failed during the process.
 func buildMembershipEvent(
 	ctx context.Context,
-	builder *gomatrixserverlib.EventBuilder, rsAPI roomserverAPI.RoomserverInternalAPI,
+	builder *gomatrixserverlib.EventBuilder, rsAPI api.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 ) (*gomatrixserverlib.Event, error) {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
@@ -274,11 +275,11 @@ func buildMembershipEvent(
 	}
 
 	// Ask the roomserver for information about this room
-	queryReq := roomserverAPI.QueryLatestEventsAndStateRequest{
+	queryReq := api.QueryLatestEventsAndStateRequest{
 		RoomID:       builder.RoomID,
 		StateToFetch: eventsNeeded.Tuples(),
 	}
-	var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
+	var queryRes api.QueryLatestEventsAndStateResponse
 	if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil {
 		return nil, err
 	}

federationsender/api/api.go

@@ -15,14 +15,6 @@ type FederationSenderInternalAPI interface {
 		request *PerformDirectoryLookupRequest,
 		response *PerformDirectoryLookupResponse,
 	) error
-	// Query the joined hosts and the membership events accounting for their participation in a room.
-	// Note that if a server has multiple users in the room, it will have multiple entries in the returned slice.
-	// See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version.
-	QueryJoinedHostsInRoom(
-		ctx context.Context,
-		request *QueryJoinedHostsInRoomRequest,
-		response *QueryJoinedHostsInRoomResponse,
-	) error
 	// Query the server names of the joined hosts in a room.
 	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
 	// containing only the server names (without information for membership events).
@@ -88,22 +80,12 @@ type PerformServersAliveRequest struct {
 type PerformServersAliveResponse struct {
 }
 
-// QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom
+// QueryJoinedHostServerNamesInRoomRequest is a request to QueryJoinedHostServerNames
-type QueryJoinedHostsInRoomRequest struct {
-	RoomID string `json:"room_id"`
-}
-
-// QueryJoinedHostsInRoomResponse is a response to QueryJoinedHostsInRoom
-type QueryJoinedHostsInRoomResponse struct {
-	JoinedHosts []types.JoinedHost `json:"joined_hosts"`
-}
-
-// QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames
 type QueryJoinedHostServerNamesInRoomRequest struct {
 	RoomID string `json:"room_id"`
 }
 
-// QueryJoinedHostServerNamesResponse is a response to QueryJoinedHostServerNames
+// QueryJoinedHostServerNamesInRoomResponse is a response to QueryJoinedHostServerNames
 type QueryJoinedHostServerNamesInRoomResponse struct {
 	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
 }

federationsender/consumers/roomserver.go

@@ -86,11 +86,6 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 	switch output.Type {
 	case api.OutputTypeNewRoomEvent:
 		ev := &output.NewRoomEvent.Event
-		log.WithFields(log.Fields{
-			"event_id":       ev.EventID(),
-			"room_id":        ev.RoomID(),
-			"send_as_server": output.NewRoomEvent.SendAsServer,
-		}).Info("received room event from roomserver")
 
 		if err := s.processMessage(*output.NewRoomEvent); err != nil {
 			// panic rather than continue with an inconsistent database
@@ -131,11 +126,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 // processMessage updates the list of currently joined hosts in the room
 // and then sends the event to the hosts that were joined before the event.
 func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error {
-	addsStateEvents, err := s.lookupStateEvents(ore.AddsStateEventIDs, ore.Event.Event)
+	addsJoinedHosts, err := joinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(ore.AddsState()))
-	if err != nil {
-		return err
-	}
-	addsJoinedHosts, err := joinedHostsFromEvents(addsStateEvents)
 	if err != nil {
 		return err
 	}

federationsender/federationsender.go

@@ -23,7 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/federationsender/queue"
 	"github.com/matrix-org/dendrite/federationsender/storage"
 	"github.com/matrix-org/dendrite/federationsender/types"
-	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
@@ -38,7 +38,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.FederationSenderInternalAP
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *basecomponent.BaseDendrite,
+	base *setup.BaseDendrite,
 	federation *gomatrixserverlib.FederationClient,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	keyRing *gomatrixserverlib.KeyRing,

federationsender/internal/query.go

@@ -7,16 +7,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// QueryJoinedHostsInRoom implements api.FederationSenderInternalAPI
-func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom(
-	ctx context.Context,
-	request *api.QueryJoinedHostsInRoomRequest,
-	response *api.QueryJoinedHostsInRoomResponse,
-) (err error) {
-	response.JoinedHosts, err = f.db.GetJoinedHosts(ctx, request.RoomID)
-	return
-}
-
 // QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI
 func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom(
 	ctx context.Context,

federationsender/inthttp/client.go

@@ -6,13 +6,12 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/federationsender/api"
-	internalHTTP "github.com/matrix-org/dendrite/internal/http"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/opentracing/opentracing-go"
 )
 
 // HTTP paths for the internal HTTP API
 const (
-	FederationSenderQueryJoinedHostsInRoomPath           = "/federationsender/queryJoinedHostsInRoom"
 	FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom"
 
 	FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup"
@@ -45,7 +44,7 @@ func (h *httpFederationSenderInternalAPI) PerformLeave(
 	defer span.Finish()
 
 	apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 func (h *httpFederationSenderInternalAPI) PerformServersAlive(
@@ -57,7 +56,7 @@ func (h *httpFederationSenderInternalAPI) PerformServersAlive(
 	defer span.Finish()
 
 	apiURL := h.federationSenderURL + FederationSenderPerformServersAlivePath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI
@@ -70,20 +69,7 @@ func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom(
 	defer span.Finish()
 
 	apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}
-
-// QueryJoinedHostsInRoom implements FederationSenderInternalAPI
-func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom(
-	ctx context.Context,
-	request *api.QueryJoinedHostsInRoomRequest,
-	response *api.QueryJoinedHostsInRoomResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom")
-	defer span.Finish()
-
-	apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // Handle an instruction to make_join & send_join with a remote server.
@@ -96,7 +82,7 @@ func (h *httpFederationSenderInternalAPI) PerformJoin(
 	defer span.Finish()
 
 	apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // Handle an instruction to make_join & send_join with a remote server.
@@ -109,5 +95,5 @@ func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup(
 	defer span.Finish()
 
 	apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath
-	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }

federationsender/inthttp/server.go

@@ -6,29 +6,15 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/federationsender/api"
-	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/util"
 )
 
 // AddRoutes adds the FederationSenderInternalAPI handlers to the http.ServeMux.
 func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Router) {
-	internalAPIMux.Handle(
-		FederationSenderQueryJoinedHostsInRoomPath,
-		internal.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryJoinedHostsInRoomRequest
-			var response api.QueryJoinedHostsInRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := intAPI.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
 	internalAPIMux.Handle(
 		FederationSenderQueryJoinedHostServerNamesInRoomPath,
-		internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
 			var request api.QueryJoinedHostServerNamesInRoomRequest
 			var response api.QueryJoinedHostServerNamesInRoomResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -41,7 +27,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route
 		}),
 	)
 	internalAPIMux.Handle(FederationSenderPerformJoinRequestPath,
-		internal.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse {
 			var request api.PerformJoinRequest
 			var response api.PerformJoinResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -54,7 +40,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route
 		}),
 	)
 	internalAPIMux.Handle(FederationSenderPerformLeaveRequestPath,
-		internal.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse {
 			var request api.PerformLeaveRequest
 			var response api.PerformLeaveResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -67,7 +53,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route
 		}),
 	)
 	internalAPIMux.Handle(FederationSenderPerformDirectoryLookupRequestPath,
-		internal.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse {
 			var request api.PerformDirectoryLookupRequest
 			var response api.PerformDirectoryLookupResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -80,7 +66,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route
 		}),
 	)
 	internalAPIMux.Handle(FederationSenderPerformServersAlivePath,
-		internal.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse {
 			var request api.PerformServersAliveRequest
 			var response api.PerformServersAliveResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {

federationsender/queue/queue.go

@@ -107,6 +107,9 @@ func (oqs *OutgoingQueues) SendEvent(
 
 	// Remove our own server from the list of destinations.
 	destinations = filterAndDedupeDests(oqs.origin, destinations)
+	if len(destinations) == 0 {
+		return nil
+	}
 
 	log.WithFields(log.Fields{
 		"destinations": destinations, "event": ev.EventID(),

federationsender/storage/postgres/joined_hosts_table.go

@@ -22,6 +22,7 @@ import (
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -85,7 +86,7 @@ func (s *joinedHostsStatements) insertJoinedHosts(
 	roomID, eventID string,
 	serverName gomatrixserverlib.ServerName,
 ) error {
-	stmt := internal.TxStmt(txn, s.insertJoinedHostsStmt)
+	stmt := sqlutil.TxStmt(txn, s.insertJoinedHostsStmt)
 	_, err := stmt.ExecContext(ctx, roomID, eventID, serverName)
 	return err
 }
@@ -93,7 +94,7 @@ func (s *joinedHostsStatements) insertJoinedHosts(
 func (s *joinedHostsStatements) deleteJoinedHosts(
 	ctx context.Context, txn *sql.Tx, eventIDs []string,
 ) error {
-	stmt := internal.TxStmt(txn, s.deleteJoinedHostsStmt)
+	stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsStmt)
 	_, err := stmt.ExecContext(ctx, pq.StringArray(eventIDs))
 	return err
 }
@@ -101,7 +102,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts(
 func (s *joinedHostsStatements) selectJoinedHostsWithTx(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) ([]types.JoinedHost, error) {
-	stmt := internal.TxStmt(txn, s.selectJoinedHostsStmt)
+	stmt := sqlutil.TxStmt(txn, s.selectJoinedHostsStmt)
 	return joinedHostsFromStmt(ctx, stmt, roomID)
 }
 

federationsender/storage/postgres/room_table.go

@@ -19,7 +19,7 @@ import (
 	"context"
 	"database/sql"
 
-	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
 const roomSchema = `
@@ -71,7 +71,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) {
 func (s *roomStatements) insertRoom(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) error {
-	_, err := internal.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID)
+	_, err := sqlutil.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID)
 	return err
 }
 
@@ -82,7 +82,7 @@ func (s *roomStatements) selectRoomForUpdate(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) (string, error) {
 	var lastEventID string
-	stmt := internal.TxStmt(txn, s.selectRoomForUpdateStmt)
+	stmt := sqlutil.TxStmt(txn, s.selectRoomForUpdateStmt)
 	err := stmt.QueryRowContext(ctx, roomID).Scan(&lastEventID)
 	if err != nil {
 		return "", err
@@ -95,7 +95,7 @@ func (s *roomStatements) selectRoomForUpdate(
 func (s *roomStatements) updateRoom(
 	ctx context.Context, txn *sql.Tx, roomID, lastEventID string,
 ) error {
-	stmt := internal.TxStmt(txn, s.updateRoomStmt)
+	stmt := sqlutil.TxStmt(txn, s.updateRoomStmt)
 	_, err := stmt.ExecContext(ctx, roomID, lastEventID)
 	return err
 }

federationsender/storage/postgres/storage.go

@@ -20,7 +20,6 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/federationsender/types"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
@@ -28,12 +27,12 @@ import (
 type Database struct {
 	joinedHostsStatements
 	roomStatements
-	internal.PartitionOffsetStatements
+	sqlutil.PartitionOffsetStatements
 	db *sql.DB
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
 	var result Database
 	var err error
 	if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
@@ -70,7 +69,7 @@ func (d *Database) UpdateRoom(
 	addHosts []types.JoinedHost,
 	removeHosts []string,
 ) (joinedHosts []types.JoinedHost, err error) {
-	err = internal.WithTransaction(d.db, func(txn *sql.Tx) error {
+	err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
 		err = d.insertRoom(ctx, txn, roomID)
 		if err != nil {
 			return err

federationsender/storage/sqlite3/joined_hosts_table.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -84,7 +85,7 @@ func (s *joinedHostsStatements) insertJoinedHosts(
 	roomID, eventID string,
 	serverName gomatrixserverlib.ServerName,
 ) error {
-	stmt := internal.TxStmt(txn, s.insertJoinedHostsStmt)
+	stmt := sqlutil.TxStmt(txn, s.insertJoinedHostsStmt)
 	_, err := stmt.ExecContext(ctx, roomID, eventID, serverName)
 	return err
 }
@@ -93,7 +94,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts(
 	ctx context.Context, txn *sql.Tx, eventIDs []string,
 ) error {
 	for _, eventID := range eventIDs {
-		stmt := internal.TxStmt(txn, s.deleteJoinedHostsStmt)
+		stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsStmt)
 		if _, err := stmt.ExecContext(ctx, eventID); err != nil {
 			return err
 		}
@@ -104,7 +105,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts(
 func (s *joinedHostsStatements) selectJoinedHostsWithTx(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) ([]types.JoinedHost, error) {
-	stmt := internal.TxStmt(txn, s.selectJoinedHostsStmt)
+	stmt := sqlutil.TxStmt(txn, s.selectJoinedHostsStmt)
 	return joinedHostsFromStmt(ctx, stmt, roomID)
 }
 

federationsender/storage/sqlite3/room_table.go

@@ -19,7 +19,7 @@ import (
 	"context"
 	"database/sql"
 
-	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
 const roomSchema = `
@@ -71,7 +71,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) {
 func (s *roomStatements) insertRoom(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) error {
-	_, err := internal.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID)
+	_, err := sqlutil.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID)
 	return err
 }
 
@@ -82,7 +82,7 @@ func (s *roomStatements) selectRoomForUpdate(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) (string, error) {
 	var lastEventID string
-	stmt := internal.TxStmt(txn, s.selectRoomForUpdateStmt)
+	stmt := sqlutil.TxStmt(txn, s.selectRoomForUpdateStmt)
 	err := stmt.QueryRowContext(ctx, roomID).Scan(&lastEventID)
 	if err != nil {
 		return "", err
@@ -95,7 +95,7 @@ func (s *roomStatements) selectRoomForUpdate(
 func (s *roomStatements) updateRoom(
 	ctx context.Context, txn *sql.Tx, roomID, lastEventID string,
 ) error {
-	stmt := internal.TxStmt(txn, s.updateRoomStmt)
+	stmt := sqlutil.TxStmt(txn, s.updateRoomStmt)
 	_, err := stmt.ExecContext(ctx, roomID, lastEventID)
 	return err
 }

federationsender/storage/sqlite3/storage.go

@@ -22,7 +22,6 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 
 	"github.com/matrix-org/dendrite/federationsender/types"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
@@ -30,7 +29,7 @@ import (
 type Database struct {
 	joinedHostsStatements
 	roomStatements
-	internal.PartitionOffsetStatements
+	sqlutil.PartitionOffsetStatements
 	db *sql.DB
 }
 
@@ -42,7 +41,7 @@ func NewDatabase(dataSourceName string) (*Database, error) {
 	if err != nil {
 		return nil, err
 	}
-	if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil {
+	if result.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {
@@ -76,7 +75,7 @@ func (d *Database) UpdateRoom(
 	addHosts []types.JoinedHost,
 	removeHosts []string,
 ) (joinedHosts []types.JoinedHost, err error) {
-	err = internal.WithTransaction(d.db, func(txn *sql.Tx) error {
+	err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
 		err = d.insertRoom(ctx, txn, roomID)
 		if err != nil {
 			return err

federationsender/storage/storage.go

@@ -21,11 +21,11 @@ import (
 
 	"github.com/matrix-org/dendrite/federationsender/storage/postgres"
 	"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) {
+func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return postgres.NewDatabase(dataSourceName, dbProperties)

federationsender/storage/storage_wasm.go

@@ -19,13 +19,13 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
 // NewDatabase opens a new database
 func NewDatabase(
 	dataSourceName string,
-	dbProperties internal.DbProperties, // nolint:unparam
+	dbProperties sqlutil.DbProperties, // nolint:unparam
 ) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200617141855-5539854e4abc
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible

go.sum

@@ -131,7 +131,6 @@ github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hjson/hjson-go v3.0.2-0.20200316202735-d5d0e8b0617d+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -372,8 +371,10 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1 h1:BfrvDrbjoPBvYua/3F/FmrqiZTRGrvtoMRgCVnrufMI=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65 h1:2CcCcBnWdDPDOqFKiGOM+mi/KDDZXSTKmvFy/0/+ZJI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200617141855-5539854e4abc h1:Oxidxr/H1Nh8aOFWAhTzQYLDOc9OuoJwfDjgEHpyqNE=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200617141855-5539854e4abc/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
@@ -566,8 +567,6 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yggdrasil-network/yggdrasil-extras v0.0.0-20200525205615-6c8a4a2e8855/go.mod h1:xQdsh08Io6nV4WRnOVTe6gI8/2iTvfLDQ0CYa5aMt+I=
-github.com/yggdrasil-network/yggdrasil-go v0.3.14 h1:vWzYzCQxOruS+J5FkLfXOS0JhCJx1yI9Erj/h2wfZ/E=
-github.com/yggdrasil-network/yggdrasil-go v0.3.14/go.mod h1:rkQzLzVHlFdzsEMG+bDdTI+KeWPCZq1HpXRFzwinf6M=
 github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b h1:ELOisSxFXCcptRs4LFub+Hz5fYUvV12wZrTps99Eb3E=
 github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -670,6 +669,7 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/caching/cache_serverkeys.go

@@ -15,17 +15,32 @@ const (
 // ServerKeyCache contains the subset of functions needed for
 // a server key cache.
 type ServerKeyCache interface {
-	GetServerKey(request gomatrixserverlib.PublicKeyLookupRequest) (response gomatrixserverlib.PublicKeyLookupResult, ok bool)
+	// request -> timestamp is emulating gomatrixserverlib.FetchKeys:
+	// https://github.com/matrix-org/gomatrixserverlib/blob/f69539c86ea55d1e2cc76fd8e944e2d82d30397c/keyring.go#L95
+	// The timestamp should be the timestamp of the event that is being
+	// verified. We will not return keys from the cache that are not valid
+	// at this timestamp.
+	GetServerKey(request gomatrixserverlib.PublicKeyLookupRequest, timestamp gomatrixserverlib.Timestamp) (response gomatrixserverlib.PublicKeyLookupResult, ok bool)
+
+	// request -> result is emulating gomatrixserverlib.StoreKeys:
+	// https://github.com/matrix-org/gomatrixserverlib/blob/f69539c86ea55d1e2cc76fd8e944e2d82d30397c/keyring.go#L112
 	StoreServerKey(request gomatrixserverlib.PublicKeyLookupRequest, response gomatrixserverlib.PublicKeyLookupResult)
 }
 
 func (c Caches) GetServerKey(
 	request gomatrixserverlib.PublicKeyLookupRequest,
+	timestamp gomatrixserverlib.Timestamp,
 ) (gomatrixserverlib.PublicKeyLookupResult, bool) {
 	key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID)
 	val, found := c.ServerKeys.Get(key)
 	if found && val != nil {
 		if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok {
+			if !keyLookupResult.WasValidAt(timestamp, true) {
+				// The key wasn't valid at the requested timestamp so don't
+				// return it. The caller will have to work out what to do.
+				c.ServerKeys.Unset(key)
+				return gomatrixserverlib.PublicKeyLookupResult{}, false
+			}
 			return keyLookupResult, true
 		}
 	}

internal/caching/caches.go

@@ -12,4 +12,5 @@ type Caches struct {
 type Cache interface {
 	Get(key string) (value interface{}, ok bool)
 	Set(key string, value interface{})
+	Unset(key string)
 }