Merge branch 'master' into tmp-file-delete

.dockerignore

@@ -0,0 +1,3 @@
+bin
+*.wasm
+.git

.editorconfig

@@ -1,18 +0,0 @@
-root = true
-
-[*]
-charset = utf-8
-
-end_of_line = lf
-insert_final_newline = true
-trim_trailing_whitespace = true
-
-[*.go]
-indent_style = tab
-indent_size = 4
-
-[*.md]
-trim_trailing_whitespace = false
-
-[*.{yml,yaml}]
-indent_style = space

.gitignore

@@ -49,3 +49,6 @@ dendrite.yaml
 
 # Log files
 *.log* 
+
+# Generated code
+cmd/dendrite-demo-yggdrasil/embed/fs*.go

README.md

@@ -3,15 +3,15 @@
 Dendrite will be a second-generation Matrix homeserver written in Go.
 
 It's still very much a work in progress, but installation instructions can be
-found in [INSTALL.md](INSTALL.md). It is not recommended to use Dendrite as a
+found in [INSTALL.md](docs/INSTALL.md). It is not recommended to use Dendrite as a
 production homeserver at this time.
 
-An overview of the design can be found in [DESIGN.md](DESIGN.md).
+An overview of the design can be found in [DESIGN.md](docs/DESIGN.md).
 
 # Contributing
 
 Everyone is welcome to help out and contribute! See
-[CONTRIBUTING.md](CONTRIBUTING.md) to get started!
+[CONTRIBUTING.md](docs/CONTRIBUTING.md) to get started!
 
 Please note that, as of February 2020, Dendrite now only targets Go 1.13 or
 later. Please ensure that you are using at least Go 1.13 when developing for

appservice/api/query.go

@@ -20,16 +20,11 @@ package api
 import (
 	"context"
 	"database/sql"
-	"errors"
-	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/matrix-org/dendrite/common"
-	commonHTTP "github.com/matrix-org/dendrite/common/http"
-	opentracing "github.com/opentracing/opentracing-go"
 )
 
 // RoomAliasExistsRequest is a request to an application service
@@ -83,60 +78,9 @@ type AppServiceQueryAPI interface {
 	) error
 }
 
-// AppServiceRoomAliasExistsPath is the HTTP path for the RoomAliasExists API
-const AppServiceRoomAliasExistsPath = "/api/appservice/RoomAliasExists"
-
-// AppServiceUserIDExistsPath is the HTTP path for the UserIDExists API
-const AppServiceUserIDExistsPath = "/api/appservice/UserIDExists"
-
-// httpAppServiceQueryAPI contains the URL to an appservice query API and a
-// reference to a httpClient used to reach it
-type httpAppServiceQueryAPI struct {
-	appserviceURL string
-	httpClient    *http.Client
-}
-
-// NewAppServiceQueryAPIHTTP creates a AppServiceQueryAPI implemented by talking
-// to a HTTP POST API.
-// If httpClient is nil an error is returned
-func NewAppServiceQueryAPIHTTP(
-	appserviceURL string,
-	httpClient *http.Client,
-) (AppServiceQueryAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is <nil>")
-	}
-	return &httpAppServiceQueryAPI{appserviceURL, httpClient}, nil
-}
-
-// RoomAliasExists implements AppServiceQueryAPI
-func (h *httpAppServiceQueryAPI) RoomAliasExists(
-	ctx context.Context,
-	request *RoomAliasExistsRequest,
-	response *RoomAliasExistsResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceRoomAliasExists")
-	defer span.Finish()
-
-	apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath
-	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}
-
-// UserIDExists implements AppServiceQueryAPI
-func (h *httpAppServiceQueryAPI) UserIDExists(
-	ctx context.Context,
-	request *UserIDExistsRequest,
-	response *UserIDExistsResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceUserIDExists")
-	defer span.Finish()
-
-	apiURL := h.appserviceURL + AppServiceUserIDExistsPath
-	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}
-
 // RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
+// TODO: Remove this, it's called from federationapi and clientapi but is a pure function
 func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
@@ -165,7 +109,7 @@ func RetrieveUserProfile(
 
 	// If no user exists, return
 	if !userResp.UserIDExists {
-		return nil, common.ErrProfileNoExists
+		return nil, eventutil.ErrProfileNoExists
 	}
 
 	// Try to query the user from the local database again

appservice/appservice.go

@@ -20,36 +20,35 @@ import (
 	"sync"
 	"time"
 
+	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/appservice/consumers"
+	"github.com/matrix-org/dendrite/appservice/inthttp"
 	"github.com/matrix-org/dendrite/appservice/query"
-	"github.com/matrix-org/dendrite/appservice/routing"
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/appservice/workers"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/common/basecomponent"
-	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/common/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
 )
 
-// SetupAppServiceAPIComponent sets up and registers HTTP handlers for the AppServices
+// AddInternalRoutes registers HTTP handlers for internal API calls
-// component.
+func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQueryAPI) {
-func SetupAppServiceAPIComponent(
+	inthttp.AddRoutes(queryAPI, router)
-	base *basecomponent.BaseDendrite,
+}
-	accountsDB accounts.Database,
+
-	deviceDB devices.Database,
+// NewInternalAPI returns a concerete implementation of the internal API. Callers
-	federation *gomatrixserverlib.FederationClient,
+// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
-	roomserverAliasAPI roomserverAPI.RoomserverAliasAPI,
+func NewInternalAPI(
-	roomserverQueryAPI roomserverAPI.RoomserverQueryAPI,
+	base *setup.BaseDendrite,
-	transactionsCache *transactions.Cache,
+	userAPI userapi.UserInternalAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceQueryAPI {
 	// Create a connection to the appservice postgres DB
-	appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService))
+	appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to appservice db")
 	}
@@ -67,7 +66,7 @@ func SetupAppServiceAPIComponent(
 		workerStates[i] = ws
 
 		// Create bot account for this AS if it doesn't already exist
-		if err = generateAppServiceAccount(accountsDB, deviceDB, appservice); err != nil {
+		if err = generateAppServiceAccount(userAPI, appservice); err != nil {
 			logrus.WithFields(logrus.Fields{
 				"appservice": appservice.ID,
 			}).WithError(err).Panicf("failed to generate bot account for appservice")
@@ -76,57 +75,55 @@ func SetupAppServiceAPIComponent(
 
 	// Create appserivce query API with an HTTP client that will be used for all
 	// outbound and inbound requests (inbound only for the internal API)
-	appserviceQueryAPI := query.AppServiceQueryAPI{
+	appserviceQueryAPI := &query.AppServiceQueryAPI{
 		HTTPClient: &http.Client{
 			Timeout: time.Second * 30,
 		},
 		Cfg: base.Cfg,
 	}
 
-	appserviceQueryAPI.SetupHTTP(http.DefaultServeMux)
+	// Only consume if we actually have ASes to track, else we'll just chew cycles needlessly.
-
+	// We can't add ASes at runtime so this is safe to do.
-	consumer := consumers.NewOutputRoomEventConsumer(
+	if len(workerStates) > 0 {
-		base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB,
+		consumer := consumers.NewOutputRoomEventConsumer(
-		roomserverQueryAPI, roomserverAliasAPI, workerStates,
+			base.Cfg, base.KafkaConsumer, appserviceDB,
-	)
+			rsAPI, workerStates,
-	if err := consumer.Start(); err != nil {
+		)
-		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
+		if err := consumer.Start(); err != nil {
+			logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
+		}
 	}
 
 	// Create application service transaction workers
 	if err := workers.SetupTransactionWorkers(appserviceDB, workerStates); err != nil {
 		logrus.WithError(err).Panicf("failed to start app service transaction workers")
 	}
-
+	return appserviceQueryAPI
-	// Set up HTTP Endpoints
-	routing.Setup(
-		base.APIMux, base.Cfg, roomserverQueryAPI, roomserverAliasAPI,
-		accountsDB, federation, transactionsCache,
-	)
-
-	return &appserviceQueryAPI
 }
 
 // generateAppServiceAccounts creates a dummy account based off the
 // `sender_localpart` field of each application service if it doesn't
 // exist already
 func generateAppServiceAccount(
-	accountsDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 	as config.ApplicationService,
 ) error {
-	ctx := context.Background()
+	var accRes userapi.PerformAccountCreationResponse
-
+	err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{
-	// Create an account for the application service
+		AccountType:  userapi.AccountTypeUser,
-	acc, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID)
+		Localpart:    as.SenderLocalpart,
+		AppServiceID: as.ID,
+		OnConflict:   userapi.ConflictUpdate,
+	}, &accRes)
 	if err != nil {
 		return err
-	} else if acc == nil {
-		// This account already exists
-		return nil
 	}
-
+	var devRes userapi.PerformDeviceCreationResponse
-	// Create a dummy device with a dummy token for the application service
+	err = userAPI.PerformDeviceCreation(context.Background(), &userapi.PerformDeviceCreationRequest{
-	_, err = deviceDB.CreateDevice(ctx, as.SenderLocalpart, nil, as.ASToken, &as.SenderLocalpart)
+		Localpart:         as.SenderLocalpart,
+		AccessToken:       as.ASToken,
+		DeviceID:          &as.SenderLocalpart,
+		DeviceDisplayName: &as.SenderLocalpart,
+	}, &devRes)
 	return err
 }

appservice/consumers/roomserver.go

@@ -20,23 +20,20 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/Shopify/sarama"
 	log "github.com/sirupsen/logrus"
-	sarama "gopkg.in/Shopify/sarama.v1"
 )
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	roomServerConsumer *common.ContinualConsumer
+	roomServerConsumer *internal.ContinualConsumer
-	db                 accounts.Database
 	asDB               storage.Database
-	query              api.RoomserverQueryAPI
+	rsAPI              api.RoomserverInternalAPI
-	alias              api.RoomserverAliasAPI
 	serverName         string
 	workerStates       []types.ApplicationServiceWorkerState
 }
@@ -46,23 +43,19 @@ type OutputRoomEventConsumer struct {
 func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
-	store accounts.Database,
 	appserviceDB storage.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
-	aliasAPI api.RoomserverAliasAPI,
 	workerStates []types.ApplicationServiceWorkerState,
 ) *OutputRoomEventConsumer {
-	consumer := common.ContinualConsumer{
+	consumer := internal.ContinualConsumer{
 		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
 		Consumer:       kafkaConsumer,
-		PartitionStore: store,
+		PartitionStore: appserviceDB,
 	}
 	s := &OutputRoomEventConsumer{
 		roomServerConsumer: &consumer,
-		db:                 store,
 		asDB:               appserviceDB,
-		query:              queryAPI,
+		rsAPI:              rsAPI,
-		alias:              aliasAPI,
 		serverName:         string(cfg.Matrix.ServerName),
 		workerStates:       workerStates,
 	}
@@ -94,60 +87,13 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 		return nil
 	}
 
-	ev := output.NewRoomEvent.Event
+	events := []gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
-	log.WithFields(log.Fields{
+	events = append(events, output.NewRoomEvent.AddStateEvents...)
-		"event_id": ev.EventID(),
-		"room_id":  ev.RoomID(),
-		"type":     ev.Type(),
-	}).Info("appservice received an event from roomserver")
-
-	missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev)
-	if err != nil {
-		return err
-	}
-	events := append(missingEvents, ev)
 
 	// Send event to any relevant application services
 	return s.filterRoomserverEvents(context.TODO(), events)
 }
 
-// lookupMissingStateEvents looks up the state events that are added by a new event,
-// and returns any not already present.
-func (s *OutputRoomEventConsumer) lookupMissingStateEvents(
-	addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent,
-) ([]gomatrixserverlib.HeaderedEvent, error) {
-	// Fast path if there aren't any new state events.
-	if len(addsStateEventIDs) == 0 {
-		return []gomatrixserverlib.HeaderedEvent{}, nil
-	}
-
-	// Fast path if the only state event added is the event itself.
-	if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() {
-		return []gomatrixserverlib.HeaderedEvent{}, nil
-	}
-
-	result := []gomatrixserverlib.HeaderedEvent{}
-	missing := []string{}
-	for _, id := range addsStateEventIDs {
-		if id != event.EventID() {
-			// If the event isn't the current one, add it to the list of events
-			// to retrieve from the roomserver
-			missing = append(missing, id)
-		}
-	}
-
-	// Request the missing events from the roomserver
-	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
-	var eventResp api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
-		return nil, err
-	}
-
-	result = append(result, eventResp.Events...)
-
-	return result, nil
-}
-
 // filterRoomserverEvents takes in events and decides whether any of them need
 // to be passed on to an external application service. It does this by checking
 // each namespace of each registered application service, and if there is a
@@ -200,7 +146,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	// Check all known room aliases of the room the event came from
 	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
 	var queryRes api.GetAliasesForRoomIDResponse
-	if err := s.alias.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
+	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
 		for _, alias := range queryRes.Aliases {
 			if appservice.IsInterestedInRoomAlias(alias) {
 				return true

appservice/inthttp/client.go

@@ -0,0 +1,63 @@
+package inthttp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/opentracing/opentracing-go"
+)
+
+// HTTP paths for the internal HTTP APIs
+const (
+	AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists"
+	AppServiceUserIDExistsPath    = "/appservice/UserIDExists"
+)
+
+// httpAppServiceQueryAPI contains the URL to an appservice query API and a
+// reference to a httpClient used to reach it
+type httpAppServiceQueryAPI struct {
+	appserviceURL string
+	httpClient    *http.Client
+}
+
+// NewAppserviceClient creates a AppServiceQueryAPI implemented by talking
+// to a HTTP POST API.
+// If httpClient is nil an error is returned
+func NewAppserviceClient(
+	appserviceURL string,
+	httpClient *http.Client,
+) (api.AppServiceQueryAPI, error) {
+	if httpClient == nil {
+		return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is <nil>")
+	}
+	return &httpAppServiceQueryAPI{appserviceURL, httpClient}, nil
+}
+
+// RoomAliasExists implements AppServiceQueryAPI
+func (h *httpAppServiceQueryAPI) RoomAliasExists(
+	ctx context.Context,
+	request *api.RoomAliasExistsRequest,
+	response *api.RoomAliasExistsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceRoomAliasExists")
+	defer span.Finish()
+
+	apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
+// UserIDExists implements AppServiceQueryAPI
+func (h *httpAppServiceQueryAPI) UserIDExists(
+	ctx context.Context,
+	request *api.UserIDExistsRequest,
+	response *api.UserIDExistsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceUserIDExists")
+	defer span.Finish()
+
+	apiURL := h.appserviceURL + AppServiceUserIDExistsPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

appservice/inthttp/server.go

@@ -0,0 +1,43 @@
+package inthttp
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/util"
+)
+
+// AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
+func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) {
+	internalAPIMux.Handle(
+		AppServiceRoomAliasExistsPath,
+		httputil.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
+			var request api.RoomAliasExistsRequest
+			var response api.RoomAliasExistsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := a.RoomAliasExists(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(
+		AppServiceUserIDExistsPath,
+		httputil.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse {
+			var request api.UserIDExistsRequest
+			var response api.UserIDExistsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := a.UserIDExists(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+}

appservice/query/query.go

@@ -18,15 +18,12 @@ package query
 
 import (
 	"context"
-	"encoding/json"
 	"net/http"
 	"net/url"
 	"time"
 
 	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/util"
 	opentracing "github.com/opentracing/opentracing-go"
 	log "github.com/sirupsen/logrus"
 )
@@ -179,36 +176,3 @@ func makeHTTPClient() *http.Client {
 		Timeout: time.Second * 30,
 	}
 }
-
-// SetupHTTP adds the AppServiceQueryPAI handlers to the http.ServeMux. This
-// handles and muxes incoming api requests the to internal AppServiceQueryAPI.
-func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) {
-	servMux.Handle(
-		api.AppServiceRoomAliasExistsPath,
-		common.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
-			var request api.RoomAliasExistsRequest
-			var response api.RoomAliasExistsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := a.RoomAliasExists(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	servMux.Handle(
-		api.AppServiceUserIDExistsPath,
-		common.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse {
-			var request api.UserIDExistsRequest
-			var response api.UserIDExistsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := a.UserIDExists(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-}

appservice/routing/routing.go

@@ -1,65 +0,0 @@
-// Copyright 2018 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package routing
-
-import (
-	"net/http"
-
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/common/transactions"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-const pathPrefixApp = "/_matrix/app/v1"
-
-// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
-// to clients which need to make outbound HTTP requests.
-//
-// Due to Setup being used to call many other functions, a gocyclo nolint is
-// applied:
-// nolint: gocyclo
-func Setup(
-	apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam
-	queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam
-	accountDB accounts.Database, // nolint: unparam
-	federation *gomatrixserverlib.FederationClient, // nolint: unparam
-	transactionsCache *transactions.Cache, // nolint: unparam
-) {
-	appMux := apiMux.PathPrefix(pathPrefixApp).Subrouter()
-
-	appMux.Handle("/alias",
-		common.MakeExternalAPI("alias", func(req *http.Request) util.JSONResponse {
-			// TODO: Implement
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: nil,
-			}
-		}),
-	).Methods(http.MethodGet, http.MethodOptions)
-	appMux.Handle("/user",
-		common.MakeExternalAPI("user", func(req *http.Request) util.JSONResponse {
-			// TODO: Implement
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: nil,
-			}
-		}),
-	).Methods(http.MethodGet, http.MethodOptions)
-}

appservice/storage/interface.go

@@ -17,10 +17,12 @@ package storage
 import (
 	"context"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
+	internal.PartitionStorer
 	StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.HeaderedEvent) error
 	GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error)
 	CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error)

appservice/storage/postgres/storage.go

@@ -27,21 +27,25 @@ import (
 
 // Database stores events intended to be later sent to application services
 type Database struct {
+	sqlutil.PartitionOffsetStatements
 	events eventsStatements
 	txnID  txnStatements
 	db     *sql.DB
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {
 		return nil, err
 	}
+	if err = result.PartitionOffsetStatements.Prepare(result.db, "appservice"); err != nil {
+		return nil, err
+	}
 	return &result, nil
 }
 

appservice/storage/sqlite3/storage.go

@@ -20,7 +20,6 @@ import (
 	"database/sql"
 
 	// Import SQLite database driver
-	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	_ "github.com/mattn/go-sqlite3"
@@ -28,6 +27,7 @@ import (
 
 // Database stores events intended to be later sent to application services
 type Database struct {
+	sqlutil.PartitionOffsetStatements
 	events eventsStatements
 	txnID  txnStatements
 	db     *sql.DB
@@ -37,12 +37,19 @@ type Database struct {
 func NewDatabase(dataSourceName string) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	cs, err := sqlutil.ParseFileURI(dataSourceName)
+	if err != nil {
+		return nil, err
+	}
+	if result.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {
 		return nil, err
 	}
+	if err = result.PartitionOffsetStatements.Prepare(result.db, "appservice"); err != nil {
+		return nil, err
+	}
 	return &result, nil
 }
 

appservice/storage/storage.go

@@ -21,19 +21,22 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage/postgres"
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func NewDatabase(dataSourceName string) (Database, error) {
+// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
+// and sets DB connection parameters
+func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName)
 	default:
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 }

appservice/storage/storage_wasm.go

@@ -19,9 +19,13 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func NewDatabase(dataSourceName string) (Database, error) {
+func NewDatabase(
+	dataSourceName string,
+	dbProperties sqlutil.DbProperties, // nolint:unparam
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

appservice/types/types.go

@@ -15,7 +15,7 @@ package types
 import (
 	"sync"
 
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
 const (

appservice/workers/transaction_scheduler.go

@@ -21,11 +21,12 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	log "github.com/sirupsen/logrus"
 )
@@ -207,9 +208,15 @@ func send(
 	txnID int,
 	transaction []byte,
 ) error {
-	// POST a transaction to our AS
+	// PUT a transaction to our AS
-	address := fmt.Sprintf("%s/transactions/%d", appservice.URL, txnID)
+	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
-	resp, err := client.Post(address, "application/json", bytes.NewBuffer(transaction))
+	address := fmt.Sprintf("%s/transactions/%d?access_token=%s", appservice.URL, txnID, url.QueryEscape(appservice.HSToken))
+	req, err := http.NewRequest("PUT", address, bytes.NewBuffer(transaction))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}

are-we-synapse-yet.list

@@ -28,6 +28,10 @@ log POST /login returns the same device_id as that in the request
 log POST /login can log in as a user with just the local part of the id 
 log POST /login as non-existing user is rejected 
 log POST /login wrong password is rejected 
+log Interactive authentication types include SSO
+log Can perform interactive authentication with SSO
+log The user must be consistent through an interactive authentication session with SSO
+log The operation must be consistent through an interactive authentication session
 v1s GET /events initially
 v1s GET /initialSync initially
 csa Version responds 200 OK with valid structure
@@ -44,6 +48,7 @@ dev DELETE /device/{deviceId}
 dev DELETE /device/{deviceId} requires UI auth user to match device owner 
 dev DELETE /device/{deviceId} with no body gives a 401 
 dev The deleted device must be consistent through an interactive auth session 
+dev Users receive device_list updates for their own devices
 pre GET /presence/:user_id/status fetches initial status 
 pre PUT /presence/:user_id/status updates my presence 
 crm POST /createRoom makes a public room
@@ -92,8 +97,8 @@ rst PUT power_levels should not explode if the old power levels were empty
 rst Both GET and PUT work
 rct POST /rooms/:room_id/receipt can create receipts 
 red POST /rooms/:room_id/read_markers can create read marker
-med POST /media/v1/upload can create an upload 
+med POST /media/r0/upload can create an upload 
-med GET /media/v1/download can fetch the value again 
+med GET /media/r0/download can fetch the value again 
 cap GET /capabilities is present and well formed for registered user 
 cap GET /r0/capabilities is not public
 reg Register with a recaptcha 
@@ -450,6 +455,19 @@ rmv User can invite remote user to room with version 5
 rmv Remote user can backfill in a room with version 5 
 rmv Can reject invites over federation for rooms with version 5 
 rmv Can receive redactions from regular users over federation in room version 5 
+rmv User can create and send/receive messages in a room with version 6 
+rmv User can create and send/receive messages in a room with version 6 (2 subtests)
+rmv local user can join room with version 6 
+rmv User can invite local user to room with version 6 
+rmv remote user can join room with version 6 
+rmv User can invite remote user to room with version 6 
+rmv Remote user can backfill in a room with version 6 
+rmv Can reject invites over federation for rooms with version 6 
+rmv Can receive redactions from regular users over federation in room version 6 
+rmv Inbound federation rejects invites which include invalid JSON for room version 6
+rmv Outbound federation rejects invite response which include invalid JSON for room version 6
+rmv Inbound federation rejects invite rejections which include invalid JSON for room version 6
+rmv Server rejects invalid JSON in a version 6 room
 pre Presence changes are reported to local room members 
 f,pre Presence changes are also reported to remote room members 
 pre Presence changes to UNAVAILABLE are reported to local room members 
@@ -531,11 +549,11 @@ std Can recv device messages until they are acknowledged
 std Device messages with the same txn_id are deduplicated 
 std Device messages wake up /sync 
 std Can recv device messages over federation 
-std Device messages over federation wake up /sync 
+fsd Device messages over federation wake up /sync 
 std Can send messages with a wildcard device id 
 std Can send messages with a wildcard device id to two devices 
 std Wildcard device messages wake up /sync 
-std Wildcard device messages over federation wake up /sync 
+fsd Wildcard device messages over federation wake up /sync 
 adm /whois 
 nsp /purge_history 
 nsp /purge_history by ts 
@@ -573,6 +591,7 @@ frv A pair of servers can establish a join in a v2 room
 fsj Outbound federation rejects send_join responses with no m.room.create event 
 frv Outbound federation rejects m.room.create events with an unknown room version 
 fsj Event with an invalid signature in the send_join response should not cause room join to fail 
+fsj Inbound: send_join rejects invalid JSON for room version 6
 fed Outbound federation can send events
 fed Inbound federation can receive events 
 fed Inbound federation can receive redacted events 
@@ -631,6 +650,7 @@ fst Name/topic keys are correct
 fau Remote servers cannot set power levels in rooms without existing powerlevels 
 fau Remote servers should reject attempts by non-creators to set the power levels 
 fau Inbound federation rejects typing notifications from wrong remote 
+fau Users cannot set notifications powerlevel higher than their own
 fed Forward extremities remain so even after the next events are populated as outliers 
 fau Banned servers cannot send events 
 fau Banned servers cannot /make_join 
@@ -827,4 +847,10 @@ syn Multiple calls to /sync should not cause 500 errors
 gst Guest user can call /events on another world_readable room (SYN-606) 
 gst Real user can call /events on another world_readable room (SYN-606) 
 gst Events come down the correct room
-pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list
+pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list
+std Can send a to-device message to two users which both receive it using /sync
+fme Outbound federation will ignore a missing event with bad JSON for room version 6
+fbk Outbound federation rejects backfill containing invalid JSON for events in room version 6
+jso Invalid JSON integers
+jso Invalid JSON floats
+jso Invalid JSON special values

are-we-synapse-yet.py

@@ -33,6 +33,7 @@ import sys
 
 test_mappings = {
     "nsp": "Non-Spec API",
+    "unk": "Unknown API (no group specified)",
     "f": "Federation", # flag to mark test involves federation
 
     "federation_apis": {
@@ -50,6 +51,7 @@ test_mappings = {
         "fpb": "Public Room API",
         "fdk": "Device Key APIs",
         "fed": "Federation API",
+		"fsd": "Send-to-Device APIs",
     },
 
     "client_apis": {
@@ -99,6 +101,7 @@ test_mappings = {
         "ign": "Ignore Users",
         "udr": "User Directory APIs",
         "app": "Application Services API",
+		"jso": "Enforced canonical JSON",
     },
 }
 
@@ -212,7 +215,8 @@ def main(results_tap_path, verbose):
             # }
         },
         "nonspec": {
-            "nsp": {}
+            "nsp": {},
+            "unk": {}
         },
     }
     with open(results_tap_path, "r") as f:
@@ -223,7 +227,7 @@ def main(results_tap_path, verbose):
             name = test_result["name"]
             group_id = test_name_to_group_id.get(name)
             if not group_id:
-                raise Exception("The test '%s' doesn't have a group" % (name,))
+                summary["nonspec"]["unk"][name] = test_result["ok"]
             if group_id == "nsp":
                 summary["nonspec"]["nsp"][name] = test_result["ok"]
             elif group_id in test_mappings["federation_apis"]:

build-dendritejs.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -eu
+
+export GIT_COMMIT=$(git rev-list -1 HEAD) && \
+GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs

build.sh

@@ -3,4 +3,6 @@
 # Put installed packages into ./bin
 export GOBIN=$PWD/`dirname $0`/bin
 
-go install -v $PWD/`dirname $0`/cmd/...
+go install -v $PWD/`dirname $0`/cmd/...
+
+GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs

build/docker/DendriteJS.Dockerfile

@@ -0,0 +1,111 @@
+# This dockerfile will build dendritejs and hook it up to riot-web, build that then dump the
+# resulting HTML/JS onto an nginx container for hosting. It requires no specific build context
+# as it pulls archives straight from github branches.
+#
+# $ docker build -t dendritejs -f DendriteJS.Dockerfile .
+# $ docker run --rm -p 8888:80 dendritejs
+# Then visit http://localhost:8888
+FROM golang:1.13.7-alpine3.11 AS gobuild
+
+# Download and build dendrite
+WORKDIR /build
+ADD https://github.com/matrix-org/dendrite/archive/master.tar.gz /build/master.tar.gz
+RUN tar xvfz master.tar.gz
+WORKDIR /build/dendrite-master
+RUN GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs
+
+
+FROM node:14-stretch AS jsbuild
+# apparently some deps require python
+RUN apt-get update && apt-get -y install python
+
+# Download riot-web and libp2p repos
+WORKDIR /build
+ADD https://github.com/matrix-org/go-http-js-libp2p/archive/master.tar.gz /build/libp2p.tar.gz
+RUN tar xvfz libp2p.tar.gz
+ADD https://github.com/vector-im/riot-web/archive/matthew/p2p.tar.gz /build/p2p.tar.gz
+RUN tar xvfz p2p.tar.gz
+
+# Install deps for riot-web, symlink in libp2p repo and build that too
+WORKDIR /build/riot-web-matthew-p2p
+RUN yarn install
+RUN ln -s /build/go-http-js-libp2p-master /build/riot-web-matthew-p2p/node_modules/go-http-js-libp2p
+RUN (cd node_modules/go-http-js-libp2p && yarn install)
+COPY --from=gobuild /build/dendrite-master/main.wasm ./src/vector/dendrite.wasm
+# build it all
+RUN yarn build:p2p
+
+SHELL ["/bin/bash", "-c"]
+RUN echo $'\ 
+{ \n\
+    "default_server_config": { \n\
+        "m.homeserver": { \n\
+            "base_url": "https://p2p.riot.im", \n\
+            "server_name": "p2p.riot.im" \n\
+        }, \n\
+        "m.identity_server": { \n\
+            "base_url": "https://vector.im" \n\
+        } \n\
+    }, \n\
+    "disable_custom_urls": false, \n\
+    "disable_guests": true, \n\
+    "disable_login_language_selector": false, \n\
+    "disable_3pid_login": true, \n\
+    "brand": "Riot", \n\
+    "integrations_ui_url": "https://scalar.vector.im/", \n\
+    "integrations_rest_url": "https://scalar.vector.im/api", \n\
+    "integrations_widgets_urls": [ \n\
+        "https://scalar.vector.im/_matrix/integrations/v1", \n\
+        "https://scalar.vector.im/api", \n\
+        "https://scalar-staging.vector.im/_matrix/integrations/v1", \n\
+        "https://scalar-staging.vector.im/api", \n\
+        "https://scalar-staging.riot.im/scalar/api" \n\
+    ], \n\
+    "integrations_jitsi_widget_url": "https://scalar.vector.im/api/widgets/jitsi.html", \n\
+    "bug_report_endpoint_url": "https://riot.im/bugreports/submit", \n\
+    "defaultCountryCode": "GB", \n\
+    "showLabsSettings": false, \n\
+    "features": { \n\
+        "feature_pinning": "labs", \n\
+        "feature_custom_status": "labs", \n\
+        "feature_custom_tags": "labs", \n\
+        "feature_state_counters": "labs" \n\
+    }, \n\
+    "default_federate": true, \n\
+    "default_theme": "light", \n\
+    "roomDirectory": { \n\
+        "servers": [ \n\
+            "matrix.org" \n\
+        ] \n\
+    }, \n\
+    "welcomeUserId": "", \n\
+    "piwik": { \n\
+        "url": "https://piwik.riot.im/", \n\
+        "whitelistedHSUrls": ["https://matrix.org"], \n\
+        "whitelistedISUrls": ["https://vector.im", "https://matrix.org"], \n\
+        "siteId": 1 \n\
+    }, \n\
+    "enable_presence_by_hs_url": { \n\
+        "https://matrix.org": false, \n\
+        "https://matrix-client.matrix.org": false \n\
+    }, \n\
+    "settingDefaults": { \n\
+        "breadcrumbs": true \n\
+    } \n\
+}' > webapp/config.json
+
+FROM nginx
+# Add "Service-Worker-Allowed: /" header so the worker can sniff traffic on this domain rather
+# than just the path this gets hosted under. NB this newline echo syntax only works on bash.
+SHELL ["/bin/bash", "-c"]
+RUN echo $'\
+server { \n\
+    listen 80; \n\
+    add_header \'Service-Worker-Allowed\' \'/\'; \n\
+    location / { \n\
+        root   /usr/share/nginx/html; \n\
+        index  index.html index.htm; \n\
+    } \n\
+}' > /etc/nginx/conf.d/default.conf
+RUN sed -i 's/}/    application\/wasm  wasm;\n}/g' /etc/nginx/mime.types
+COPY --from=jsbuild /build/riot-web-matthew-p2p/webapp /usr/share/nginx/html

build/docker/Dockerfile

@@ -0,0 +1,10 @@
+FROM docker.io/golang:1.13.7-alpine3.11 AS builder
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN sh ./build.sh

build/docker/Dockerfile.component

@@ -0,0 +1,13 @@
+FROM matrixdotorg/dendrite:latest AS base
+
+FROM alpine:latest
+
+ARG component=monolith
+ENV entrypoint=${component}
+
+COPY --from=base /build/bin/${component} /usr/bin
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT /usr/bin/${entrypoint} $@

build/docker/README.md

@@ -0,0 +1,70 @@
+# Docker images
+
+These are Docker images for Dendrite!
+
+## Dockerfiles
+
+The `Dockerfile` builds the base image which contains all of the Dendrite
+components. The `Dockerfile.component` file takes the given component, as
+specified with `--buildarg component=` from the base image and produce
+smaller component-specific images, which are substantially smaller and do
+not contain the Go toolchain etc.
+
+## Compose files
+
+There are three sample `docker-compose` files:
+
+- `docker-compose.deps.yml` which runs the Postgres and Kafka prerequisites
+- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment
+- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment
+
+## Configuration
+
+The `docker-compose` files refer to the `/etc/dendrite` volume as where the
+runtime config should come from. The mounted folder must contain:
+
+- `dendrite.yaml` configuration file (based on the sample `dendrite-config.yaml`
+   in the `docker/config` folder in the [Dendrite repository](https://github.com/matrix-org/dendrite)
+- `matrix_key.pem` server key, as generated using `cmd/generate-keys`
+- `server.crt` certificate file
+- `server.key` private key file for the above certificate
+
+To generate keys:
+
+```
+go run github.com/matrix-org/dendrite/cmd/generate-keys \
+  --private-key=matrix_key.pem \
+  --tls-cert=server.crt \
+  --tls-key=server.key
+```
+
+## Starting Dendrite
+
+Once in place, start the dependencies:
+
+```
+docker-compose -f docker-compose.deps.yml up
+```
+
+Wait a few seconds for Kafka and Postgres to finish starting up, and then start a monolith:
+
+```
+docker-compose -f docker-compose.monolith.yml up
+```
+
+... or start the polylith components:
+
+```
+docker-compose -f docker-compose.polylith.yml up
+```
+
+## Building the images
+
+The `docker/images-build.sh` script will build the base image, followed by
+all of the component images.
+
+The `docker/images-push.sh` script will push them to Docker Hub (subject
+to permissions).
+
+If you wish to build and push your own images, rename `matrixdotorg/dendrite` to
+the name of another Docker Hub repository in `images-build.sh` and `images-push.sh`.

build/docker/config/dendrite-config.yaml

@@ -80,7 +80,7 @@ kafka:
     # Kafka can be used both with a monolithic server and when running the
     # components as separate servers.
     # If enabled database.naffka must also be specified.
-    use_naffka: true
+    use_naffka: false
     # The names of the kafka topics to use.
     topics:
         output_room_event: roomserverOutput
@@ -101,7 +101,7 @@ database:
     public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable"
     appservice: "postgres://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable"
     # If using naffka you need to specify a naffka database
-    naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable"
+    #naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable"
 
 # The TCP host:port pairs to bind the internal HTTP APIs to.
 # These shouldn't be exposed to the public internet.
@@ -110,11 +110,15 @@ listen:
     room_server: "room_server:7770"
     client_api: "client_api:7771"
     federation_api: "federation_api:7772"
+    server_key_api: "server_key_api:7778"
     sync_api: "sync_api:7773"
     media_api: "media_api:7774"
     public_rooms_api: "public_rooms_api:7775"
     federation_sender: "federation_sender:7776"
-    edu_server: "typing_server:7777"
+    edu_server: "edu_server:7777"
+    key_server: "key_server:7779"
+    user_api: "user_api:7780"
+    appservice_api: "appservice_api:7781"
 
 # The configuration for tracing the dendrite components.
 tracing:

build/docker/docker-compose.deps.yml

@@ -0,0 +1,36 @@
+version: "3.4"
+services:
+  postgres:
+    hostname: postgres
+    image: postgres:9.5
+    restart: always
+    volumes:
+      - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
+    environment:
+      POSTGRES_PASSWORD: itsasecret
+      POSTGRES_USER: dendrite
+    networks:
+      - internal
+
+  zookeeper:
+    hostname: zookeeper
+    image: zookeeper
+    networks:
+      - internal
+
+  kafka:
+    container_name: dendrite_kafka
+    hostname: kafka
+    image: wurstmeister/kafka
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: "kafka"
+      KAFKA_DELETE_TOPIC_ENABLE: "true"
+      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+    depends_on:
+      - zookeeper
+    networks:
+      - internal
+
+networks:
+  internal:
+    attachable: true

build/docker/docker-compose.monolith.yml

@@ -0,0 +1,18 @@
+version: "3.4"
+services:
+  monolith:
+    hostname: monolith
+    image: matrixdotorg/dendrite:monolith
+    command: [
+      "--config=dendrite.yaml",
+      "--tls-cert=server.crt",
+      "--tls-key=server.key"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+networks:
+  internal:
+    attachable: true

build/docker/docker-compose.polylith.yml

@@ -0,0 +1,182 @@
+version: "3.4"
+services:
+  client_api_proxy:
+    hostname: client_api_proxy
+    image: matrixdotorg/dendrite:clientproxy
+    command: [
+      "--bind-address=:8008",
+      "--client-api-server-url=http://client_api:7771",
+      "--sync-api-server-url=http://sync_api:7773",
+      "--media-api-server-url=http://media_api:7774",
+      "--public-rooms-api-server-url=http://public_rooms_api:7775"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+    depends_on:
+      - sync_api
+      - client_api
+      - media_api
+      - public_rooms_api
+    ports:
+      - "8008:8008"
+
+  client_api:
+    hostname: client_api
+    image: matrixdotorg/dendrite:clientapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+      - room_server
+    networks:
+      - internal
+
+  media_api:
+    hostname: media_api
+    image: matrixdotorg/dendrite:mediaapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  public_rooms_api:
+    hostname: public_rooms_api
+    image: matrixdotorg/dendrite:publicroomsapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  sync_api:
+    hostname: sync_api
+    image: matrixdotorg/dendrite:syncapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  room_server:
+    hostname: room_server
+    image: matrixdotorg/dendrite:roomserver
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  edu_server:
+    hostname: edu_server
+    image: matrixdotorg/dendrite:eduserver
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  federation_api_proxy:
+    hostname: federation_api_proxy
+    image: matrixdotorg/dendrite:federationproxy
+    command: [
+      "--bind-address=:8448",
+      "--federation-api-url=http://federation_api:7772",
+      "--media-api-server-url=http://media_api:7774"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    depends_on:
+      - federation_api
+      - federation_sender
+      - media_api
+    networks:
+      - internal
+    ports:
+      - "8448:8448"
+
+  federation_api:
+    hostname: federation_api
+    image: matrixdotorg/dendrite:federationapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  federation_sender:
+    hostname: federation_sender
+    image: matrixdotorg/dendrite:federationsender
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+      
+  key_server:
+    hostname: key_server
+    image: matrixdotorg/dendrite:keyserver
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+        - internal
+
+  server_key_api:
+    hostname: server_key_api
+    image: matrixdotorg/dendrite:serverkeyapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  user_api:
+    hostname: user_api
+    image: matrixdotorg/dendrite:userapi
+    command: [
+        "--config=dendrite.yaml"
+    ]
+    volumes:
+        - ./config:/etc/dendrite
+    networks:
+        - internal
+
+  appservice_api:
+    hostname: appservice_api
+    image: matrixdotorg/dendrite:appservice
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+    depends_on:
+      - room_server
+      - user_api
+
+networks:
+  internal:
+    attachable: true

build/docker/images-build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+cd $(git rev-parse --show-toplevel)
+
+docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest .
+
+docker build -t matrixdotorg/dendrite:monolith          --build-arg component=dendrite-monolith-server          -f build/docker/Dockerfile.component .
+
+docker build -t matrixdotorg/dendrite:appservice        --build-arg component=dendrite-appservice-server        -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:clientapi         --build-arg component=dendrite-client-api-server        -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:clientproxy       --build-arg component=client-api-proxy                  -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:eduserver         --build-arg component=dendrite-edu-server               -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:federationapi     --build-arg component=dendrite-federation-api-server    -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:federationsender  --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:federationproxy   --build-arg component=federation-api-proxy              -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:keyserver         --build-arg component=dendrite-key-server               -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:mediaapi          --build-arg component=dendrite-media-api-server         -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:publicroomsapi    --build-arg component=dendrite-public-rooms-api-server  -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:roomserver        --build-arg component=dendrite-room-server              -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:syncapi           --build-arg component=dendrite-sync-api-server          -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:serverkeyapi      --build-arg component=dendrite-server-key-api-server    -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:userapi           --build-arg component=dendrite-user-api-server          -f build/docker/Dockerfile.component .

build/docker/images-pull.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+docker pull matrixdotorg/dendrite:monolith
+
+docker pull matrixdotorg/dendrite:appservice
+docker pull matrixdotorg/dendrite:clientapi
+docker pull matrixdotorg/dendrite:clientproxy
+docker pull matrixdotorg/dendrite:eduserver
+docker pull matrixdotorg/dendrite:federationapi
+docker pull matrixdotorg/dendrite:federationsender
+docker pull matrixdotorg/dendrite:federationproxy
+docker pull matrixdotorg/dendrite:keyserver
+docker pull matrixdotorg/dendrite:mediaapi
+docker pull matrixdotorg/dendrite:publicroomsapi
+docker pull matrixdotorg/dendrite:roomserver
+docker pull matrixdotorg/dendrite:syncapi
+docker pull matrixdotorg/dendrite:userapi

build/docker/images-push.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+docker push matrixdotorg/dendrite:monolith
+
+docker push matrixdotorg/dendrite:appservice
+docker push matrixdotorg/dendrite:clientapi
+docker push matrixdotorg/dendrite:clientproxy
+docker push matrixdotorg/dendrite:eduserver
+docker push matrixdotorg/dendrite:federationapi
+docker push matrixdotorg/dendrite:federationsender
+docker push matrixdotorg/dendrite:federationproxy
+docker push matrixdotorg/dendrite:keyserver
+docker push matrixdotorg/dendrite:mediaapi
+docker push matrixdotorg/dendrite:publicroomsapi
+docker push matrixdotorg/dendrite:roomserver
+docker push matrixdotorg/dendrite:syncapi
+docker push matrixdotorg/dendrite:serverkeyapi
+docker push matrixdotorg/dendrite:userapi

build/gobind/build.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+gomobile bind -v \
+    -ldflags "-X $github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=riot-ios-p2p" \
+    -target ios \
+    github.com/matrix-org/dendrite/build/gobind

build/gobind/monolith.go

@@ -0,0 +1,161 @@
+package gobind
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/publicroomsapi/storage"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+type DendriteMonolith struct {
+	StorageDirectory string
+	listener         net.Listener
+}
+
+func (m *DendriteMonolith) BaseURL() string {
+	return fmt.Sprintf("http://%s", m.listener.Addr().String())
+}
+
+func (m *DendriteMonolith) Start() {
+	logger := logrus.Logger{
+		Out: BindLogger{},
+	}
+	logrus.SetOutput(BindLogger{})
+
+	var err error
+	m.listener, err = net.Listen("tcp", "localhost:65432")
+	if err != nil {
+		panic(err)
+	}
+
+	ygg, err := yggconn.Setup("dendrite", "", m.StorageDirectory)
+	if err != nil {
+		panic(err)
+	}
+
+	cfg := &config.Dendrite{}
+	cfg.SetDefaults()
+	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
+	cfg.Matrix.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Kafka.UseNaffka = true
+	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput"
+	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory))
+	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory))
+	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory))
+	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory))
+	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory))
+	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory))
+	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory))
+	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory))
+	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory))
+	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory))
+	if err = cfg.Derive(); err != nil {
+		panic(err)
+	}
+
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	deviceDB := base.CreateDeviceDB()
+	federation := ygg.CreateFederationClient(base)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices)
+
+	rsAPI := roomserver.NewInternalAPI(
+		base, keyRing, federation,
+	)
+
+	eduInputAPI := eduserver.NewInternalAPI(
+		base, cache.New(), userAPI,
+	)
+
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing,
+	)
+
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this dependency
+	rsAPI.SetFederationSenderAPI(fsAPI)
+
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to public rooms db")
+	}
+
+	monolith := setup.Monolith{
+		Config:        base.Cfg,
+		AccountDB:     accountDB,
+		DeviceDB:      deviceDB,
+		Client:        ygg.CreateClient(base),
+		FedClient:     federation,
+		KeyRing:       keyRing,
+		KafkaConsumer: base.KafkaConsumer,
+		KafkaProducer: base.KafkaProducer,
+
+		AppserviceAPI:       asAPI,
+		EDUInternalAPI:      eduInputAPI,
+		FederationSenderAPI: fsAPI,
+		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
+		//ServerKeyAPI:        serverKeyAPI,
+
+		PublicRoomsDB: publicRoomsDB,
+	}
+	monolith.AddAllPublicRoutes(base.PublicAPIMux)
+
+	httputil.SetupHTTPAPI(
+		base.BaseMux,
+		base.PublicAPIMux,
+		base.InternalAPIMux,
+		cfg,
+		base.UseHTTPAPIs,
+	)
+
+	// Build both ends of a HTTP multiplex.
+	httpServer := &http.Server{
+		Addr:         ":0",
+		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+		ReadTimeout:  15 * time.Second,
+		WriteTimeout: 45 * time.Second,
+		IdleTimeout:  60 * time.Second,
+		BaseContext: func(_ net.Listener) context.Context {
+			return context.Background()
+		},
+		Handler: base.BaseMux,
+	}
+
+	go func() {
+		logger.Info("Listening on ", ygg.DerivedServerName())
+		logger.Fatal(httpServer.Serve(ygg))
+	}()
+	go func() {
+		logger.Info("Listening on ", m.BaseURL())
+		logger.Fatal(httpServer.Serve(m.listener))
+	}()
+}

build/gobind/platform_ios.go

@@ -0,0 +1,25 @@
+// +build ios
+
+package gobind
+
+/*
+#cgo CFLAGS: -x objective-c
+#cgo LDFLAGS: -framework Foundation
+#import <Foundation/Foundation.h>
+void Log(const char *text) {
+  NSString *nss = [NSString stringWithUTF8String:text];
+  NSLog(@"%@", nss);
+}
+*/
+import "C"
+import "unsafe"
+
+type BindLogger struct {
+}
+
+func (nsl BindLogger) Write(p []byte) (n int, err error) {
+	p = append(p, 0)
+	cstr := (*C.char)(unsafe.Pointer(&p[0]))
+	C.Log(cstr)
+	return len(p), nil
+}

build/gobind/platform_other.go

@@ -0,0 +1,12 @@
+// +build !ios
+
+package gobind
+
+import "log"
+
+type BindLogger struct{}
+
+func (nsl BindLogger) Write(p []byte) (n int, err error) {
+	log.Println(string(p))
+	return len(p), nil
+}

build/scripts/build-test-lint.sh

@@ -10,7 +10,7 @@ set -eu
 echo "Checking that it builds..."
 go build ./cmd/...
 
-./scripts/find-lint.sh
+./build/scripts/find-lint.sh
 
 echo "Testing..."
-go test ./...
+go test -v ./...

build/scripts/find-lint.sh

@@ -14,7 +14,7 @@
 
 set -eux
 
-cd `dirname $0`/..
+cd `dirname $0`/../..
 
 args=""
 if [ ${1:-""} = "fast" ]

clientapi/auth/auth.go

@@ -18,17 +18,13 @@ package auth
 import (
 	"context"
 	"crypto/rand"
-	"database/sql"
 	"encoding/base64"
 	"fmt"
 	"net/http"
 	"strings"
 
-	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/util"
 )
 
@@ -39,21 +35,13 @@ var tokenByteLength = 32
 // DeviceDatabase represents a device database.
 type DeviceDatabase interface {
 	// Look up the device matching the given access token.
-	GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error)
+	GetDeviceByAccessToken(ctx context.Context, token string) (*api.Device, error)
 }
 
 // AccountDatabase represents an account database.
 type AccountDatabase interface {
 	// Look up the account matching the given localpart.
-	GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error)
+	GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error)
-}
-
-// Data contains information required to authenticate a request.
-type Data struct {
-	AccountDB AccountDatabase
-	DeviceDB  DeviceDatabase
-	// AppServices is the list of all registered AS
-	AppServices []config.ApplicationService
 }
 
 // VerifyUserFromRequest authenticates the HTTP request,
@@ -62,8 +50,8 @@ type Data struct {
 // Note: For an AS user, AS dummy device is returned.
 // On failure returns an JSON error response which can be sent to the client.
 func VerifyUserFromRequest(
-	req *http.Request, data Data,
+	req *http.Request, userAPI api.UserInternalAPI,
-) (*authtypes.Device, *util.JSONResponse) {
+) (*api.Device, *util.JSONResponse) {
 	// Try to find the Application Service user
 	token, err := ExtractAccessToken(req)
 	if err != nil {
@@ -72,105 +60,31 @@ func VerifyUserFromRequest(
 			JSON: jsonerror.MissingToken(err.Error()),
 		}
 	}
-
+	var res api.QueryAccessTokenResponse
-	// Search for app service with given access_token
+	err = userAPI.QueryAccessToken(req.Context(), &api.QueryAccessTokenRequest{
-	var appService *config.ApplicationService
+		AccessToken:      token,
-	for _, as := range data.AppServices {
+		AppServiceUserID: req.URL.Query().Get("user_id"),
-		if as.ASToken == token {
+	}, &res)
-			appService = &as
+	if err != nil {
-			break
+		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccessToken failed")
-		}
+		jsonErr := jsonerror.InternalServerError()
+		return nil, &jsonErr
 	}
-
+	if res.Err != nil {
-	if appService != nil {
+		if forbidden, ok := res.Err.(*api.ErrorForbidden); ok {
-		// Create a dummy device for AS user
-		dev := authtypes.Device{
-			// Use AS dummy device ID
-			ID: types.AppServiceDeviceID,
-			// AS dummy device has AS's token.
-			AccessToken: token,
-		}
-
-		userID := req.URL.Query().Get("user_id")
-		localpart, err := userutil.ParseUsernameParam(userID, nil)
-		if err != nil {
-			return nil, &util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidUsername(err.Error()),
-			}
-		}
-
-		if localpart != "" { // AS is masquerading as another user
-			// Verify that the user is registered
-			account, err := data.AccountDB.GetAccountByLocalpart(req.Context(), localpart)
-			// Verify that account exists & appServiceID matches
-			if err == nil && account.AppServiceID == appService.ID {
-				// Set the userID of dummy device
-				dev.UserID = userID
-				return &dev, nil
-			}
-
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("Application service has not registered this user"),
+				JSON: jsonerror.Forbidden(forbidden.Message),
 			}
 		}
-
-		// AS is not masquerading as any user, so use AS's sender_localpart
-		dev.UserID = appService.SenderLocalpart
-		return &dev, nil
 	}
-
+	if res.Device == nil {
-	// Try to find local user from device database
+		return nil, &util.JSONResponse{
-	dev, devErr := verifyAccessToken(req, data.DeviceDB)
-	if devErr == nil {
-		return dev, verifyUserParameters(req)
-	}
-
-	return nil, &util.JSONResponse{
-		Code: http.StatusUnauthorized,
-		JSON: jsonerror.UnknownToken("Unrecognized access token"), // nolint: misspell
-	}
-}
-
-// verifyUserParameters ensures that a request coming from a regular user is not
-// using any query parameters reserved for an application service
-func verifyUserParameters(req *http.Request) *util.JSONResponse {
-	if req.URL.Query().Get("ts") != "" {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown("parameter 'ts' not allowed without valid parameter 'access_token'"),
-		}
-	}
-	return nil
-}
-
-// verifyAccessToken verifies that an access token was supplied in the given HTTP request
-// and returns the device it corresponds to. Returns resErr (an error response which can be
-// sent to the client) if the token is invalid or there was a problem querying the database.
-func verifyAccessToken(req *http.Request, deviceDB DeviceDatabase) (device *authtypes.Device, resErr *util.JSONResponse) {
-	token, err := ExtractAccessToken(req)
-	if err != nil {
-		resErr = &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.MissingToken(err.Error()),
+			JSON: jsonerror.UnknownToken("Unknown token"),
-		}
-		return
-	}
-	device, err = deviceDB.GetDeviceByAccessToken(req.Context(), token)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			resErr = &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: jsonerror.UnknownToken("Unknown token"),
-			}
-		} else {
-			util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDeviceByAccessToken failed")
-			jsonErr := jsonerror.InternalServerError()
-			resErr = &jsonErr
 		}
 	}
-	return
+	return res.Device, nil
 }
 
 // GenerateAccessToken creates a new access token. Returns an error if failed to generate

clientapi/clientapi.go

@@ -15,60 +15,55 @@
 package clientapi
 
 import (
+	"github.com/Shopify/sarama"
+	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/consumers"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/routing"
-	"github.com/matrix-org/dendrite/common/basecomponent"
-	"github.com/matrix-org/dendrite/common/transactions"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
 
-// SetupClientAPIComponent sets up and registers HTTP handlers for the ClientAPI
+// AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
-// component.
+func AddPublicRoutes(
-func SetupClientAPIComponent(
+	router *mux.Router,
-	base *basecomponent.BaseDendrite,
+	cfg *config.Dendrite,
+	consumer sarama.Consumer,
+	producer sarama.SyncProducer,
 	deviceDB devices.Database,
 	accountsDB accounts.Database,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing *gomatrixserverlib.KeyRing,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
-	inputAPI roomserverAPI.RoomserverInputAPI,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
 	eduInputAPI eduServerAPI.EDUServerInputAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	transactionsCache *transactions.Cache,
-	fedSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	fsAPI federationSenderAPI.FederationSenderInternalAPI,
+	userAPI userapi.UserInternalAPI,
 ) {
-	roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI)
-	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-
-	userUpdateProducer := &producers.UserUpdateProducer{
-		Producer: base.KafkaProducer,
-		Topic:    string(base.Cfg.Kafka.Topics.UserUpdates),
-	}
-
 	syncProducer := &producers.SyncAPIProducer{
-		Producer: base.KafkaProducer,
+		Producer: producer,
-		Topic:    string(base.Cfg.Kafka.Topics.OutputClientData),
+		Topic:    string(cfg.Kafka.Topics.OutputClientData),
 	}
 
-	consumer := consumers.NewOutputRoomEventConsumer(
+	roomEventConsumer := consumers.NewOutputRoomEventConsumer(
-		base.Cfg, base.KafkaConsumer, accountsDB, queryAPI,
+		cfg, consumer, accountsDB, rsAPI,
 	)
-	if err := consumer.Start(); err != nil {
+	if err := roomEventConsumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start room server consumer")
 	}
 
 	routing.Setup(
-		base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI,
+		router, cfg, eduInputAPI, rsAPI, asAPI,
-		accountsDB, deviceDB, federation, *keyRing, userUpdateProducer,
+		accountsDB, deviceDB, userAPI, federation,
-		syncProducer, eduProducer, transactionsCache, fedSenderAPI,
+		syncProducer, transactionsCache, fsAPI,
 	)
 }

clientapi/consumers/roomserver.go

@@ -18,22 +18,22 @@ import (
 	"context"
 	"encoding/json"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/Shopify/sarama"
 	log "github.com/sirupsen/logrus"
-	sarama "gopkg.in/Shopify/sarama.v1"
 )
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	roomServerConsumer *common.ContinualConsumer
+	rsAPI      api.RoomserverInternalAPI
-	db                 accounts.Database
+	rsConsumer *internal.ContinualConsumer
-	query              api.RoomserverQueryAPI
+	db         accounts.Database
-	serverName         string
+	serverName string
 }
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
@@ -41,19 +41,19 @@ func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
 	store accounts.Database,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) *OutputRoomEventConsumer {
 
-	consumer := common.ContinualConsumer{
+	consumer := internal.ContinualConsumer{
 		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
 		Consumer:       kafkaConsumer,
 		PartitionStore: store,
 	}
 	s := &OutputRoomEventConsumer{
-		roomServerConsumer: &consumer,
+		rsConsumer: &consumer,
-		db:                 store,
+		db:         store,
-		query:              queryAPI,
+		rsAPI:      rsAPI,
-		serverName:         string(cfg.Matrix.ServerName),
+		serverName: string(cfg.Matrix.ServerName),
 	}
 	consumer.ProcessMessage = s.onMessage
 
@@ -62,7 +62,7 @@ func NewOutputRoomEventConsumer(
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	return s.roomServerConsumer.Start()
+	return s.rsConsumer.Start()
 }
 
 // onMessage is called when the sync server receives a new event from the room server output log.
@@ -84,63 +84,9 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 		return nil
 	}
 
-	ev := output.NewRoomEvent.Event
+	return s.db.UpdateMemberships(
-	log.WithFields(log.Fields{
+		context.TODO(),
-		"event_id": ev.EventID(),
+		gomatrixserverlib.UnwrapEventHeaders(output.NewRoomEvent.AddsState()),
-		"room_id":  ev.RoomID(),
+		output.NewRoomEvent.RemovesStateEventIDs,
-		"type":     ev.Type(),
+	)
-	}).Info("received event from roomserver")
-
-	events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event)
-	if err != nil {
-		return err
-	}
-
-	return s.db.UpdateMemberships(context.TODO(), events, output.NewRoomEvent.RemovesStateEventIDs)
-}
-
-// lookupStateEvents looks up the state events that are added by a new event.
-func (s *OutputRoomEventConsumer) lookupStateEvents(
-	addsStateEventIDs []string, event gomatrixserverlib.Event,
-) ([]gomatrixserverlib.Event, error) {
-	// Fast path if there aren't any new state events.
-	if len(addsStateEventIDs) == 0 {
-		// If the event is a membership update (e.g. for a profile update), it won't
-		// show up in AddsStateEventIDs, so we need to add it manually
-		if event.Type() == "m.room.member" {
-			return []gomatrixserverlib.Event{event}, nil
-		}
-		return nil, nil
-	}
-
-	// Fast path if the only state event added is the event itself.
-	if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() {
-		return []gomatrixserverlib.Event{event}, nil
-	}
-
-	result := []gomatrixserverlib.Event{}
-	missing := []string{}
-	for _, id := range addsStateEventIDs {
-		// Append the current event in the results if its ID is in the events list
-		if id == event.EventID() {
-			result = append(result, event)
-		} else {
-			// If the event isn't the current one, add it to the list of events
-			// to retrieve from the roomserver
-			missing = append(missing, id)
-		}
-	}
-
-	// Request the missing events from the roomserver
-	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
-	var eventResp api.QueryEventsByIDResponse
-	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
-		return nil, err
-	}
-
-	for _, headeredEvent := range eventResp.Events {
-		result = append(result, headeredEvent.Event)
-	}
-
-	return result, nil
 }

clientapi/jsonerror/jsonerror.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
@@ -124,6 +125,12 @@ func GuestAccessForbidden(msg string) *MatrixError {
 	return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg}
 }
 
+// IncompatibleRoomVersion is an error which is returned when the client
+// requests a room with a version that is unsupported.
+func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *MatrixError {
+	return &MatrixError{"M_INCOMPATIBLE_ROOM_VERSION", string(roomVersion)}
+}
+
 // UnsupportedRoomVersion is an error which is returned when the client
 // requests a room with a version that is unsupported.
 func UnsupportedRoomVersion(msg string) *MatrixError {

clientapi/producers/eduserver.go

@@ -1,54 +0,0 @@
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package producers
-
-import (
-	"context"
-	"time"
-
-	"github.com/matrix-org/dendrite/eduserver/api"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// EDUServerProducer produces events for the EDU server to consume
-type EDUServerProducer struct {
-	InputAPI api.EDUServerInputAPI
-}
-
-// NewEDUServerProducer creates a new EDUServerProducer
-func NewEDUServerProducer(inputAPI api.EDUServerInputAPI) *EDUServerProducer {
-	return &EDUServerProducer{
-		InputAPI: inputAPI,
-	}
-}
-
-// SendTyping sends a typing event to EDU server
-func (p *EDUServerProducer) SendTyping(
-	ctx context.Context, userID, roomID string,
-	typing bool, timeoutMS int64,
-) error {
-	requestData := api.InputTypingEvent{
-		UserID:         userID,
-		RoomID:         roomID,
-		Typing:         typing,
-		TimeoutMS:      timeoutMS,
-		OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
-	}
-
-	var response api.InputTypingEventResponse
-	err := p.InputAPI.InputTypingEvent(
-		ctx, &api.InputTypingEventRequest{InputTypingEvent: requestData}, &response,
-	)
-
-	return err
-}

clientapi/producers/roomserver.go

@@ -1,119 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package producers
-
-import (
-	"context"
-
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// RoomserverProducer produces events for the roomserver to consume.
-type RoomserverProducer struct {
-	InputAPI api.RoomserverInputAPI
-	QueryAPI api.RoomserverQueryAPI
-}
-
-// NewRoomserverProducer creates a new RoomserverProducer
-func NewRoomserverProducer(inputAPI api.RoomserverInputAPI, queryAPI api.RoomserverQueryAPI) *RoomserverProducer {
-	return &RoomserverProducer{
-		InputAPI: inputAPI,
-		QueryAPI: queryAPI,
-	}
-}
-
-// SendEvents writes the given events to the roomserver input log. The events are written with KindNew.
-func (c *RoomserverProducer) SendEvents(
-	ctx context.Context, events []gomatrixserverlib.HeaderedEvent, sendAsServer gomatrixserverlib.ServerName,
-	txnID *api.TransactionID,
-) (string, error) {
-	ires := make([]api.InputRoomEvent, len(events))
-	for i, event := range events {
-		ires[i] = api.InputRoomEvent{
-			Kind:          api.KindNew,
-			Event:         event,
-			AuthEventIDs:  event.AuthEventIDs(),
-			SendAsServer:  string(sendAsServer),
-			TransactionID: txnID,
-		}
-	}
-	return c.SendInputRoomEvents(ctx, ires)
-}
-
-// SendEventWithState writes an event with KindNew to the roomserver input log
-// with the state at the event as KindOutlier before it.
-func (c *RoomserverProducer) SendEventWithState(
-	ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent,
-) error {
-	outliers, err := state.Events()
-	if err != nil {
-		return err
-	}
-
-	var ires []api.InputRoomEvent
-	for _, outlier := range outliers {
-		ires = append(ires, api.InputRoomEvent{
-			Kind:         api.KindOutlier,
-			Event:        outlier.Headered(event.RoomVersion),
-			AuthEventIDs: outlier.AuthEventIDs(),
-		})
-	}
-
-	stateEventIDs := make([]string, len(state.StateEvents))
-	for i := range state.StateEvents {
-		stateEventIDs[i] = state.StateEvents[i].EventID()
-	}
-
-	ires = append(ires, api.InputRoomEvent{
-		Kind:          api.KindNew,
-		Event:         event,
-		AuthEventIDs:  event.AuthEventIDs(),
-		HasState:      true,
-		StateEventIDs: stateEventIDs,
-	})
-
-	_, err = c.SendInputRoomEvents(ctx, ires)
-	return err
-}
-
-// SendInputRoomEvents writes the given input room events to the roomserver input API.
-func (c *RoomserverProducer) SendInputRoomEvents(
-	ctx context.Context, ires []api.InputRoomEvent,
-) (eventID string, err error) {
-	request := api.InputRoomEventsRequest{InputRoomEvents: ires}
-	var response api.InputRoomEventsResponse
-	err = c.InputAPI.InputRoomEvents(ctx, &request, &response)
-	eventID = response.EventID
-	return
-}
-
-// SendInvite writes the invite event to the roomserver input API.
-// This should only be needed for invite events that occur outside of a known room.
-// If we are in the room then the event should be sent using the SendEvents method.
-func (c *RoomserverProducer) SendInvite(
-	ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent,
-	inviteRoomState []gomatrixserverlib.InviteV2StrippedState,
-) error {
-	request := api.InputRoomEventsRequest{
-		InputInviteEvents: []api.InputInviteEvent{{
-			Event:           inviteEvent,
-			InviteRoomState: inviteRoomState,
-			RoomVersion:     inviteEvent.RoomVersion,
-		}},
-	}
-	var response api.InputRoomEventsResponse
-	return c.InputAPI.InputRoomEvents(ctx, &request, &response)
-}

clientapi/producers/syncapi.go

@@ -17,9 +17,9 @@ package producers
 import (
 	"encoding/json"
 
-	"github.com/matrix-org/dendrite/common"
+	"github.com/Shopify/sarama"
-
+	"github.com/matrix-org/dendrite/internal/eventutil"
-	sarama "gopkg.in/Shopify/sarama.v1"
+	log "github.com/sirupsen/logrus"
 )
 
 // SyncAPIProducer produces events for the sync API server to consume
@@ -32,7 +32,7 @@ type SyncAPIProducer struct {
 func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string) error {
 	var m sarama.ProducerMessage
 
-	data := common.AccountData{
+	data := eventutil.AccountData{
 		RoomID: roomID,
 		Type:   dataType,
 	}
@@ -44,6 +44,11 @@ func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string
 	m.Topic = string(p.Topic)
 	m.Key = sarama.StringEncoder(userID)
 	m.Value = sarama.ByteEncoder(value)
+	log.WithFields(log.Fields{
+		"user_id":   userID,
+		"room_id":   roomID,
+		"data_type": dataType,
+	}).Infof("Producing to topic '%s'", p.Topic)
 
 	_, _, err = p.Producer.SendMessage(&m)
 	return err

clientapi/producers/userupdate.go

@@ -1,62 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package producers
-
-import (
-	"encoding/json"
-
-	sarama "gopkg.in/Shopify/sarama.v1"
-)
-
-// UserUpdateProducer produces events related to user updates.
-type UserUpdateProducer struct {
-	Topic    string
-	Producer sarama.SyncProducer
-}
-
-// TODO: Move this struct to `common` so the components that consume the topic
-// can use it when parsing incoming messages
-type profileUpdate struct {
-	Updated  string `json:"updated"`   // Which attribute is updated (can be either `avatar_url` or `displayname`)
-	OldValue string `json:"old_value"` // The attribute's value before the update
-	NewValue string `json:"new_value"` // The attribute's value after the update
-}
-
-// SendUpdate sends an update using kafka to notify the roomserver of the
-// profile update. Returns an error if the update failed to send.
-func (p *UserUpdateProducer) SendUpdate(
-	userID string, updatedAttribute string, oldValue string, newValue string,
-) error {
-	var update profileUpdate
-	var m sarama.ProducerMessage
-
-	m.Topic = string(p.Topic)
-	m.Key = sarama.StringEncoder(userID)
-
-	update = profileUpdate{
-		Updated:  updatedAttribute,
-		OldValue: oldValue,
-		NewValue: newValue,
-	}
-
-	value, err := json.Marshal(update)
-	if err != nil {
-		return err
-	}
-	m.Value = sarama.ByteEncoder(value)
-
-	_, _, err = p.Producer.SendMessage(&m)
-	return err
-}

clientapi/routing/account_data.go

@@ -16,21 +16,20 @@ package routing
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/util"
 )
 
 // GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type}
 func GetAccountData(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
 	userID string, roomID string, dataType string,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -40,15 +39,25 @@ func GetAccountData(
 		}
 	}
 
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	dataReq := api.QueryAccountDataRequest{
-	if err != nil {
+		UserID:   userID,
-		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
+		DataType: dataType,
-		return jsonerror.InternalServerError()
+		RoomID:   roomID,
+	}
+	dataRes := api.QueryAccountDataResponse{}
+	if err := userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed")
+		return util.ErrorResponse(fmt.Errorf("userAPI.QueryAccountData: %w", err))
 	}
 
-	if data, err := accountDB.GetAccountDataByType(
+	var data json.RawMessage
-		req.Context(), localpart, roomID, dataType,
+	var ok bool
-	); err == nil {
+	if roomID != "" {
+		data, ok = dataRes.RoomAccountData[roomID][dataType]
+	} else {
+		data, ok = dataRes.GlobalAccountData[dataType]
+	}
+	if ok {
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: data,
@@ -63,7 +72,7 @@ func GetAccountData(
 
 // SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type}
 func SaveAccountData(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
 	userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -73,12 +82,6 @@ func SaveAccountData(
 		}
 	}
 
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
-	}
-
 	defer req.Body.Close() // nolint: errcheck
 
 	if req.Body == http.NoBody {
@@ -101,13 +104,19 @@ func SaveAccountData(
 		}
 	}
 
-	if err := accountDB.SaveAccountData(
+	dataReq := api.InputAccountDataRequest{
-		req.Context(), localpart, roomID, dataType, string(body),
+		UserID:      userID,
-	); err != nil {
+		DataType:    dataType,
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.SaveAccountData failed")
+		RoomID:      roomID,
-		return jsonerror.InternalServerError()
+		AccountData: json.RawMessage(body),
+	}
+	dataRes := api.InputAccountDataResponse{}
+	if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed")
+		return util.ErrorResponse(err)
 	}
 
+	// TODO: user API should do this since it's account data
 	if err := syncProducer.SendData(userID, roomID, dataType); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
 		return jsonerror.InternalServerError()

clientapi/routing/auth_fallback.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/util"
 )
 

clientapi/routing/capabilities.go

@@ -26,11 +26,11 @@ import (
 // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
 // by building a m.room.member event then sending it to the room server
 func GetCapabilities(
-	req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI,
+	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{}
 	roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{}
-	if err := queryAPI.QueryRoomVersionCapabilities(
+	if err := rsAPI.QueryRoomVersionCapabilities(
 		req.Context(),
 		&roomVersionsQueryReq,
 		&roomVersionsQueryRes,

clientapi/routing/createroom.go

@@ -24,14 +24,14 @@ import (
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/dendrite/userapi/api"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/clientapi/threepid"
-	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	log "github.com/sirupsen/logrus"
@@ -98,7 +98,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 
 	// Validate creation_content fields defined in the spec by marshalling the
 	// creation_content map into bytes and then unmarshalling the bytes into
-	// common.CreateContent.
+	// eventutil.CreateContent.
 
 	creationContentBytes, err := json.Marshal(r.CreationContent)
 	if err != nil {
@@ -135,23 +135,23 @@ type fledglingEvent struct {
 
 // CreateRoom implements /createRoom
 func CreateRoom(
-	req *http.Request, device *authtypes.Device,
+	req *http.Request, device *api.Device,
-	cfg *config.Dendrite, producer *producers.RoomserverProducer,
+	cfg *config.Dendrite,
-	accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	// TODO (#267): Check room ID doesn't clash with an existing one, and we
 	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
 	roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
-	return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI, asAPI)
+	return createRoom(req, device, cfg, roomID, accountDB, rsAPI, asAPI)
 }
 
 // createRoom implements /createRoom
 // nolint: gocyclo
 func createRoom(
-	req *http.Request, device *authtypes.Device,
+	req *http.Request, device *api.Device,
-	cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer,
+	cfg *config.Dendrite, roomID string,
-	accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	logger := util.GetLogger(req.Context())
@@ -212,6 +212,25 @@ func createRoom(
 		return jsonerror.InternalServerError()
 	}
 
+	var roomAlias string
+	if r.RoomAliasName != "" {
+		roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName)
+		// check it's free TODO: This races but is better than nothing
+		hasAliasReq := roomserverAPI.GetRoomIDForAliasRequest{
+			Alias: roomAlias,
+		}
+
+		var aliasResp roomserverAPI.GetRoomIDForAliasResponse
+		err = rsAPI.GetRoomIDForAlias(req.Context(), &hasAliasReq, &aliasResp)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed")
+			return jsonerror.InternalServerError()
+		}
+		if aliasResp.RoomID != "" {
+			return util.MessageResponse(400, "Alias already exists")
+		}
+	}
+
 	membershipContent := gomatrixserverlib.MemberContent{
 		Membership:  gomatrixserverlib.Join,
 		DisplayName: profile.DisplayName,
@@ -243,9 +262,9 @@ func createRoom(
 	//  1- m.room.create
 	//  2- room creator join member
 	//  3- m.room.power_levels
-	//  4- m.room.canonical_alias (opt) TODO
+	//  4- m.room.join_rules
-	//  5- m.room.join_rules
+	//  5- m.room.history_visibility
-	//  6- m.room.history_visibility
+	//  6- m.room.canonical_alias (opt)
 	//  7- m.room.guest_access (opt)
 	//  8- other initial state items
 	//  9- m.room.name (opt)
@@ -260,24 +279,28 @@ func createRoom(
 	eventsToMake := []fledglingEvent{
 		{"m.room.create", "", r.CreationContent},
 		{"m.room.member", userID, membershipContent},
-		{"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)},
+		{"m.room.power_levels", "", eventutil.InitialPowerLevelsContent(userID)},
-		// TODO: m.room.canonical_alias
 		{"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}},
-		{"m.room.history_visibility", "", common.HistoryVisibilityContent{HistoryVisibility: historyVisibility}},
+		{"m.room.history_visibility", "", eventutil.HistoryVisibilityContent{HistoryVisibility: historyVisibility}},
+	}
+	if roomAlias != "" {
+		// TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room.
+		// This means we might fail creating the alias but say the canonical alias is something that doesn't exist.
+		// m.room.aliases is handled when we call roomserver.SetRoomAlias
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", eventutil.CanonicalAlias{Alias: roomAlias}})
 	}
 	if r.GuestCanJoin {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", common.GuestAccessContent{GuestAccess: "can_join"}})
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", eventutil.GuestAccessContent{GuestAccess: "can_join"}})
 	}
 	eventsToMake = append(eventsToMake, r.InitialState...)
 	if r.Name != "" {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", common.NameContent{Name: r.Name}})
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", eventutil.NameContent{Name: r.Name}})
 	}
 	if r.Topic != "" {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", common.TopicContent{Topic: r.Topic}})
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", eventutil.TopicContent{Topic: r.Topic}})
 	}
 	// TODO: invite events
 	// TODO: 3pid invite events
-	// TODO: m.room.aliases
 
 	authEvents := gomatrixserverlib.NewAuthEvents(nil)
 	for i, e := range eventsToMake {
@@ -320,19 +343,16 @@ func createRoom(
 	}
 
 	// send events to the room server
-	_, err = producer.SendEvents(req.Context(), builtEvents, cfg.Matrix.ServerName, nil)
+	_, err = roomserverAPI.SendEvents(req.Context(), rsAPI, builtEvents, cfg.Matrix.ServerName, nil)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed")
+		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
 		return jsonerror.InternalServerError()
 	}
 
 	// TODO(#269): Reserve room alias while we create the room. This stops us
 	// from creating the room but still failing due to the alias having already
 	// been taken.
-	var roomAlias string
+	if roomAlias != "" {
-	if r.RoomAliasName != "" {
-		roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName)
-
 		aliasReq := roomserverAPI.SetRoomAliasRequest{
 			Alias:  roomAlias,
 			RoomID: roomID,
@@ -340,7 +360,7 @@ func createRoom(
 		}
 
 		var aliasResp roomserverAPI.SetRoomAliasResponse
-		err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp)
+		err = rsAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
 			return jsonerror.InternalServerError()
@@ -351,6 +371,50 @@ func createRoom(
 		}
 	}
 
+	// If this is a direct message then we should invite the participants.
+	for _, invitee := range r.Invite {
+		// Build the membership request.
+		body := threepid.MembershipRequest{
+			UserID: invitee,
+		}
+		// Build the invite event.
+		inviteEvent, err := buildMembershipEvent(
+			req.Context(), body, accountDB, device, gomatrixserverlib.Invite,
+			roomID, true, cfg, evTime, rsAPI, asAPI,
+		)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvent failed")
+			continue
+		}
+		// Build some stripped state for the invite.
+		candidates := append(gomatrixserverlib.UnwrapEventHeaders(builtEvents), *inviteEvent)
+		var strippedState []gomatrixserverlib.InviteV2StrippedState
+		for _, event := range candidates {
+			switch event.Type() {
+			// TODO: case gomatrixserverlib.MRoomEncryption:
+			//	fallthrough
+			case gomatrixserverlib.MRoomMember:
+				fallthrough
+			case gomatrixserverlib.MRoomJoinRules:
+				strippedState = append(
+					strippedState,
+					gomatrixserverlib.NewInviteV2StrippedState(&event),
+				)
+			}
+		}
+		// Send the invite event to the roomserver.
+		if perr := roomserverAPI.SendInvite(
+			req.Context(), rsAPI,
+			inviteEvent.Headered(roomVersion),
+			strippedState,         // invite room state
+			cfg.Matrix.ServerName, // send as server
+			nil,                   // transaction ID
+		); perr != nil {
+			util.GetLogger(req.Context()).WithError(perr).Error("SendInvite failed")
+			return perr.JSONResponse()
+		}
+	}
+
 	response := createRoomResponse{
 		RoomID:    roomID,
 		RoomAlias: roomAlias,

clientapi/routing/device.go

@@ -19,16 +19,19 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
+// https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-devices
 type deviceJSON struct {
-	DeviceID string `json:"device_id"`
+	DeviceID    string `json:"device_id"`
-	UserID   string `json:"user_id"`
+	DisplayName string `json:"display_name"`
+	LastSeenIP  string `json:"last_seen_ip"`
+	LastSeenTS  uint64 `json:"last_seen_ts"`
 }
 
 type devicesJSON struct {
@@ -45,7 +48,7 @@ type devicesDeleteJSON struct {
 
 // GetDeviceByID handles /devices/{deviceID}
 func GetDeviceByID(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -70,14 +73,13 @@ func GetDeviceByID(
 		Code: http.StatusOK,
 		JSON: deviceJSON{
 			DeviceID: dev.ID,
-			UserID:   dev.UserID,
 		},
 	}
 }
 
 // GetDevicesByLocalpart handles /devices
 func GetDevicesByLocalpart(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
@@ -98,7 +100,6 @@ func GetDevicesByLocalpart(
 	for _, dev := range deviceList {
 		res.Devices = append(res.Devices, deviceJSON{
 			DeviceID: dev.ID,
-			UserID:   dev.UserID,
 		})
 	}
 
@@ -110,7 +111,7 @@ func GetDevicesByLocalpart(
 
 // UpdateDeviceByID handles PUT on /devices/{deviceID}
 func UpdateDeviceByID(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -160,7 +161,7 @@ func UpdateDeviceByID(
 
 // DeleteDeviceById handles DELETE requests to /devices/{deviceId}
 func DeleteDeviceById(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -185,7 +186,7 @@ func DeleteDeviceById(
 
 // DeleteDevices handles POST requests to /delete_devices
 func DeleteDevices(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {

clientapi/routing/directory.go

@@ -18,12 +18,12 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/common/config"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/internal/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -46,8 +46,8 @@ func DirectoryRoom(
 	roomAlias string,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.Dendrite,
-	rsAPI roomserverAPI.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	fedSenderAPI federationSenderAPI.FederationSenderQueryAPI,
+	fedSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
 	if err != nil {
@@ -77,7 +77,7 @@ func DirectoryRoom(
 			if fedErr != nil {
 				// TODO: Return 502 if the remote server errored.
 				// TODO: Return 504 if the remote server timed out.
-				util.GetLogger(req.Context()).WithError(err).Error("federation.LookupRoomAlias failed")
+				util.GetLogger(req.Context()).WithError(fedErr).Error("federation.LookupRoomAlias failed")
 				return jsonerror.InternalServerError()
 			}
 			res.RoomID = fedRes.RoomID
@@ -112,10 +112,10 @@ func DirectoryRoom(
 // TODO: Check if the user has the power level to set an alias
 func SetLocalAlias(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	alias string,
 	cfg *config.Dendrite,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', alias)
 	if err != nil {
@@ -188,9 +188,9 @@ func SetLocalAlias(
 // RemoveLocalAlias implements DELETE /directory/room/{roomAlias}
 func RemoveLocalAlias(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	alias string,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 
 	creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{

clientapi/routing/filter.go

@@ -17,17 +17,17 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 // GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId}
 func GetFilter(
-	req *http.Request, device *authtypes.Device, accountDB accounts.Database, userID string, filterID string,
+	req *http.Request, device *api.Device, accountDB accounts.Database, userID string, filterID string,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -64,7 +64,7 @@ type filterResponse struct {
 
 //PutFilter implements POST /_matrix/client/r0/user/{userId}/filter
 func PutFilter(
-	req *http.Request, device *authtypes.Device, accountDB accounts.Database, userID string,
+	req *http.Request, device *api.Device, accountDB accounts.Database, userID string,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{

clientapi/routing/getevent.go

@@ -17,22 +17,21 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 type getEventRequest struct {
 	req            *http.Request
-	device         *authtypes.Device
+	device         *userapi.Device
 	roomID         string
 	eventID        string
 	cfg            *config.Dendrite
 	federation     *gomatrixserverlib.FederationClient
-	keyRing        gomatrixserverlib.KeyRing
 	requestedEvent gomatrixserverlib.Event
 }
 
@@ -40,19 +39,18 @@ type getEventRequest struct {
 // https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid
 func GetEvent(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	roomID string,
 	eventID string,
 	cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing gomatrixserverlib.KeyRing,
 ) util.JSONResponse {
 	eventsReq := api.QueryEventsByIDRequest{
 		EventIDs: []string{eventID},
 	}
 	var eventsResp api.QueryEventsByIDResponse
-	err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
+	err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed")
 		return jsonerror.InternalServerError()
@@ -75,7 +73,6 @@ func GetEvent(
 		eventID:        eventID,
 		cfg:            cfg,
 		federation:     federation,
-		keyRing:        keyRing,
 		requestedEvent: requestedEvent,
 	}
 
@@ -88,7 +85,7 @@ func GetEvent(
 		}},
 	}
 	var stateResp api.QueryStateAfterEventsResponse
-	if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
+	if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/joinroom.go

@@ -15,434 +15,64 @@
 package routing
 
 import (
-	"fmt"
 	"net/http"
-	"strings"
-	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/producers"
-	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
-// JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API.
-// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-join-roomidoralias
 func JoinRoomByIDOrAlias(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
-	roomIDOrAlias string,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	cfg *config.Dendrite,
-	federation *gomatrixserverlib.FederationClient,
-	producer *producers.RoomserverProducer,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
-	keyRing gomatrixserverlib.KeyRing,
 	accountDB accounts.Database,
+	roomIDOrAlias string,
 ) util.JSONResponse {
-	var content map[string]interface{} // must be a JSON object
+	// Prepare to ask the roomserver to perform the room join.
-	if resErr := httputil.UnmarshalJSONRequest(req, &content); resErr != nil {
+	joinReq := roomserverAPI.PerformJoinRequest{
-		return *resErr
+		RoomIDOrAlias: roomIDOrAlias,
+		UserID:        device.UserID,
+		Content:       map[string]interface{}{},
 	}
+	joinRes := roomserverAPI.PerformJoinResponse{}
 
-	evTime, err := httputil.ParseTSParam(req)
+	// If content was provided in the request then incude that
-	if err != nil {
+	// in the request. It'll get used as a part of the membership
-		return util.JSONResponse{
+	// event content.
-			Code: http.StatusBadRequest,
+	_ = httputil.UnmarshalJSONRequest(req, &joinReq.Content)
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
-		}
-	}
 
+	// Work out our localpart for the client profile request.
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+	} else {
-	}
+		// Request our profile content to populate the request content with.
-
+		var profile *authtypes.Profile
-	profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
+		profile, err = accountDB.GetProfileByLocalpart(req.Context(), localpart)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed")
-		return jsonerror.InternalServerError()
-	}
-
-	content["membership"] = gomatrixserverlib.Join
-	content["displayname"] = profile.DisplayName
-	content["avatar_url"] = profile.AvatarURL
-
-	r := joinRoomReq{
-		req, evTime, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing,
-	}
-
-	if strings.HasPrefix(roomIDOrAlias, "!") {
-		return r.joinRoomByID(roomIDOrAlias)
-	}
-	if strings.HasPrefix(roomIDOrAlias, "#") {
-		return r.joinRoomByAlias(roomIDOrAlias)
-	}
-	return util.JSONResponse{
-		Code: http.StatusBadRequest,
-		JSON: jsonerror.BadJSON(
-			fmt.Sprintf("Invalid first character '%s' for room ID or alias",
-				string([]rune(roomIDOrAlias)[0])), // Wrapping with []rune makes this call UTF-8 safe
-		),
-	}
-}
-
-type joinRoomReq struct {
-	req        *http.Request
-	evTime     time.Time
-	content    map[string]interface{}
-	userID     string
-	cfg        *config.Dendrite
-	federation *gomatrixserverlib.FederationClient
-	producer   *producers.RoomserverProducer
-	queryAPI   roomserverAPI.RoomserverQueryAPI
-	aliasAPI   roomserverAPI.RoomserverAliasAPI
-	keyRing    gomatrixserverlib.KeyRing
-}
-
-// joinRoomByID joins a room by room ID
-func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse {
-	// A client should only join a room by room ID when it has an invite
-	// to the room. If the server is already in the room then we can
-	// lookup the invite and process the request as a normal state event.
-	// If the server is not in the room the we will need to look up the
-	// remote server the invite came from in order to request a join event
-	// from that server.
-	queryReq := roomserverAPI.QueryInvitesForUserRequest{
-		RoomID: roomID, TargetUserID: r.userID,
-	}
-	var queryRes roomserverAPI.QueryInvitesForUserResponse
-	if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil {
-		util.GetLogger(r.req.Context()).WithError(err).Error("r.queryAPI.QueryInvitesForUser failed")
-		return jsonerror.InternalServerError()
-	}
-
-	servers := []gomatrixserverlib.ServerName{}
-	seenInInviterIDs := map[gomatrixserverlib.ServerName]bool{}
-	for _, userID := range queryRes.InviteSenderUserIDs {
-		_, domain, err := gomatrixserverlib.SplitID('@', userID)
 		if err != nil {
-			util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
+			util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed")
-			return jsonerror.InternalServerError()
+		} else {
-		}
+			joinReq.Content["displayname"] = profile.DisplayName
-		if !seenInInviterIDs[domain] {
+			joinReq.Content["avatar_url"] = profile.AvatarURL
-			servers = append(servers, domain)
-			seenInInviterIDs[domain] = true
 		}
 	}
 
-	// Also add the domain extracted from the roomID as a last resort to join
+	// Ask the roomserver to perform the join.
-	// in case the client is erroneously trying to join by ID without an invite
+	rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes)
-	// or all previous attempts at domains extracted from the inviter IDs fail
+	if joinRes.Error != nil {
-	// Note: It's no guarantee we'll succeed because a room isn't bound to the domain in its ID
+		return joinRes.Error.JSONResponse()
-	_, domain, err := gomatrixserverlib.SplitID('!', roomID)
-	if err != nil {
-		util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
-	}
-	if domain != r.cfg.Matrix.ServerName && !seenInInviterIDs[domain] {
-		servers = append(servers, domain)
 	}
 
-	return r.joinRoomUsingServers(roomID, servers)
+	return util.JSONResponse{
-
-}
-
-// joinRoomByAlias joins a room using a room alias.
-func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse {
-	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
-		}
-	}
-	if domain == r.cfg.Matrix.ServerName {
-		queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
-		var queryRes roomserverAPI.GetRoomIDForAliasResponse
-		if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil {
-			util.GetLogger(r.req.Context()).WithError(err).Error("r.aliasAPI.GetRoomIDForAlias failed")
-			return jsonerror.InternalServerError()
-		}
-
-		if len(queryRes.RoomID) > 0 {
-			return r.joinRoomUsingServers(queryRes.RoomID, []gomatrixserverlib.ServerName{r.cfg.Matrix.ServerName})
-		}
-		// If the response doesn't contain a non-empty string, return an error
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Room alias " + roomAlias + " not found."),
-		}
-	}
-	// If the room isn't local, use federation to join
-	return r.joinRoomByRemoteAlias(domain, roomAlias)
-}
-
-func (r joinRoomReq) joinRoomByRemoteAlias(
-	domain gomatrixserverlib.ServerName, roomAlias string,
-) util.JSONResponse {
-	resp, err := r.federation.LookupRoomAlias(r.req.Context(), domain, roomAlias)
-	if err != nil {
-		switch x := err.(type) {
-		case gomatrix.HTTPError:
-			if x.Code == http.StatusNotFound {
-				return util.JSONResponse{
-					Code: http.StatusNotFound,
-					JSON: jsonerror.NotFound("Room alias not found"),
-				}
-			}
-		}
-		util.GetLogger(r.req.Context()).WithError(err).Error("r.federation.LookupRoomAlias failed")
-		return jsonerror.InternalServerError()
-	}
-
-	return r.joinRoomUsingServers(resp.RoomID, resp.Servers)
-}
-
-func (r joinRoomReq) writeToBuilder(eb *gomatrixserverlib.EventBuilder, roomID string) error {
-	eb.Type = "m.room.member"
-
-	err := eb.SetContent(r.content)
-	if err != nil {
-		return err
-	}
-
-	err = eb.SetUnsigned(struct{}{})
-	if err != nil {
-		return err
-	}
-
-	eb.Sender = r.userID
-	eb.StateKey = &r.userID
-	eb.RoomID = roomID
-	eb.Redacts = ""
-
-	return nil
-}
-
-func (r joinRoomReq) joinRoomUsingServers(
-	roomID string, servers []gomatrixserverlib.ServerName,
-) util.JSONResponse {
-	var eb gomatrixserverlib.EventBuilder
-	err := r.writeToBuilder(&eb, roomID)
-	if err != nil {
-		util.GetLogger(r.req.Context()).WithError(err).Error("r.writeToBuilder failed")
-		return jsonerror.InternalServerError()
-	}
-
-	queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{}
-	event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes)
-	if err == nil {
-		// If we have successfully built an event at this point then we can
-		// assert that the room is a local room, as BuildEvent was able to
-		// add prev_events etc successfully.
-		if _, err = r.producer.SendEvents(
-			r.req.Context(),
-			[]gomatrixserverlib.HeaderedEvent{
-				(*event).Headered(queryRes.RoomVersion),
-			},
-			r.cfg.Matrix.ServerName,
-			nil,
-		); err != nil {
-			util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEvents failed")
-			return jsonerror.InternalServerError()
-		}
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: struct {
-				RoomID string `json:"room_id"`
-			}{roomID},
-		}
-	}
-
-	// Otherwise, if we've reached here, then we haven't been able to populate
-	// prev_events etc for the room, therefore the room is probably federated.
-
-	// TODO: This needs to be re-thought, as in the case of an invite, the room
-	// will exist in the database in roomserver_rooms but won't have any state
-	// events, therefore this below check fails.
-	if err != common.ErrRoomNoExists {
-		util.GetLogger(r.req.Context()).WithError(err).Error("common.BuildEvent failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if len(servers) == 0 {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("No candidate servers found for room"),
-		}
-	}
-
-	var lastErr error
-	for _, server := range servers {
-		var response *util.JSONResponse
-		response, lastErr = r.joinRoomUsingServer(roomID, server)
-		if lastErr != nil {
-			// There was a problem talking to one of the servers.
-			util.GetLogger(r.req.Context()).WithError(lastErr).WithField("server", server).Warn("Failed to join room using server")
-			// Try the next server.
-			if r.req.Context().Err() != nil {
-				// The request context has expired so don't bother trying any
-				// more servers - they will immediately fail due to the expired
-				// context.
-				break
-			} else {
-				// The request context hasn't expired yet so try the next server.
-				continue
-			}
-		}
-		return *response
-	}
-
-	// Every server we tried to join through resulted in an error.
-	// We return the error from the last server.
-
-	// TODO: Generate the correct HTTP status code for all different
-	// kinds of errors that could have happened.
-	// The possible errors include:
-	//   1) We can't connect to the remote servers.
-	//   2) None of the servers we could connect to think we are allowed
-	//	    to join the room.
-	//   3) The remote server returned something invalid.
-	//   4) We couldn't fetch the public keys needed to verify the
-	//      signatures on the state events.
-	//   5) ...
-	util.GetLogger(r.req.Context()).WithError(lastErr).Error("failed to join through any server")
-	return jsonerror.InternalServerError()
-}
-
-// joinRoomUsingServer tries to join a remote room using a given matrix server.
-// If there was a failure communicating with the server or the response from the
-// server was invalid this returns an error.
-// Otherwise this returns a JSONResponse.
-func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) {
-	// Ask the room server for information about room versions.
-	var request api.QueryRoomVersionCapabilitiesRequest
-	var response api.QueryRoomVersionCapabilitiesResponse
-	if err := r.queryAPI.QueryRoomVersionCapabilities(r.req.Context(), &request, &response); err != nil {
-		return nil, err
-	}
-	var supportedVersions []gomatrixserverlib.RoomVersion
-	for version := range response.AvailableRoomVersions {
-		supportedVersions = append(supportedVersions, version)
-	}
-	respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, supportedVersions)
-	if err != nil {
-		// TODO: Check if the user was not allowed to join the room.
-		return nil, fmt.Errorf("r.federation.MakeJoin: %w", err)
-	}
-
-	// Set all the fields to be what they should be, this should be a no-op
-	// but it's possible that the remote server returned us something "odd"
-	err = r.writeToBuilder(&respMakeJoin.JoinEvent, roomID)
-	if err != nil {
-		return nil, fmt.Errorf("r.writeToBuilder: %w", err)
-	}
-
-	if respMakeJoin.RoomVersion == "" {
-		respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1
-	}
-	if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion(
-				fmt.Sprintf("Room version '%s' is not supported", respMakeJoin.RoomVersion),
-			),
-		}, nil
-	}
-
-	event, err := respMakeJoin.JoinEvent.Build(
-		r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID,
-		r.cfg.Matrix.PrivateKey, respMakeJoin.RoomVersion,
-	)
-	if err != nil {
-		return nil, fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err)
-	}
-
-	respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event, respMakeJoin.RoomVersion)
-	if err != nil {
-		return nil, fmt.Errorf("r.federation.SendJoin: %w", err)
-	}
-
-	if err = r.checkSendJoinResponse(event, server, respMakeJoin, respSendJoin); err != nil {
-		return nil, err
-	}
-
-	util.GetLogger(r.req.Context()).WithFields(logrus.Fields{
-		"room_id":          roomID,
-		"num_auth_events":  len(respSendJoin.AuthEvents),
-		"num_state_events": len(respSendJoin.StateEvents),
-	}).Info("Room join signature and auth verification passed")
-
-	if err = r.producer.SendEventWithState(
-		r.req.Context(),
-		gomatrixserverlib.RespState(respSendJoin.RespState),
-		event.Headered(respMakeJoin.RoomVersion),
-	); err != nil {
-		util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState")
-	}
-
-	return &util.JSONResponse{
 		Code: http.StatusOK,
-		// TODO: Put the response struct somewhere common.
+		// TODO: Put the response struct somewhere internal.
 		JSON: struct {
 			RoomID string `json:"room_id"`
-		}{roomID},
+		}{joinRes.RoomID},
-	}, nil
-}
-
-// checkSendJoinResponse checks that all of the signatures are correct
-// and that the join is allowed by the supplied state.
-func (r joinRoomReq) checkSendJoinResponse(
-	event gomatrixserverlib.Event,
-	server gomatrixserverlib.ServerName,
-	respMakeJoin gomatrixserverlib.RespMakeJoin,
-	respSendJoin gomatrixserverlib.RespSendJoin,
-) error {
-	// A list of events that we have retried, if they were not included in
-	// the auth events supplied in the send_join.
-	retries := map[string]bool{}
-
-retryCheck:
-	// TODO: Can we expand Check here to return a list of missing auth
-	// events rather than failing one at a time?
-	if err := respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil {
-		switch e := err.(type) {
-		case gomatrixserverlib.MissingAuthEventError:
-			// Check that we haven't already retried for this event, prevents
-			// us from ending up in endless loops
-			if !retries[e.AuthEventID] {
-				// Ask the server that we're talking to right now for the event
-				tx, txerr := r.federation.GetEvent(r.req.Context(), server, e.AuthEventID)
-				if txerr != nil {
-					return fmt.Errorf("r.federation.GetEvent: %w", txerr)
-				}
-				// For each event returned, add it to the auth events.
-				for _, pdu := range tx.PDUs {
-					ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion)
-					if everr != nil {
-						return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr)
-					}
-					respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev)
-				}
-				// Mark the event as retried and then give the check another go.
-				retries[e.AuthEventID] = true
-				goto retryCheck
-			}
-			return fmt.Errorf("respSendJoin (after retries): %w", e)
-		default:
-			return fmt.Errorf("respSendJoin: %w", err)
-		}
 	}
-	return nil
 }

clientapi/routing/leaveroom.go

@@ -0,0 +1,51 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+)
+
+func LeaveRoomByID(
+	req *http.Request,
+	device *api.Device,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
+	roomID string,
+) util.JSONResponse {
+	// Prepare to ask the roomserver to perform the room join.
+	leaveReq := roomserverAPI.PerformLeaveRequest{
+		RoomID: roomID,
+		UserID: device.UserID,
+	}
+	leaveRes := roomserverAPI.PerformLeaveResponse{}
+
+	// Ask the roomserver to perform the leave.
+	if err := rsAPI.PerformLeave(req.Context(), &leaveReq, &leaveRes); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown(err.Error()),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/login.go

@@ -20,13 +20,13 @@ import (
 	"context"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -47,6 +47,7 @@ type loginIdentifier struct {
 
 type passwordRequest struct {
 	Identifier loginIdentifier `json:"identifier"`
+	User       string          `json:"user"` // deprecated in favour of identifier
 	Password   string          `json:"password"`
 	// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
 	// Thus a pointer is needed to differentiate between the two
@@ -80,7 +81,8 @@ func Login(
 		}
 	} else if req.Method == http.MethodPost {
 		var r passwordRequest
-		var acc *authtypes.Account
+		var acc *api.Account
+		var errJSON *util.JSONResponse
 		resErr := httputil.UnmarshalJSONRequest(req, &r)
 		if resErr != nil {
 			return *resErr
@@ -93,30 +95,22 @@ func Login(
 					JSON: jsonerror.BadJSON("'user' must be supplied."),
 				}
 			}
-
+			acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.Identifier.User)
-			util.GetLogger(req.Context()).WithField("user", r.Identifier.User).Info("Processing login request")
+			if errJSON != nil {
-
+				return *errJSON
-			localpart, err := userutil.ParseUsernameParam(r.Identifier.User, &cfg.Matrix.ServerName)
-			if err != nil {
-				return util.JSONResponse{
-					Code: http.StatusBadRequest,
-					JSON: jsonerror.InvalidUsername(err.Error()),
-				}
-			}
-
-			acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password)
-			if err != nil {
-				// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
-				// but that would leak the existence of the user.
-				return util.JSONResponse{
-					Code: http.StatusForbidden,
-					JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"),
-				}
 			}
 		default:
-			return util.JSONResponse{
+			// TODO: The below behaviour is deprecated but without it Riot iOS won't log in
-				Code: http.StatusBadRequest,
+			if r.User != "" {
-				JSON: jsonerror.BadJSON("login identifier '" + r.Identifier.Type + "' not supported"),
+				acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.User)
+				if errJSON != nil {
+					return *errJSON
+				}
+			} else {
+				return util.JSONResponse{
+					Code: http.StatusBadRequest,
+					JSON: jsonerror.BadJSON("login identifier '" + r.Identifier.Type + "' not supported"),
+				}
 			}
 		}
 
@@ -155,11 +149,40 @@ func getDevice(
 	ctx context.Context,
 	r passwordRequest,
 	deviceDB devices.Database,
-	acc *authtypes.Account,
+	acc *api.Account,
 	token string,
-) (dev *authtypes.Device, err error) {
+) (dev *api.Device, err error) {
 	dev, err = deviceDB.CreateDevice(
 		ctx, acc.Localpart, r.DeviceID, token, r.InitialDisplayName,
 	)
 	return
 }
+
+func (r *passwordRequest) processUsernamePasswordLoginRequest(
+	req *http.Request, accountDB accounts.Database,
+	cfg *config.Dendrite, username string,
+) (acc *api.Account, errJSON *util.JSONResponse) {
+	util.GetLogger(req.Context()).WithField("user", username).Info("Processing login request")
+
+	localpart, err := userutil.ParseUsernameParam(username, &cfg.Matrix.ServerName)
+	if err != nil {
+		errJSON = &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidUsername(err.Error()),
+		}
+		return
+	}
+
+	acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password)
+	if err != nil {
+		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
+		// but that would leak the existence of the user.
+		errJSON = &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"),
+		}
+		return
+	}
+
+	return
+}

clientapi/routing/logout.go

@@ -17,16 +17,16 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 // Logout handles POST /logout
 func Logout(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
@@ -47,7 +47,7 @@ func Logout(
 
 // LogoutAll handles POST /logout/all
 func LogoutAll(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {

clientapi/routing/membership.go

@@ -22,15 +22,15 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
-	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/util"
@@ -40,15 +40,16 @@ var errMissingUserID = errors.New("'user_id' must be supplied")
 
 // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
 // by building a m.room.member event then sending it to the room server
+// TODO: Can we improve the cyclo count here? Separate code paths for invites?
+// nolint:gocyclo
 func SendMembership(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
 	roomID string, membership string, cfg *config.Dendrite,
-	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
-	producer *producers.RoomserverProducer,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -69,7 +70,7 @@ func SendMembership(
 	}
 
 	inviteStored, jsonErrResp := checkAndProcessThreepid(
-		req, device, &body, cfg, queryAPI, accountDB, producer,
+		req, device, &body, cfg, rsAPI, accountDB,
 		membership, roomID, evTime,
 	)
 	if jsonErrResp != nil {
@@ -87,14 +88,15 @@ func SendMembership(
 	}
 
 	event, err := buildMembershipEvent(
-		req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, queryAPI, asAPI,
+		req.Context(), body, accountDB, device, membership,
+		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
 	if err == errMissingUserID {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.BadJSON(err.Error()),
 		}
-	} else if err == common.ErrRoomNoExists {
+	} else if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound(err.Error()),
@@ -104,23 +106,39 @@ func SendMembership(
 		return jsonerror.InternalServerError()
 	}
 
-	if _, err := producer.SendEvents(
-		req.Context(),
-		[]gomatrixserverlib.HeaderedEvent{(*event).Headered(verRes.RoomVersion)},
-		cfg.Matrix.ServerName,
-		nil,
-	); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed")
-		return jsonerror.InternalServerError()
-	}
-
 	var returnData interface{} = struct{}{}
 
-	// The join membership requires the room id to be sent in the response
+	switch membership {
-	if membership == gomatrixserverlib.Join {
+	case gomatrixserverlib.Invite:
+		// Invites need to be handled specially
+		perr := roomserverAPI.SendInvite(
+			req.Context(), rsAPI,
+			event.Headered(verRes.RoomVersion),
+			nil, // ask the roomserver to draw up invite room state for us
+			cfg.Matrix.ServerName,
+			nil,
+		)
+		if perr != nil {
+			util.GetLogger(req.Context()).WithError(perr).Error("producer.SendInvite failed")
+			return perr.JSONResponse()
+		}
+	case gomatrixserverlib.Join:
+		// The join membership requires the room id to be sent in the response
 		returnData = struct {
 			RoomID string `json:"room_id"`
 		}{roomID}
+		fallthrough
+	default:
+		_, err = roomserverAPI.SendEvents(
+			req.Context(), rsAPI,
+			[]gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)},
+			cfg.Matrix.ServerName,
+			nil,
+		)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
+			return jsonerror.InternalServerError()
+		}
 	}
 
 	return util.JSONResponse{
@@ -132,10 +150,10 @@ func SendMembership(
 func buildMembershipEvent(
 	ctx context.Context,
 	body threepid.MembershipRequest, accountDB accounts.Database,
-	device *authtypes.Device,
+	device *userapi.Device,
-	membership, roomID string,
+	membership, roomID string, isDirect bool,
 	cfg *config.Dendrite, evTime time.Time,
-	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*gomatrixserverlib.Event, error) {
 	stateKey, reason, err := getMembershipStateKey(body, device, membership)
 	if err != nil {
@@ -164,13 +182,14 @@ func buildMembershipEvent(
 		DisplayName: profile.DisplayName,
 		AvatarURL:   profile.AvatarURL,
 		Reason:      reason,
+		IsDirect:    isDirect,
 	}
 
 	if err = builder.SetContent(content); err != nil {
 		return nil, err
 	}
 
-	return common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil)
+	return eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
 }
 
 // loadProfile lookups the profile of a given user from the database and returns
@@ -205,7 +224,7 @@ func loadProfile(
 // In the latter case, if there was an issue retrieving the user ID from the request body,
 // returns a JSONResponse with a corresponding error code and message.
 func getMembershipStateKey(
-	body threepid.MembershipRequest, device *authtypes.Device, membership string,
+	body threepid.MembershipRequest, device *userapi.Device, membership string,
 ) (stateKey string, reason string, err error) {
 	if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite {
 		// If we're in this case, the state key is contained in the request body,
@@ -227,18 +246,17 @@ func getMembershipStateKey(
 
 func checkAndProcessThreepid(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	body *threepid.MembershipRequest,
 	cfg *config.Dendrite,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 	accountDB accounts.Database,
-	producer *producers.RoomserverProducer,
 	membership, roomID string,
 	evTime time.Time,
 ) (inviteStored bool, errRes *util.JSONResponse) {
 
 	inviteStored, err := threepid.CheckAndProcessInvite(
-		req.Context(), device, body, cfg, queryAPI, accountDB, producer,
+		req.Context(), device, body, cfg, rsAPI, accountDB,
 		membership, roomID, evTime,
 	)
 	if err == threepid.ErrMissingParameter {
@@ -251,12 +269,18 @@ func checkAndProcessThreepid(
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.NotTrusted(body.IDServer),
 		}
-	} else if err == common.ErrRoomNoExists {
+	} else if err == eventutil.ErrRoomNoExists {
 		return inviteStored, &util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound(err.Error()),
 		}
-	} else if err != nil {
+	} else if e, ok := err.(gomatrixserverlib.BadJSONError); ok {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(e.Error()),
+		}
+	}
+	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAndProcessInvite failed")
 		er := jsonerror.InternalServerError()
 		return inviteStored, &er

clientapi/routing/memberships.go

@@ -15,14 +15,15 @@
 package routing
 
 import (
+	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -35,11 +36,21 @@ type getJoinedRoomsResponse struct {
 	JoinedRooms []string `json:"joined_rooms"`
 }
 
+// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
+type getJoinedMembersResponse struct {
+	Joined map[string]joinedMember `json:"joined"`
+}
+
+type joinedMember struct {
+	DisplayName string `json:"display_name"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
 // GetMemberships implements GET /rooms/{roomId}/members
 func GetMemberships(
-	req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool,
+	req *http.Request, device *userapi.Device, roomID string, joinedOnly bool,
 	_ *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	queryReq := api.QueryMembershipsForRoomRequest{
 		JoinedOnly: joinedOnly,
@@ -47,8 +58,8 @@ func GetMemberships(
 		Sender:     device.UserID,
 	}
 	var queryRes api.QueryMembershipsForRoomResponse
-	if err := queryAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil {
+	if err := rsAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryMembershipsForRoom failed")
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -59,6 +70,22 @@ func GetMemberships(
 		}
 	}
 
+	if joinedOnly {
+		var res getJoinedMembersResponse
+		res.Joined = make(map[string]joinedMember)
+		for _, ev := range queryRes.JoinEvents {
+			var content joinedMember
+			if err := json.Unmarshal(ev.Content, &content); err != nil {
+				util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
+				return jsonerror.InternalServerError()
+			}
+			res.Joined[ev.Sender] = content
+		}
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: res,
+		}
+	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: getMembershipResponse{queryRes.JoinEvents},
@@ -67,7 +94,7 @@ func GetMemberships(
 
 func GetJoinedRooms(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	accountsDB accounts.Database,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)

clientapi/routing/profile.go

@@ -21,13 +21,13 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/eventutil"
-	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/gomatrix"
@@ -43,7 +43,7 @@ func GetProfile(
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
 	if err != nil {
-		if err == common.ErrProfileNoExists {
+		if err == eventutil.ErrProfileNoExists {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
 				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
@@ -56,7 +56,7 @@ func GetProfile(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: common.ProfileResponse{
+		JSON: eventutil.ProfileResponse{
 			AvatarURL:   profile.AvatarURL,
 			DisplayName: profile.DisplayName,
 		},
@@ -71,7 +71,7 @@ func GetAvatarURL(
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
 	if err != nil {
-		if err == common.ErrProfileNoExists {
+		if err == eventutil.ErrProfileNoExists {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
 				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
@@ -84,17 +84,17 @@ func GetAvatarURL(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: common.AvatarURL{
+		JSON: eventutil.AvatarURL{
 			AvatarURL: profile.AvatarURL,
 		},
 	}
 }
 
 // SetAvatarURL implements PUT /profile/{userID}/avatar_url
+// nolint:gocyclo
 func SetAvatarURL(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
-	userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite,
+	userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
-	rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -103,9 +103,7 @@ func SetAvatarURL(
 		}
 	}
 
-	changedKey := "avatar_url"
+	var r eventutil.AvatarURL
-
-	var r common.AvatarURL
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
@@ -154,20 +152,22 @@ func SetAvatarURL(
 	}
 
 	events, err := buildMembershipEvents(
-		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+		req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
 	)
-	if err != nil {
+	switch e := err.(type) {
+	case nil:
+	case gomatrixserverlib.BadJSONError:
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(e.Error()),
+		}
+	default:
 		util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed")
 		return jsonerror.InternalServerError()
 	}
 
-	if _, err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil {
+	if _, err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("rsProducer.SendEvents failed")
+		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if err := producer.SendUpdate(userID, changedKey, oldProfile.AvatarURL, r.AvatarURL); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("producer.SendUpdate failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -185,7 +185,7 @@ func GetDisplayName(
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
 	if err != nil {
-		if err == common.ErrProfileNoExists {
+		if err == eventutil.ErrProfileNoExists {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
 				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
@@ -198,17 +198,17 @@ func GetDisplayName(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: common.DisplayName{
+		JSON: eventutil.DisplayName{
 			DisplayName: profile.DisplayName,
 		},
 	}
 }
 
 // SetDisplayName implements PUT /profile/{userID}/displayname
+// nolint:gocyclo
 func SetDisplayName(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
-	userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite,
+	userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
-	rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -217,9 +217,7 @@ func SetDisplayName(
 		}
 	}
 
-	changedKey := "displayname"
+	var r eventutil.DisplayName
-
-	var r common.DisplayName
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
@@ -268,20 +266,22 @@ func SetDisplayName(
 	}
 
 	events, err := buildMembershipEvents(
-		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+		req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
 	)
-	if err != nil {
+	switch e := err.(type) {
+	case nil:
+	case gomatrixserverlib.BadJSONError:
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(e.Error()),
+		}
+	default:
 		util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed")
 		return jsonerror.InternalServerError()
 	}
 
-	if _, err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil {
+	if _, err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("rsProducer.SendEvents failed")
+		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if err := producer.SendUpdate(userID, changedKey, oldProfile.DisplayName, r.DisplayName); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("producer.SendUpdate failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -294,7 +294,7 @@ func SetDisplayName(
 // getProfile gets the full profile of a user by querying the database or a
 // remote homeserver.
 // Returns an error when something goes wrong or specifically
-// common.ErrProfileNoExists when the profile doesn't exist.
+// eventutil.ErrProfileNoExists when the profile doesn't exist.
 func getProfile(
 	ctx context.Context, accountDB accounts.Database, cfg *config.Dendrite,
 	userID string,
@@ -311,7 +311,7 @@ func getProfile(
 		if fedErr != nil {
 			if x, ok := fedErr.(gomatrix.HTTPError); ok {
 				if x.Code == http.StatusNotFound {
-					return nil, common.ErrProfileNoExists
+					return nil, eventutil.ErrProfileNoExists
 				}
 			}
 
@@ -337,14 +337,14 @@ func buildMembershipEvents(
 	ctx context.Context,
 	memberships []authtypes.Membership,
 	newProfile authtypes.Profile, userID string, cfg *config.Dendrite,
-	evTime time.Time, queryAPI api.RoomserverQueryAPI,
+	evTime time.Time, rsAPI api.RoomserverInternalAPI,
 ) ([]gomatrixserverlib.HeaderedEvent, error) {
 	evs := []gomatrixserverlib.HeaderedEvent{}
 
 	for _, membership := range memberships {
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
-		if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
+		if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
 			return []gomatrixserverlib.HeaderedEvent{}, err
 		}
 
@@ -366,7 +366,7 @@ func buildMembershipEvents(
 			return nil, err
 		}
 
-		event, err := common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil)
+		event, err := eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
 		if err != nil {
 			return nil, err
 		}

clientapi/routing/register.go

@@ -32,16 +32,16 @@ import (
 	"sync"
 	"time"
 
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
-	"github.com/matrix-org/dendrite/common"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/tokens"
 	"github.com/matrix-org/util"
@@ -136,7 +136,7 @@ type registerRequest struct {
 	DeviceID           *string `json:"device_id"`
 
 	// Prevent this user from logging in
-	InhibitLogin common.WeakBoolean `json:"inhibit_login"`
+	InhibitLogin eventutil.WeakBoolean `json:"inhibit_login"`
 
 	// Application Services place Type in the root of their registration
 	// request, whereas clients place it in the authDict struct.
@@ -440,8 +440,8 @@ func validateApplicationService(
 // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
 func Register(
 	req *http.Request,
+	userAPI userapi.UserInternalAPI,
 	accountDB accounts.Database,
-	deviceDB devices.Database,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var r registerRequest
@@ -450,7 +450,7 @@ func Register(
 		return *resErr
 	}
 	if req.URL.Query().Get("kind") == "guest" {
-		return handleGuestRegistration(req, r, cfg, accountDB, deviceDB)
+		return handleGuestRegistration(req, r, cfg, userAPI)
 	}
 
 	// Retrieve or generate the sessionID
@@ -506,17 +506,19 @@ func Register(
 		"session_id": r.Auth.Session,
 	}).Info("Processing registration request")
 
-	return handleRegistrationFlow(req, r, sessionID, cfg, accountDB, deviceDB)
+	return handleRegistrationFlow(req, r, sessionID, cfg, userAPI)
 }
 
 func handleGuestRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 ) util.JSONResponse {
-	acc, err := accountDB.CreateGuestAccount(req.Context())
+	var res userapi.PerformAccountCreationResponse
+	err := userAPI.PerformAccountCreation(req.Context(), &userapi.PerformAccountCreationRequest{
+		AccountType: userapi.AccountTypeGuest,
+	}, &res)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -525,8 +527,8 @@ func handleGuestRegistration(
 	}
 	token, err := tokens.GenerateLoginToken(tokens.TokenOptions{
 		ServerPrivateKey: cfg.Matrix.PrivateKey.Seed(),
-		ServerName:       string(acc.ServerName),
+		ServerName:       string(res.Account.ServerName),
-		UserID:           acc.UserID,
+		UserID:           res.Account.UserID,
 	})
 
 	if err != nil {
@@ -536,7 +538,12 @@ func handleGuestRegistration(
 		}
 	}
 	//we don't allow guests to specify their own device_id
-	dev, err := deviceDB.CreateDevice(req.Context(), acc.Localpart, nil, token, r.InitialDisplayName)
+	var devRes userapi.PerformDeviceCreationResponse
+	err = userAPI.PerformDeviceCreation(req.Context(), &userapi.PerformDeviceCreationRequest{
+		Localpart:         res.Account.Localpart,
+		DeviceDisplayName: r.InitialDisplayName,
+		AccessToken:       token,
+	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -546,10 +553,10 @@ func handleGuestRegistration(
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: registerResponse{
-			UserID:      dev.UserID,
+			UserID:      devRes.Device.UserID,
-			AccessToken: dev.AccessToken,
+			AccessToken: devRes.Device.AccessToken,
-			HomeServer:  acc.ServerName,
+			HomeServer:  res.Account.ServerName,
-			DeviceID:    dev.ID,
+			DeviceID:    devRes.Device.ID,
 		},
 	}
 }
@@ -562,8 +569,7 @@ func handleRegistrationFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 ) util.JSONResponse {
 	// TODO: Shared secret registration (create new user scripts)
 	// TODO: Enable registration config flag
@@ -614,7 +620,7 @@ func handleRegistrationFlow(
 		// by whether the request contains an access token.
 		if err == nil {
 			return handleApplicationServiceRegistration(
-				accessToken, err, req, r, cfg, accountDB, deviceDB,
+				accessToken, err, req, r, cfg, userAPI,
 			)
 		}
 
@@ -625,7 +631,7 @@ func handleRegistrationFlow(
 		// don't need a condition on that call since the registration is clearly
 		// stated as being AS-related.
 		return handleApplicationServiceRegistration(
-			accessToken, err, req, r, cfg, accountDB, deviceDB,
+			accessToken, err, req, r, cfg, userAPI,
 		)
 
 	case authtypes.LoginTypeDummy:
@@ -644,7 +650,7 @@ func handleRegistrationFlow(
 	// A response with current registration flow and remaining available methods
 	// will be returned if a flow has not been successfully completed yet
 	return checkAndCompleteFlow(sessions.GetCompletedStages(sessionID),
-		req, r, sessionID, cfg, accountDB, deviceDB)
+		req, r, sessionID, cfg, userAPI)
 }
 
 // handleApplicationServiceRegistration handles the registration of an
@@ -661,8 +667,7 @@ func handleApplicationServiceRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 ) util.JSONResponse {
 	// Check if we previously had issues extracting the access token from the
 	// request.
@@ -686,7 +691,7 @@ func handleApplicationServiceRegistration(
 	// Don't need to worry about appending to registration stages as
 	// application service registration is entirely separate.
 	return completeRegistration(
-		req.Context(), accountDB, deviceDB, r.Username, "", appserviceID,
+		req.Context(), userAPI, r.Username, "", appserviceID,
 		r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 	)
 }
@@ -700,13 +705,12 @@ func checkAndCompleteFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 ) util.JSONResponse {
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
 		return completeRegistration(
-			req.Context(), accountDB, deviceDB, r.Username, r.Password, "",
+			req.Context(), userAPI, r.Username, r.Password, "",
 			r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 		)
 	}
@@ -723,8 +727,7 @@ func checkAndCompleteFlow(
 // LegacyRegister process register requests from the legacy v1 API
 func LegacyRegister(
 	req *http.Request,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var r legacyRegisterRequest
@@ -759,10 +762,10 @@ func LegacyRegister(
 			return util.MessageResponse(http.StatusForbidden, "HMAC incorrect")
 		}
 
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
+		return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil)
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
+		return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil)
 	default:
 		return util.JSONResponse{
 			Code: http.StatusNotImplemented,
@@ -808,10 +811,9 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u
 // not all
 func completeRegistration(
 	ctx context.Context,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
-	deviceDB devices.Database,
 	username, password, appserviceID string,
-	inhibitLogin common.WeakBoolean,
+	inhibitLogin eventutil.WeakBoolean,
 	displayName, deviceID *string,
 ) util.JSONResponse {
 	if username == "" {
@@ -828,17 +830,25 @@ func completeRegistration(
 		}
 	}
 
-	acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID)
+	var accRes userapi.PerformAccountCreationResponse
+	err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{
+		AppServiceID: appserviceID,
+		Localpart:    username,
+		Password:     password,
+		AccountType:  userapi.AccountTypeUser,
+		OnConflict:   userapi.ConflictAbort,
+	}, &accRes)
 	if err != nil {
+		if _, ok := err.(*userapi.ErrorConflict); ok { // user already exists
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.UserInUse("Desired user ID is already taken."),
+			}
+		}
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.Unknown("failed to create account: " + err.Error()),
 		}
-	} else if acc == nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.UserInUse("Desired user ID is already taken."),
-		}
 	}
 
 	// Increment prometheus counter for created users
@@ -850,8 +860,8 @@ func completeRegistration(
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: registerResponse{
-				UserID:     userutil.MakeUserID(username, acc.ServerName),
+				UserID:     userutil.MakeUserID(username, accRes.Account.ServerName),
-				HomeServer: acc.ServerName,
+				HomeServer: accRes.Account.ServerName,
 			},
 		}
 	}
@@ -864,7 +874,13 @@ func completeRegistration(
 		}
 	}
 
-	dev, err := deviceDB.CreateDevice(ctx, username, deviceID, token, displayName)
+	var devRes userapi.PerformDeviceCreationResponse
+	err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
+		Localpart:         username,
+		AccessToken:       token,
+		DeviceDisplayName: displayName,
+		DeviceID:          deviceID,
+	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -875,10 +891,10 @@ func completeRegistration(
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: registerResponse{
-			UserID:      dev.UserID,
+			UserID:      devRes.Device.UserID,
-			AccessToken: dev.AccessToken,
+			AccessToken: devRes.Device.AccessToken,
-			HomeServer:  acc.ServerName,
+			HomeServer:  accRes.Account.ServerName,
-			DeviceID:    dev.ID,
+			DeviceID:    devRes.Device.ID,
 		},
 	}
 }

clientapi/routing/register_test.go

@@ -19,7 +19,7 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
 var (

clientapi/routing/room_tagging.go

@@ -20,28 +20,19 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
-// newTag creates and returns a new gomatrix.TagContent
-func newTag() gomatrix.TagContent {
-	return gomatrix.TagContent{
-		Tags: make(map[string]gomatrix.TagProperties),
-	}
-}
-
 // GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags
 func GetTags(
 	req *http.Request,
-	accountDB accounts.Database,
+	userAPI api.UserInternalAPI,
-	device *authtypes.Device,
+	device *api.Device,
 	userID string,
 	roomID string,
 	syncProducer *producers.SyncAPIProducer,
@@ -54,22 +45,15 @@ func GetTags(
 		}
 	}
 
-	_, data, err := obtainSavedTags(req, userID, roomID, accountDB)
+	tagContent, err := obtainSavedTags(req, userID, roomID, userAPI)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed")
 		return jsonerror.InternalServerError()
 	}
 
-	if data == nil {
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: struct{}{},
-		}
-	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: data.Content,
+		JSON: tagContent,
 	}
 }
 
@@ -78,8 +62,8 @@ func GetTags(
 // the tag to the "map" and saving the new "map" to the DB
 func PutTag(
 	req *http.Request,
-	accountDB accounts.Database,
+	userAPI api.UserInternalAPI,
-	device *authtypes.Device,
+	device *api.Device,
 	userID string,
 	roomID string,
 	tag string,
@@ -98,34 +82,25 @@ func PutTag(
 		return *reqErr
 	}
 
-	localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB)
+	tagContent, err := obtainSavedTags(req, userID, roomID, userAPI)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed")
 		return jsonerror.InternalServerError()
 	}
 
-	var tagContent gomatrix.TagContent
+	if tagContent.Tags == nil {
-	if data != nil {
+		tagContent.Tags = make(map[string]gomatrix.TagProperties)
-		if err = json.Unmarshal(data.Content, &tagContent); err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed")
-			return jsonerror.InternalServerError()
-		}
-	} else {
-		tagContent = newTag()
 	}
 	tagContent.Tags[tag] = properties
-	if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil {
+
+	if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed")
 		return jsonerror.InternalServerError()
 	}
 
-	// Send data to syncProducer in order to inform clients of changes
+	if err = syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
-	// Run in a goroutine in order to prevent blocking the tag request response
+		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-	go func() {
+	}
-		if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
-			logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-		}
-	}()
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -138,8 +113,8 @@ func PutTag(
 // the "map" and then saving the new "map" in the DB
 func DeleteTag(
 	req *http.Request,
-	accountDB accounts.Database,
+	userAPI api.UserInternalAPI,
-	device *authtypes.Device,
+	device *api.Device,
 	userID string,
 	roomID string,
 	tag string,
@@ -153,28 +128,12 @@ func DeleteTag(
 		}
 	}
 
-	localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB)
+	tagContent, err := obtainSavedTags(req, userID, roomID, userAPI)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed")
 		return jsonerror.InternalServerError()
 	}
 
-	// If there are no tags in the database, exit
-	if data == nil {
-		// Spec only defines 200 responses for this endpoint so we don't return anything else.
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: struct{}{},
-		}
-	}
-
-	var tagContent gomatrix.TagContent
-	err = json.Unmarshal(data.Content, &tagContent)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed")
-		return jsonerror.InternalServerError()
-	}
-
 	// Check whether the tag to be deleted exists
 	if _, ok := tagContent.Tags[tag]; ok {
 		delete(tagContent.Tags, tag)
@@ -185,18 +144,16 @@ func DeleteTag(
 			JSON: struct{}{},
 		}
 	}
-	if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil {
+
+	if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed")
 		return jsonerror.InternalServerError()
 	}
 
-	// Send data to syncProducer in order to inform clients of changes
+	// TODO: user API should do this since it's account data
-	// Run in a goroutine in order to prevent blocking the tag request response
+	if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
-	go func() {
+		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-		if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
+	}
-			logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-		}
-	}()
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -210,32 +167,46 @@ func obtainSavedTags(
 	req *http.Request,
 	userID string,
 	roomID string,
-	accountDB accounts.Database,
+	userAPI api.UserInternalAPI,
-) (string, *gomatrixserverlib.ClientEvent, error) {
+) (tags gomatrix.TagContent, err error) {
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	dataReq := api.QueryAccountDataRequest{
-	if err != nil {
+		UserID:   userID,
-		return "", nil, err
+		RoomID:   roomID,
+		DataType: "m.tag",
 	}
-
+	dataRes := api.QueryAccountDataResponse{}
-	data, err := accountDB.GetAccountDataByType(
+	err = userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes)
-		req.Context(), localpart, roomID, "m.tag",
+	if err != nil {
-	)
+		return
-
+	}
-	return localpart, data, err
+	data, ok := dataRes.RoomAccountData[roomID]["m.tag"]
+	if !ok {
+		return
+	}
+	if err = json.Unmarshal(data, &tags); err != nil {
+		return
+	}
+	return tags, nil
 }
 
 // saveTagData saves the provided tag data into the database
 func saveTagData(
 	req *http.Request,
-	localpart string,
+	userID string,
 	roomID string,
-	accountDB accounts.Database,
+	userAPI api.UserInternalAPI,
 	Tag gomatrix.TagContent,
 ) error {
 	newTagData, err := json.Marshal(Tag)
 	if err != nil {
 		return err
 	}
-
+	dataReq := api.InputAccountDataRequest{
-	return accountDB.SaveAccountData(req.Context(), localpart, roomID, "m.tag", string(newTagData))
+		UserID:      userID,
+		RoomID:      roomID,
+		DataType:    "m.tag",
+		AccountData: json.RawMessage(newTagData),
+	}
+	dataRes := api.InputAccountDataResponse{}
+	return userAPI.InputAccountData(req.Context(), &dataReq, &dataRes)
 }

clientapi/routing/routing.go

@@ -21,24 +21,24 @@ import (
 
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
-	"github.com/matrix-org/dendrite/common"
+	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
-	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/common/transactions"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
-const pathPrefixV1 = "/_matrix/client/api/v1"
+const pathPrefixV1 = "/client/api/v1"
-const pathPrefixR0 = "/_matrix/client/r0"
+const pathPrefixR0 = "/client/r0"
-const pathPrefixUnstable = "/_matrix/client/unstable"
+const pathPrefixUnstable = "/client/unstable"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
@@ -47,24 +47,21 @@ const pathPrefixUnstable = "/_matrix/client/unstable"
 // applied:
 // nolint: gocyclo
 func Setup(
-	apiMux *mux.Router, cfg *config.Dendrite,
+	publicAPIMux *mux.Router, cfg *config.Dendrite,
-	producer *producers.RoomserverProducer,
+	eduAPI eduServerAPI.EDUServerInputAPI,
-	queryAPI roomserverAPI.RoomserverQueryAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
-	aliasAPI roomserverAPI.RoomserverAliasAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	accountDB accounts.Database,
 	deviceDB devices.Database,
+	userAPI api.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing gomatrixserverlib.KeyRing,
-	userUpdateProducer *producers.UserUpdateProducer,
 	syncProducer *producers.SyncAPIProducer,
-	eduProducer *producers.EDUServerProducer,
 	transactionsCache *transactions.Cache,
-	federationSender federationSenderAPI.FederationSenderQueryAPI,
+	federationSender federationSenderAPI.FederationSenderInternalAPI,
 ) {
 
-	apiMux.Handle("/_matrix/client/versions",
+	publicAPIMux.Handle("/client/versions",
-		common.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusOK,
 				JSON: struct {
@@ -79,104 +76,115 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+	r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter()
-	v1mux := apiMux.PathPrefix(pathPrefixV1).Subrouter()
+	v1mux := publicAPIMux.PathPrefix(pathPrefixV1).Subrouter()
-	unstableMux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter()
+	unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter()
-
-	authData := auth.Data{
-		AccountDB:   accountDB,
-		DeviceDB:    deviceDB,
-		AppServices: cfg.Derived.ApplicationServices,
-	}
 
 	r0mux.Handle("/createRoom",
-		common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI, asAPI)
+			return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
-		common.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
 			return JoinRoomByIDOrAlias(
-				req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB,
+				req, device, rsAPI, accountDB, vars["roomIDOrAlias"],
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/joined_rooms",
-		common.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetJoinedRooms(req, device, accountDB)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
-
+	r0mux.Handle("/rooms/{roomID}/leave",
-	r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}",
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-		common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer)
+			return LeaveRoomByID(
+				req, device, rsAPI, vars["roomID"],
+			)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}",
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
-		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil)
+			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, nil)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
-		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
 			txnID := vars["txnID"]
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID,
-				nil, cfg, queryAPI, producer, transactionsCache)
+				nil, cfg, rsAPI, transactionsCache)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/event/{eventID}",
-		common.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing)
+			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-		vars, err := common.URLDecodeMapValues(mux.Vars(req))
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateRequest(req.Context(), queryAPI, vars["roomID"])
+		return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-		vars, err := common.URLDecodeMapValues(mux.Vars(req))
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], "")
+		// If there's a trailing slash, remove it
+		eventType := vars["type"]
+		if strings.HasSuffix(eventType, "/") {
+			eventType = eventType[:len(eventType)-1]
+		}
+		eventFormat := req.URL.Query().Get("format") == "event"
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], eventType, "", eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-		vars, err := common.URLDecodeMapValues(mux.Vars(req))
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], vars["stateKey"])
+		eventFormat := req.URL.Query().Get("format") == "event"
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"], eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
-		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -186,87 +194,112 @@ func Setup(
 			if strings.HasSuffix(eventType, "/") {
 				eventType = eventType[:len(eventType)-1]
 			}
-			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil)
+			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, nil)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
-		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
 			stateKey := vars["stateKey"]
-			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil)
+			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, nil)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/register", common.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
+	r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
-		return Register(req, accountDB, deviceDB, cfg)
+		return Register(req, userAPI, accountDB, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
-	v1mux.Handle("/register", common.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
+	v1mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
-		return LegacyRegister(req, accountDB, deviceDB, cfg)
+		return LegacyRegister(req, userAPI, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/register/available", common.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
+	r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
 		return RegisterAvailable(req, cfg, accountDB)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		common.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return DirectoryRoom(req, vars["roomAlias"], federation, cfg, aliasAPI, federationSender)
+			return DirectoryRoom(req, vars["roomAlias"], federation, cfg, rsAPI, federationSender)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetLocalAlias(req, device, vars["roomAlias"], cfg, aliasAPI)
+			return SetLocalAlias(req, device, vars["roomAlias"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI)
+			return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/logout",
-		common.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return Logout(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/logout/all",
-		common.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return LogoutAll(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
-		common.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduProducer)
+			return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	r0mux.Handle("/sendToDevice/{eventType}/{txnID}",
+		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			txnID := vars["txnID"]
+			return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	// This is only here because sytest refers to /unstable for this endpoint
+	// rather than r0. It's an exact duplicate of the above handler.
+	// TODO: Remove this if/when sytest is fixed!
+	unstableMux.Handle("/sendToDevice/{eventType}/{txnID}",
+		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			txnID := vars["txnID"]
+			return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/account/whoami",
-		common.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return Whoami(req, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -274,20 +307,20 @@ func Setup(
 	// Stub endpoints required by Riot
 
 	r0mux.Handle("/login",
-		common.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
 			return Login(req, accountDB, deviceDB, cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/auth/{authType}/fallback/web",
-		common.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse {
+		httputil.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse {
 			vars := mux.Vars(req)
 			return AuthFallback(w, req, vars["authType"], cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/pushrules/",
-		common.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse {
 			// TODO: Implement push rules API
 			res := json.RawMessage(`{
 					"global": {
@@ -306,8 +339,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/filter",
-		common.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -316,8 +349,8 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/filter/{filterId}",
-		common.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -328,8 +361,8 @@ func Setup(
 	// Riot user settings
 
 	r0mux.Handle("/profile/{userID}",
-		common.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -338,8 +371,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
-		common.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -348,20 +381,20 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
-		common.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI)
+			return SetAvatarURL(req, accountDB, device, vars["userID"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
 	// PUT requests, so we need to allow this method
 
 	r0mux.Handle("/profile/{userID}/displayname",
-		common.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -370,44 +403,44 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/displayname",
-		common.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI)
+			return SetDisplayName(req, accountDB, device, vars["userID"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
 	// PUT requests, so we need to allow this method
 
 	r0mux.Handle("/account/3pid",
-		common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetAssociated3PIDs(req, accountDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/account/3pid",
-		common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return CheckAndSave3PIDAssociation(req, accountDB, device, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	unstableMux.Handle("/account/3pid/delete",
-		common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return Forget3PID(req, accountDB)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken",
-		common.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse {
 			return RequestEmailToken(req, accountDB, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	// Riot logs get flooded unless this is handled
 	r0mux.Handle("/presence/{userID}/status",
-		common.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
 			// TODO: Set presence (probably the responsibility of a presence server not clientapi)
 			return util.JSONResponse{
 				Code: http.StatusOK,
@@ -417,13 +450,13 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/voip/turnServer",
-		common.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return RequestTurnServer(req, device, cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/thirdparty/protocols",
-		common.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
 			// TODO: Return the third party protcols
 			return util.JSONResponse{
 				Code: http.StatusOK,
@@ -433,7 +466,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/initialSync",
-		common.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse {
 			// TODO: Allow people to peek into rooms.
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
@@ -443,81 +476,81 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
-		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer)
+			return SaveAccountData(req, userAPI, device, vars["userID"], "", vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
-		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer)
+			return SaveAccountData(req, userAPI, device, vars["userID"], vars["roomID"], vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
-		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetAccountData(req, accountDB, device, vars["userID"], "", vars["type"])
+			return GetAccountData(req, userAPI, device, vars["userID"], "", vars["type"])
 		}),
 	).Methods(http.MethodGet)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
-		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"])
+			return GetAccountData(req, userAPI, device, vars["userID"], vars["roomID"], vars["type"])
 		}),
 	).Methods(http.MethodGet)
 
 	r0mux.Handle("/rooms/{roomID}/members",
-		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI)
+			return GetMemberships(req, device, vars["roomID"], false, cfg, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/joined_members",
-		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI)
+			return GetMemberships(req, device, vars["roomID"], true, cfg, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/read_markers",
-		common.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse {
 			// TODO: return the read_markers.
 			return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/devices",
-		common.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetDevicesByLocalpart(req, deviceDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		common.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -526,8 +559,8 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		common.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -536,8 +569,8 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		common.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
@@ -546,14 +579,14 @@ func Setup(
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/delete_devices",
-		common.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return DeleteDevices(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	// Stub implementations for sytest
 	r0mux.Handle("/events",
-		common.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
 				"chunk": []interface{}{},
 				"start": "",
@@ -563,7 +596,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/initialSync",
-		common.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse {
+		httputil.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
 				"end": "",
 			}}
@@ -571,38 +604,38 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags",
-		common.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetTags(req, accountDB, device, vars["userId"], vars["roomId"], syncProducer)
+			return GetTags(req, userAPI, device, vars["userId"], vars["roomId"], syncProducer)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
-		common.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return PutTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
+			return PutTag(req, userAPI, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
-		common.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return DeleteTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
+			return DeleteTag(req, userAPI, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/capabilities",
-		common.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
-			return GetCapabilities(req, queryAPI)
+			return GetCapabilities(req, rsAPI)
 		}),
 	).Methods(http.MethodGet)
 }

clientapi/routing/sendevent.go

@@ -17,14 +17,13 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/eventutil"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/transactions"
-	"github.com/matrix-org/dendrite/common/transactions"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
@@ -42,16 +41,15 @@ type sendEventResponse struct {
 //   /rooms/{roomID}/state/{eventType}/{stateKey}
 func SendEvent(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	roomID, eventType string, txnID, stateKey *string,
 	cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
-	producer *producers.RoomserverProducer,
 	txnCache *transactions.Cache,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
@@ -65,7 +63,7 @@ func SendEvent(
 		}
 	}
 
-	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, queryAPI)
+	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI)
 	if resErr != nil {
 		return *resErr
 	}
@@ -80,8 +78,8 @@ func SendEvent(
 
 	// pass the new event to the roomserver and receive the correct event ID
 	// event ID in case of duplicate transaction is discarded
-	eventID, err := producer.SendEvents(
+	eventID, err := api.SendEvents(
-		req.Context(),
+		req.Context(), rsAPI,
 		[]gomatrixserverlib.HeaderedEvent{
 			e.Headered(verRes.RoomVersion),
 		},
@@ -89,7 +87,7 @@ func SendEvent(
 		txnAndSessionID,
 	)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed")
+		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
 		return jsonerror.InternalServerError()
 	}
 	util.GetLogger(req.Context()).WithFields(logrus.Fields{
@@ -112,10 +110,10 @@ func SendEvent(
 
 func generateSendEvent(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	roomID, eventType string, stateKey *string,
 	cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	rsAPI api.RoomserverInternalAPI,
 ) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	// parse the incoming http request
 	userID := device.UserID
@@ -148,14 +146,19 @@ func generateSendEvent(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, queryAPI, &queryRes)
+	e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes)
-	if err == common.ErrRoomNoExists {
+	if err == eventutil.ErrRoomNoExists {
 		return nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound("Room does not exist"),
 		}
+	} else if e, ok := err.(gomatrixserverlib.BadJSONError); ok {
+		return nil, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(e.Error()),
+		}
 	} else if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("common.BuildEvent failed")
+		util.GetLogger(req.Context()).WithError(err).Error("eventutil.BuildEvent failed")
 		resErr := jsonerror.InternalServerError()
 		return nil, &resErr
 	}

clientapi/routing/sendtodevice.go

@@ -0,0 +1,70 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/eduserver/api"
+	"github.com/matrix-org/dendrite/internal/transactions"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+)
+
+// SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}
+// sends the device events to the EDU Server
+func SendToDevice(
+	req *http.Request, device *userapi.Device,
+	eduAPI api.EDUServerInputAPI,
+	txnCache *transactions.Cache,
+	eventType string, txnID *string,
+) util.JSONResponse {
+	if txnID != nil {
+		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
+			return *res
+		}
+	}
+
+	var httpReq struct {
+		Messages map[string]map[string]json.RawMessage `json:"messages"`
+	}
+	resErr := httputil.UnmarshalJSONRequest(req, &httpReq)
+	if resErr != nil {
+		return *resErr
+	}
+
+	for userID, byUser := range httpReq.Messages {
+		for deviceID, message := range byUser {
+			if err := api.SendToDevice(
+				req.Context(), eduAPI, device.UserID, userID, deviceID, eventType, message,
+			); err != nil {
+				util.GetLogger(req.Context()).WithError(err).Error("eduProducer.SendToDevice failed")
+				return jsonerror.InternalServerError()
+			}
+		}
+	}
+
+	res := util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+
+	if txnID != nil {
+		txnCache.AddTransaction(device.AccessToken, *txnID, &res)
+	}
+
+	return res
+}

clientapi/routing/sendtyping.go

@@ -16,12 +16,12 @@ import (
 	"database/sql"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/eduserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/util"
 )
 
@@ -33,9 +33,9 @@ type typingContentJSON struct {
 // SendTyping handles PUT /rooms/{roomID}/typing/{userID}
 // sends the typing events to client API typingProducer
 func SendTyping(
-	req *http.Request, device *authtypes.Device, roomID string,
+	req *http.Request, device *userapi.Device, roomID string,
 	userID string, accountDB accounts.Database,
-	eduProducer *producers.EDUServerProducer,
+	eduAPI api.EDUServerInputAPI,
 ) util.JSONResponse {
 	if device.UserID != userID {
 		return util.JSONResponse{
@@ -69,8 +69,8 @@ func SendTyping(
 		return *resErr
 	}
 
-	if err = eduProducer.SendTyping(
+	if err = api.SendTyping(
-		req.Context(), userID, roomID, r.Typing, r.Timeout,
+		req.Context(), eduAPI, userID, roomID, r.Typing, r.Timeout,
 	); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("eduProducer.Send failed")
 		return jsonerror.InternalServerError()

clientapi/routing/state.go

@@ -40,7 +40,7 @@ type stateEventInStateResp struct {
 // TODO: Check if the user is in the room. If not, check if the room's history
 // is publicly visible. Current behaviour is returning an empty array if the
 // user cannot see the room's history.
-func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string) util.JSONResponse {
+func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 	stateReq := api.QueryLatestEventsAndStateRequest{
@@ -48,7 +48,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI
 	}
 	stateRes := api.QueryLatestEventsAndStateResponse{}
 
-	if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
+	if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
 		return jsonerror.InternalServerError()
 	}
@@ -98,7 +98,8 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI
 // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current
 // state to see if there is an event with that type and state key, if there
 // is then (by default) we return the content, otherwise a 404.
-func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string, evType, stateKey string) util.JSONResponse {
+// If eventFormat=true, sends the whole event else just the content.
+func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID, evType, stateKey string, eventFormat bool) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 	util.GetLogger(ctx).WithFields(log.Fields{
@@ -118,7 +119,7 @@ func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQuer
 	}
 	stateRes := api.QueryLatestEventsAndStateResponse{}
 
-	if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
+	if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
 		return jsonerror.InternalServerError()
 	}
@@ -134,8 +135,15 @@ func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQuer
 		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(stateRes.StateEvents[0], gomatrixserverlib.FormatAll),
 	}
 
+	var res interface{}
+	if eventFormat {
+		res = stateEvent
+	} else {
+		res = stateEvent.Content
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: stateEvent.Content,
+		JSON: res,
 	}
 }

clientapi/routing/threepid.go

@@ -18,11 +18,12 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -84,7 +85,7 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf
 
 // CheckAndSave3PIDAssociation implements POST /account/3pid
 func CheckAndSave3PIDAssociation(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *api.Device,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var body threepid.EmailAssociationCheckRequest
@@ -148,7 +149,7 @@ func CheckAndSave3PIDAssociation(
 
 // GetAssociated3PIDs implements GET /account/3pid
 func GetAssociated3PIDs(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {

clientapi/routing/voip.go

@@ -22,16 +22,16 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/util"
 )
 
 // RequestTurnServer implements:
 //     GET /voip/turnServer
-func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg *config.Dendrite) util.JSONResponse {
+func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Dendrite) util.JSONResponse {
 	turnConfig := cfg.TURN
 
 	// TODO Guest Support

clientapi/routing/whoami.go

@@ -15,7 +15,7 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
@@ -26,7 +26,7 @@ type whoamiResponse struct {
 
 // Whoami implements `/account/whoami` which enables client to query their account user id.
 // https://matrix.org/docs/spec/client_server/r0.3.0.html#get-matrix-client-r0-account-whoami
-func Whoami(req *http.Request, device *authtypes.Device) util.JSONResponse {
+func Whoami(req *http.Request, device *api.Device) util.JSONResponse {
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: whoamiResponse{UserID: device.UserID},

clientapi/threepid/invites.go

@@ -25,11 +25,11 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/internal/eventutil"
-	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -86,9 +86,9 @@ var (
 // can be emitted.
 func CheckAndProcessInvite(
 	ctx context.Context,
-	device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite,
+	device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI, db accounts.Database,
+	rsAPI api.RoomserverInternalAPI, db accounts.Database,
-	producer *producers.RoomserverProducer, membership string, roomID string,
+	membership string, roomID string,
 	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
 	if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") {
@@ -112,7 +112,7 @@ func CheckAndProcessInvite(
 		// "m.room.third_party_invite" have to be emitted from the data in
 		// storeInviteRes.
 		err = emit3PIDInviteEvent(
-			ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer, evTime,
+			ctx, body, storeInviteRes, device, roomID, cfg, rsAPI, evTime,
 		)
 		inviteStoredOnIDServer = err == nil
 
@@ -137,7 +137,7 @@ func CheckAndProcessInvite(
 // Returns an error if a check or a request failed.
 func queryIDServer(
 	ctx context.Context,
-	db accounts.Database, cfg *config.Dendrite, device *authtypes.Device,
+	db accounts.Database, cfg *config.Dendrite, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) {
 	if err = isTrusted(body.IDServer, cfg); err != nil {
@@ -206,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe
 // Returns an error if the request failed to send or if the response couldn't be parsed.
 func queryIDServerStoreInvite(
 	ctx context.Context,
-	db accounts.Database, cfg *config.Dendrite, device *authtypes.Device,
+	db accounts.Database, cfg *config.Dendrite, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (*idServerStoreInviteResponse, error) {
 	// Retrieve the sender's profile to get their display name
@@ -279,7 +279,7 @@ func queryIDServerPubKey(ctx context.Context, idServerName string, keyID string)
 	}
 
 	var pubKeyRes struct {
-		PublicKey gomatrixserverlib.Base64String `json:"public_key"`
+		PublicKey gomatrixserverlib.Base64Bytes `json:"public_key"`
 	}
 
 	if resp.StatusCode != http.StatusOK {
@@ -330,8 +330,8 @@ func checkIDServerSignatures(
 func emit3PIDInviteEvent(
 	ctx context.Context,
 	body *MembershipRequest, res *idServerStoreInviteResponse,
-	device *authtypes.Device, roomID string, cfg *config.Dendrite,
+	device *userapi.Device, roomID string, cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer,
+	rsAPI api.RoomserverInternalAPI,
 	evTime time.Time,
 ) error {
 	builder := &gomatrixserverlib.EventBuilder{
@@ -354,13 +354,13 @@ func emit3PIDInviteEvent(
 	}
 
 	queryRes := api.QueryLatestEventsAndStateResponse{}
-	event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, &queryRes)
+	event, err := eventutil.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes)
 	if err != nil {
 		return err
 	}
 
-	_, err = producer.SendEvents(
+	_, err = api.SendEvents(
-		ctx,
+		ctx, rsAPI,
 		[]gomatrixserverlib.HeaderedEvent{
 			(*event).Headered(queryRes.RoomVersion),
 		},

clientapi/threepid/threepid.go

@@ -24,7 +24,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
 // EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken

cmd/client-api-proxy/main.go

@@ -75,7 +75,6 @@ func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
 			// Pratically this means that any distinction between '%2F' and '/'
 			// in the URL will be lost by the time it reaches the target.
 			path := req.URL.Path
-			path = "api" + path
 			log.WithFields(log.Fields{
 				"path":   path,
 				"url":    targetURL,

cmd/create-account/main.go

@@ -20,8 +20,8 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -63,22 +63,19 @@ func main() {
 
 	serverName := gomatrixserverlib.ServerName(*serverNameStr)
 
-	accountDB, err := accounts.NewDatabase(*database, serverName)
+	accountDB, err := accounts.NewDatabase(*database, nil, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)
 	}
 
-	account, err := accountDB.CreateAccount(context.Background(), *username, *password, "")
+	_, err = accountDB.CreateAccount(context.Background(), *username, *password, "")
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)
-	} else if account == nil {
-		fmt.Println("Username already exists")
-		os.Exit(1)
 	}
 
-	deviceDB, err := devices.NewDatabase(*database, serverName)
+	deviceDB, err := devices.NewDatabase(*database, nil, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)

cmd/create-room-events/main.go

@@ -47,6 +47,7 @@ var (
 	userID           = flag.String("user-id", "@userid:$SERVER_NAME", "The user ID to use as the event sender")
 	messageCount     = flag.Int("message-count", 10, "The number of m.room.messsage events to generate")
 	format           = flag.String("Format", "InputRoomEvent", "The output format to use for the messages: InputRoomEvent or Event")
+	ver              = flag.String("version", string(gomatrixserverlib.RoomVersionV1), "Room version to generate events as")
 )
 
 // By default we use a private key of 0.
@@ -109,7 +110,7 @@ func buildAndOutput() gomatrixserverlib.EventReference {
 
 	event, err := b.Build(
 		now, name, key, privateKey,
-		gomatrixserverlib.RoomVersionV1,
+		gomatrixserverlib.RoomVersion(*ver),
 	)
 	if err != nil {
 		panic(err)
@@ -127,7 +128,7 @@ func writeEvent(event gomatrixserverlib.Event) {
 	if *format == "InputRoomEvent" {
 		var ire api.InputRoomEvent
 		ire.Kind = api.KindNew
-		ire.Event = event.Headered(gomatrixserverlib.RoomVersionV1)
+		ire.Event = event.Headered(gomatrixserverlib.RoomVersion(*ver))
 		authEventIDs := []string{}
 		for _, ref := range b.AuthEvents.([]gomatrixserverlib.EventReference) {
 			authEventIDs = append(authEventIDs, ref.EventID)

cmd/dendrite-appservice-server/main.go

@@ -16,24 +16,19 @@ package main
 
 import (
 	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/common/transactions"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags()
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "AppServiceAPI")
+	base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true)
 
 	defer base.Close() // nolint: errcheck
-	accountDB := base.CreateAccountsDB()
+	userAPI := base.UserAPIClient()
-	deviceDB := base.CreateDeviceDB()
+	rsAPI := base.RoomserverHTTPClient()
-	federation := base.CreateFederationClient()
-	alias, _, query := base.CreateHTTPRoomserverAPIs()
-	cache := transactions.New()
 
-	appservice.SetupAppServiceAPIComponent(
+	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-		base, accountDB, deviceDB, federation, alias, query, cache,
+	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
-	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI))
 

cmd/dendrite-client-api-server/main.go

@@ -16,33 +16,29 @@ package main
 
 import (
 	"github.com/matrix-org/dendrite/clientapi"
-	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/common/keydb"
+	"github.com/matrix-org/dendrite/internal/transactions"
-	"github.com/matrix-org/dendrite/common/transactions"
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags()
+	cfg := setup.ParseFlags(false)
 
-	base := basecomponent.NewBaseDendrite(cfg, "ClientAPI")
+	base := setup.NewBaseDendrite(cfg, "ClientAPI", true)
 	defer base.Close() // nolint: errcheck
 
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
-	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	asQuery := base.CreateHTTPAppServiceAPIs()
+	asQuery := base.AppserviceHTTPClient()
-	alias, input, query := base.CreateHTTPRoomserverAPIs()
+	rsAPI := base.RoomserverHTTPClient()
-	fedSenderAPI := base.CreateHTTPFederationSenderAPIs()
+	fsAPI := base.FederationSenderHTTPClient()
-	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
+	eduInputAPI := base.EDUServerClient()
+	userAPI := base.UserAPIClient()
 
-	clientapi.SetupClientAPIComponent(
+	clientapi.AddPublicRoutes(
-		base, deviceDB, accountDB, federation, &keyRing,
+		base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation,
-		alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
+		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))

cmd/dendrite-demo-libp2p/main.go

@@ -29,40 +29,26 @@ import (
 	p2phttp "github.com/libp2p/go-libp2p-http"
 	p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery"
 	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/clientapi"
-	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage"
-	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/common/keydb"
-	"github.com/matrix-org/dendrite/common/transactions"
 	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationsender"
-	"github.com/matrix-org/dendrite/mediaapi"
+	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/publicroomsapi"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/syncapi"
+	"github.com/matrix-org/dendrite/serverkeyapi"
+	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/eduserver/cache"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 )
 
 func createKeyDB(
 	base *P2PDendrite,
-) keydb.Database {
+	db gomatrixserverlib.KeyDatabase,
-	db, err := keydb.NewDatabase(
+) {
-		string(base.Base.Cfg.Database.ServerKey),
-		base.Base.Cfg.Matrix.ServerName,
-		base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
-		base.Base.Cfg.Matrix.KeyID,
-	)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to keys db")
-	}
 	mdns := mDNSListener{
 		host:  base.LibP2P,
 		keydb: db,
@@ -77,7 +63,6 @@ func createKeyDB(
 		panic(err)
 	}
 	serv.RegisterNotifee(&mdns)
-	return db
 }
 
 func createFederationClient(
@@ -95,6 +80,17 @@ func createFederationClient(
 	)
 }
 
+func createClient(
+	base *P2PDendrite,
+) *gomatrixserverlib.Client {
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix",
+		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
+	)
+	return gomatrixserverlib.NewClientWithTransport(tr)
+}
+
 func main() {
 	instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance")
 	instancePort := flag.Int("port", 8080, "the port that the client API will listen on")
@@ -117,6 +113,7 @@ func main() {
 	}
 
 	cfg := config.Dendrite{}
+	cfg.SetDefaults()
 	cfg.Matrix.ServerName = "p2p"
 	cfg.Matrix.PrivateKey = privKey
 	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
@@ -124,7 +121,6 @@ func main() {
 	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
 	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
 	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
-	cfg.Kafka.Topics.UserUpdates = "userUpdates"
 	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
 	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
 	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
@@ -144,44 +140,67 @@ func main() {
 
 	accountDB := base.Base.CreateAccountsDB()
 	deviceDB := base.Base.CreateDeviceDB()
-	keyDB := createKeyDB(base)
 	federation := createFederationClient(base)
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil)
 
-	alias, input, query := roomserver.SetupRoomServerComponent(&base.Base)
+	serverKeyAPI := serverkeyapi.NewInternalAPI(
-	eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New())
+		base.Base.Cfg, federation, base.Base.Caches,
-	asQuery := appservice.SetupAppServiceAPIComponent(
+	)
-		&base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(),
+	keyRing := serverKeyAPI.KeyRing()
+	createKeyDB(
+		base, serverKeyAPI,
 	)
-	fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query)
 
-	clientapi.SetupClientAPIComponent(
+	rsAPI := roomserver.NewInternalAPI(
-		&base.Base, deviceDB, accountDB,
+		&base.Base, keyRing, federation,
-		federation, &keyRing, alias, input, query,
-		eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
 	)
-	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
+	eduInputAPI := eduserver.NewInternalAPI(
-	federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
+		&base.Base, cache.New(), userAPI,
-	mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB)
+	)
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub)
+	asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
+	fsAPI := federationsender.NewInternalAPI(
+		&base.Base, federation, rsAPI, keyRing,
+	)
+	rsAPI.SetFederationSenderAPI(fsAPI)
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, query, federation, nil) // Check this later
-	syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, query, federation, &cfg)
 
-	httpHandler := common.WrapHandlerInCORS(base.Base.APIMux)
+	monolith := setup.Monolith{
+		Config:        base.Base.Cfg,
+		AccountDB:     accountDB,
+		DeviceDB:      deviceDB,
+		Client:        createClient(base),
+		FedClient:     federation,
+		KeyRing:       keyRing,
+		KafkaConsumer: base.Base.KafkaConsumer,
+		KafkaProducer: base.Base.KafkaProducer,
 
-	// Set up the API endpoints we handle. /metrics is for prometheus, and is
+		AppserviceAPI:       asAPI,
-	// not wrapped by CORS, while everything else is
+		EDUInternalAPI:      eduInputAPI,
-	http.Handle("/metrics", promhttp.Handler())
+		FederationSenderAPI: fsAPI,
-	http.Handle("/", httpHandler)
+		RoomserverAPI:       rsAPI,
+		ServerKeyAPI:        serverKeyAPI,
+		UserAPI:             userAPI,
+
+		PublicRoomsDB: publicRoomsDB,
+	}
+	monolith.AddAllPublicRoutes(base.Base.PublicAPIMux)
+
+	httputil.SetupHTTPAPI(
+		base.Base.BaseMux,
+		base.Base.PublicAPIMux,
+		base.Base.InternalAPIMux,
+		&cfg,
+		base.Base.UseHTTPAPIs,
+	)
 
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {
 		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
 		logrus.Info("Listening on ", httpBindAddr)
-		logrus.Fatal(http.ListenAndServe(httpBindAddr, nil))
+		logrus.Fatal(http.ListenAndServe(httpBindAddr, base.Base.BaseMux))
 	}()
 	// Expose the matrix APIs also via libp2p
 	if base.LibP2P != nil {
@@ -194,7 +213,7 @@ func main() {
 			defer func() {
 				logrus.Fatal(listener.Close())
 			}()
-			logrus.Fatal(http.Serve(listener, nil))
+			logrus.Fatal(http.Serve(listener, base.Base.BaseMux))
 		}()
 	}
 

cmd/dendrite-demo-libp2p/mdnslistener.go

@@ -21,12 +21,11 @@ import (
 
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/matrix-org/dendrite/common/keydb"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type mDNSListener struct {
-	keydb keydb.Database
+	keydb gomatrixserverlib.KeyDatabase
 	host  host.Host
 }
 
@@ -44,7 +43,7 @@ func (n *mDNSListener) HandlePeerFound(p peer.AddrInfo) {
 					KeyID:      "ed25519:p2pdemo",
 				}: {
 					VerifyKey: gomatrixserverlib.VerifyKey{
-						Key: gomatrixserverlib.Base64String(raw),
+						Key: gomatrixserverlib.Base64Bytes(raw),
 					},
 					ValidUntilTS: math.MaxUint64 >> 1,
 					ExpiredTS:    gomatrixserverlib.PublicKeyNotExpired,

cmd/dendrite-demo-libp2p/p2pdendrite.go

@@ -22,7 +22,7 @@ import (
 
 	pstore "github.com/libp2p/go-libp2p-core/peerstore"
 	record "github.com/libp2p/go-libp2p-record"
-	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/internal/setup"
 
 	"github.com/libp2p/go-libp2p"
 	circuit "github.com/libp2p/go-libp2p-circuit"
@@ -34,12 +34,12 @@ import (
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/matrix-org/gomatrixserverlib"
 
-	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
 // P2PDendrite is a Peer-to-Peer variant of BaseDendrite.
 type P2PDendrite struct {
-	Base basecomponent.BaseDendrite
+	Base setup.BaseDendrite
 
 	// Store our libp2p object so that we can make outgoing connections from it
 	// later
@@ -54,7 +54,7 @@ type P2PDendrite struct {
 // The componentName is used for logging purposes, and should be a friendly name
 // of the component running, e.g. SyncAPI.
 func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
-	baseDendrite := basecomponent.NewBaseDendrite(cfg, componentName)
+	baseDendrite := setup.NewBaseDendrite(cfg, componentName, false)
 
 	ctx, cancel := context.WithCancel(context.Background())
 

cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go

@@ -44,8 +44,8 @@ type PublicRoomsServerDatabase struct {
 }
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName)
 	if err != nil {
 		return nil, err
 	}

cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go

@@ -47,8 +47,8 @@ type PublicRoomsServerDatabase struct {
 }
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName)
 	if err != nil {
 		return nil, err
 	}

cmd/dendrite-demo-libp2p/storage/storage.go

@@ -23,39 +23,40 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const schemePostgres = "postgres"
 const schemeFile = "file"
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT) (storage.Database, error) {
+func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (storage.Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht)
+		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
 	}
 	switch uri.Scheme {
 	case schemePostgres:
-		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht)
+		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
 	case schemeFile:
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
+		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
 	default:
-		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht)
+		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
 	}
 }
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub) (storage.Database, error) {
+func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (storage.Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub)
+		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
 	}
 	switch uri.Scheme {
 	case schemePostgres:
-		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub)
+		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
 	case schemeFile:
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
+		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
 	default:
-		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub)
+		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
 	}
 }

cmd/dendrite-demo-yggdrasil/convert/25519.go

@@ -0,0 +1,53 @@
+// Copyright 2019 Google LLC
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+//
+// Original code from https://github.com/FiloSottile/age/blob/bbab440e198a4d67ba78591176c7853e62d29e04/internal/age/ssh.go
+
+package convert
+
+import (
+	"crypto/ed25519"
+	"crypto/sha512"
+	"math/big"
+
+	"golang.org/x/crypto/curve25519"
+)
+
+var curve25519P, _ = new(big.Int).SetString("57896044618658097711785492504343953926634992332820282019728792003956564819949", 10)
+
+func Ed25519PrivateKeyToCurve25519(pk ed25519.PrivateKey) []byte {
+	h := sha512.New()
+	_, _ = h.Write(pk.Seed())
+	out := h.Sum(nil)
+	return out[:curve25519.ScalarSize]
+}
+
+func Ed25519PublicKeyToCurve25519(pk ed25519.PublicKey) []byte {
+	// ed25519.PublicKey is a little endian representation of the y-coordinate,
+	// with the most significant bit set based on the sign of the x-coordinate.
+	bigEndianY := make([]byte, ed25519.PublicKeySize)
+	for i, b := range pk {
+		bigEndianY[ed25519.PublicKeySize-i-1] = b
+	}
+	bigEndianY[0] &= 0b0111_1111
+
+	// The Montgomery u-coordinate is derived through the bilinear map
+	//     u = (1 + y) / (1 - y)
+	// See https://blog.filippo.io/using-ed25519-keys-for-encryption.
+	y := new(big.Int).SetBytes(bigEndianY)
+	denom := big.NewInt(1)
+	denom.ModInverse(denom.Sub(denom, y), curve25519P) // 1 / (1 - y)
+	u := y.Mul(y.Add(y, big.NewInt(1)), denom)
+	u.Mod(u, curve25519P)
+
+	out := make([]byte, curve25519.PointSize)
+	uBytes := u.Bytes()
+	for i, b := range uBytes {
+		out[len(uBytes)-i-1] = b
+	}
+
+	return out
+}

cmd/dendrite-demo-yggdrasil/convert/25519_test.go

@@ -0,0 +1,51 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package convert
+
+import (
+	"bytes"
+	"crypto/ed25519"
+	"encoding/hex"
+	"testing"
+
+	"golang.org/x/crypto/curve25519"
+)
+
+func TestKeyConversion(t *testing.T) {
+	edPub, edPriv, err := ed25519.GenerateKey(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("Signing public:", hex.EncodeToString(edPub))
+	t.Log("Signing private:", hex.EncodeToString(edPriv))
+
+	cuPriv := Ed25519PrivateKeyToCurve25519(edPriv)
+	t.Log("Encryption private:", hex.EncodeToString(cuPriv))
+
+	cuPub := Ed25519PublicKeyToCurve25519(edPub)
+	t.Log("Converted encryption public:", hex.EncodeToString(cuPub))
+
+	var realPub, realPriv [32]byte
+	copy(realPriv[:32], cuPriv[:32])
+	curve25519.ScalarBaseMult(&realPub, &realPriv)
+	t.Log("Scalar-multed encryption public:", hex.EncodeToString(realPub[:]))
+
+	if !bytes.Equal(realPriv[:], cuPriv[:]) {
+		t.Fatal("Private keys should be equal (this means the test is broken)")
+	}
+	if !bytes.Equal(realPub[:], cuPub[:]) {
+		t.Fatal("Public keys should be equal")
+	}
+}

cmd/dendrite-demo-yggdrasil/embed/embed_other.go

@@ -0,0 +1,9 @@
+// +build !riotweb
+
+package embed
+
+import "github.com/gorilla/mux"
+
+func Embed(_ *mux.Router, _ int, _ string) {
+
+}

cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go

@@ -0,0 +1,62 @@
+// +build riotweb
+
+package embed
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/tidwall/sjson"
+)
+
+// From within the Riot Web directory:
+// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_riotweb.go -private -pkg embed .
+
+func Embed(rootMux *mux.Router, listenPort int, serverName string) {
+	url := fmt.Sprintf("http://localhost:%d", listenPort)
+	embeddedFS := _escFS(false)
+	embeddedServ := http.FileServer(embeddedFS)
+
+	rootMux.Handle("/", embeddedServ)
+	rootMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) {
+		configFile, err := embeddedFS.Open("/config.sample.json")
+		if err != nil {
+			w.WriteHeader(500)
+			io.WriteString(w, "Couldn't open the file: "+err.Error())
+			return
+		}
+		configFileInfo, err := configFile.Stat()
+		if err != nil {
+			w.WriteHeader(500)
+			io.WriteString(w, "Couldn't stat the file: "+err.Error())
+			return
+		}
+		buf := make([]byte, configFileInfo.Size())
+		n, err := configFile.Read(buf)
+		if err != nil {
+			w.WriteHeader(500)
+			io.WriteString(w, "Couldn't read the file: "+err.Error())
+			return
+		}
+		if int64(n) != configFileInfo.Size() {
+			w.WriteHeader(500)
+			io.WriteString(w, "The returned file size didn't match what we expected")
+			return
+		}
+		js, _ := sjson.SetBytes(buf, "default_server_config.m\\.homeserver.base_url", url)
+		js, _ = sjson.SetBytes(js, "default_server_config.m\\.homeserver.server_name", serverName)
+		js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Riot %s", serverName))
+		js, _ = sjson.SetBytes(js, "disable_guests", true)
+		js, _ = sjson.SetBytes(js, "disable_3pid_login", true)
+		js, _ = sjson.DeleteBytes(js, "welcomeUserId")
+		_, _ = w.Write(js)
+	})
+
+	fmt.Println("*-------------------------------*")
+	fmt.Println("| This build includes Riot Web! |")
+	fmt.Println("*-------------------------------*")
+	fmt.Println("Point your browser to:", url)
+	fmt.Println()
+}

cmd/dendrite-demo-yggdrasil/main.go

@@ -0,0 +1,171 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/publicroomsapi/storage"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	instanceName = flag.String("name", "dendrite-p2p-ygg", "the name of this P2P demo instance")
+	instancePort = flag.Int("port", 8008, "the port that the client API will listen on")
+	instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to")
+)
+
+// nolint:gocyclo
+func main() {
+	flag.Parse()
+
+	ygg, err := yggconn.Setup(*instanceName, *instancePeer, ".")
+	if err != nil {
+		panic(err)
+	}
+
+	cfg := &config.Dendrite{}
+	cfg.SetDefaults()
+	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
+	cfg.Matrix.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Kafka.UseNaffka = true
+	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
+	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
+	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
+	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName))
+	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
+	if err = cfg.Derive(); err != nil {
+		panic(err)
+	}
+
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	deviceDB := base.CreateDeviceDB()
+	federation := ygg.CreateFederationClient(base)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil)
+
+	rsComponent := roomserver.NewInternalAPI(
+		base, keyRing, federation,
+	)
+	rsAPI := rsComponent
+
+	eduInputAPI := eduserver.NewInternalAPI(
+		base, cache.New(), userAPI,
+	)
+
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing,
+	)
+
+	rsComponent.SetFederationSenderAPI(fsAPI)
+
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to public rooms db")
+	}
+
+	embed.Embed(base.BaseMux, *instancePort, "Yggdrasil Demo")
+
+	monolith := setup.Monolith{
+		Config:        base.Cfg,
+		AccountDB:     accountDB,
+		DeviceDB:      deviceDB,
+		Client:        ygg.CreateClient(base),
+		FedClient:     federation,
+		KeyRing:       keyRing,
+		KafkaConsumer: base.KafkaConsumer,
+		KafkaProducer: base.KafkaProducer,
+
+		AppserviceAPI:       asAPI,
+		EDUInternalAPI:      eduInputAPI,
+		FederationSenderAPI: fsAPI,
+		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
+		//ServerKeyAPI:        serverKeyAPI,
+
+		PublicRoomsDB: publicRoomsDB,
+	}
+	monolith.AddAllPublicRoutes(base.PublicAPIMux)
+
+	httputil.SetupHTTPAPI(
+		base.BaseMux,
+		base.PublicAPIMux,
+		base.InternalAPIMux,
+		cfg,
+		base.UseHTTPAPIs,
+	)
+
+	// Build both ends of a HTTP multiplex.
+	httpServer := &http.Server{
+		Addr:         ":0",
+		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+		ReadTimeout:  15 * time.Second,
+		WriteTimeout: 45 * time.Second,
+		IdleTimeout:  60 * time.Second,
+		BaseContext: func(_ net.Listener) context.Context {
+			return context.Background()
+		},
+		Handler: base.BaseMux,
+	}
+
+	go func() {
+		logrus.Info("Listening on ", ygg.DerivedServerName())
+		logrus.Fatal(httpServer.Serve(ygg))
+	}()
+	go func() {
+		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
+		logrus.Info("Listening on ", httpBindAddr)
+		logrus.Fatal(http.ListenAndServe(httpBindAddr, base.BaseMux))
+	}()
+
+	select {}
+}

cmd/dendrite-demo-yggdrasil/signing/fetcher.go

@@ -0,0 +1,69 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package signing
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const KeyID = "ed25519:dendrite-demo-yggdrasil"
+
+type YggdrasilKeys struct {
+}
+
+func (f *YggdrasilKeys) KeyRing() *gomatrixserverlib.KeyRing {
+	return &gomatrixserverlib.KeyRing{
+		KeyDatabase: f,
+	}
+}
+
+func (f *YggdrasilKeys) FetchKeys(
+	ctx context.Context,
+	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
+) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) {
+	res := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult)
+	for req := range requests {
+		if req.KeyID != KeyID {
+			return nil, fmt.Errorf("FetchKeys: cannot fetch key with ID %s, should be %s", req.KeyID, KeyID)
+		}
+
+		hexkey, err := hex.DecodeString(string(req.ServerName))
+		if err != nil {
+			return nil, fmt.Errorf("FetchKeys: can't decode server name %q: %w", req.ServerName, err)
+		}
+
+		res[req] = gomatrixserverlib.PublicKeyLookupResult{
+			VerifyKey: gomatrixserverlib.VerifyKey{
+				Key: hexkey,
+			},
+			ExpiredTS:    gomatrixserverlib.PublicKeyNotExpired,
+			ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(24 * time.Hour * 365)),
+		}
+	}
+	return res, nil
+}
+
+func (f *YggdrasilKeys) FetcherName() string {
+	return "YggdrasilKeys"
+}
+
+func (f *YggdrasilKeys) StoreKeys(ctx context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error {
+	return nil
+}

cmd/dendrite-demo-yggdrasil/yggconn/client.go

@@ -0,0 +1,74 @@
+package yggconn
+
+import (
+	"context"
+	"crypto/ed25519"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func (n *Node) yggdialer(_, address string) (net.Conn, error) {
+	tokens := strings.Split(address, ":")
+	raw, err := hex.DecodeString(tokens[0])
+	if err != nil {
+		return nil, fmt.Errorf("hex.DecodeString: %w", err)
+	}
+	converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw))
+	convhex := hex.EncodeToString(converted)
+	return n.Dial("curve25519", convhex)
+}
+
+func (n *Node) yggdialerctx(ctx context.Context, network, address string) (net.Conn, error) {
+	return n.yggdialer(network, address)
+}
+
+type yggroundtripper struct {
+	inner *http.Transport
+}
+
+func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.URL.Scheme = "http"
+	return y.inner.RoundTrip(req)
+}
+
+func (n *Node) CreateClient(
+	base *setup.BaseDendrite,
+) *gomatrixserverlib.Client {
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix", &yggroundtripper{
+			inner: &http.Transport{
+				ResponseHeaderTimeout: 15 * time.Second,
+				IdleConnTimeout:       60 * time.Second,
+				DialContext:           n.yggdialerctx,
+			},
+		},
+	)
+	return gomatrixserverlib.NewClientWithTransport(tr)
+}
+
+func (n *Node) CreateFederationClient(
+	base *setup.BaseDendrite,
+) *gomatrixserverlib.FederationClient {
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix", &yggroundtripper{
+			inner: &http.Transport{
+				ResponseHeaderTimeout: 15 * time.Second,
+				IdleConnTimeout:       60 * time.Second,
+				DialContext:           n.yggdialerctx,
+			},
+		},
+	)
+	return gomatrixserverlib.NewFederationClientWithTransport(
+		base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr,
+	)
+}

cmd/dendrite-demo-yggdrasil/yggconn/node.go

@@ -0,0 +1,176 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package yggconn
+
+import (
+	"context"
+	"crypto/ed25519"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
+
+	"github.com/libp2p/go-yamux"
+	yggdrasiladmin "github.com/yggdrasil-network/yggdrasil-go/src/admin"
+	yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config"
+	yggdrasilmulticast "github.com/yggdrasil-network/yggdrasil-go/src/multicast"
+	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
+
+	gologme "github.com/gologme/log"
+)
+
+type Node struct {
+	core      *yggdrasil.Core
+	config    *yggdrasilconfig.NodeConfig
+	state     *yggdrasilconfig.NodeState
+	admin     *yggdrasiladmin.AdminSocket
+	multicast *yggdrasilmulticast.Multicast
+	log       *gologme.Logger
+	listener  *yggdrasil.Listener
+	dialer    *yggdrasil.Dialer
+	sessions  sync.Map // string -> yamux.Session
+	incoming  chan *yamux.Stream
+}
+
+func (n *Node) Dialer(_, address string) (net.Conn, error) {
+	tokens := strings.Split(address, ":")
+	raw, err := hex.DecodeString(tokens[0])
+	if err != nil {
+		return nil, fmt.Errorf("hex.DecodeString: %w", err)
+	}
+	converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw))
+	convhex := hex.EncodeToString(converted)
+	return n.Dial("curve25519", convhex)
+}
+
+func (n *Node) DialerContext(ctx context.Context, network, address string) (net.Conn, error) {
+	return n.Dialer(network, address)
+}
+
+// nolint:gocyclo
+func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) {
+	n := &Node{
+		core:      &yggdrasil.Core{},
+		config:    yggdrasilconfig.GenerateConfig(),
+		admin:     &yggdrasiladmin.AdminSocket{},
+		multicast: &yggdrasilmulticast.Multicast{},
+		log:       gologme.New(os.Stdout, "YGG ", log.Flags()),
+		incoming:  make(chan *yamux.Stream),
+	}
+
+	yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
+	if _, err := os.Stat(yggfile); !os.IsNotExist(err) {
+		yggconf, e := ioutil.ReadFile(yggfile)
+		if e != nil {
+			panic(err)
+		}
+		if err := json.Unmarshal([]byte(yggconf), &n.config); err != nil {
+			panic(err)
+		}
+	} else {
+		n.config.AdminListen = "none" // fmt.Sprintf("unix://%s/%s-yggdrasil.sock", storageDirectory, instanceName)
+		n.config.MulticastInterfaces = []string{".*"}
+		n.config.EncryptionPrivateKey = hex.EncodeToString(n.EncryptionPrivateKey())
+		n.config.EncryptionPublicKey = hex.EncodeToString(n.EncryptionPublicKey())
+
+		j, err := json.MarshalIndent(n.config, "", "  ")
+		if err != nil {
+			panic(err)
+		}
+		if e := ioutil.WriteFile(yggfile, j, 0600); e != nil {
+			n.log.Printf("Couldn't write private key to file '%s': %s\n", yggfile, e)
+		}
+	}
+
+	var err error
+	n.log.EnableLevel("error")
+	n.log.EnableLevel("warn")
+	n.log.EnableLevel("info")
+	n.state, err = n.core.Start(n.config, n.log)
+	if err != nil {
+		panic(err)
+	}
+	if instancePeer != "" {
+		if err = n.core.AddPeer(instancePeer, ""); err != nil {
+			panic(err)
+		}
+	}
+	/*
+		if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil {
+			panic(err)
+		}
+		if err = n.admin.Start(); err != nil {
+			panic(err)
+		}
+	*/
+	if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil {
+		panic(err)
+	}
+	if err = n.multicast.Start(); err != nil {
+		panic(err)
+	}
+	//n.admin.SetupAdminHandlers(n.admin)
+	//n.multicast.SetupAdminHandlers(n.admin)
+	n.listener, err = n.core.ConnListen()
+	if err != nil {
+		panic(err)
+	}
+	n.dialer, err = n.core.ConnDialer()
+	if err != nil {
+		panic(err)
+	}
+
+	n.log.Println("Public curve25519:", n.core.EncryptionPublicKey())
+	n.log.Println("Public ed25519:", n.core.SigningPublicKey())
+
+	go n.listenFromYgg()
+
+	return n, nil
+}
+
+func (n *Node) DerivedServerName() string {
+	return hex.EncodeToString(n.SigningPublicKey())
+}
+
+func (n *Node) DerivedSessionName() string {
+	return hex.EncodeToString(n.EncryptionPublicKey())
+}
+
+func (n *Node) EncryptionPublicKey() []byte {
+	edkey := n.SigningPublicKey()
+	return convert.Ed25519PublicKeyToCurve25519(edkey)
+}
+
+func (n *Node) EncryptionPrivateKey() []byte {
+	edkey := n.SigningPrivateKey()
+	return convert.Ed25519PrivateKeyToCurve25519(edkey)
+}
+
+func (n *Node) SigningPublicKey() ed25519.PublicKey {
+	pubBytes, _ := hex.DecodeString(n.config.SigningPublicKey)
+	return ed25519.PublicKey(pubBytes)
+}
+
+func (n *Node) SigningPrivateKey() ed25519.PrivateKey {
+	privBytes, _ := hex.DecodeString(n.config.SigningPrivateKey)
+	return ed25519.PrivateKey(privBytes)
+}

cmd/dendrite-demo-yggdrasil/yggconn/session.go

@@ -0,0 +1,124 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package yggconn
+
+import (
+	"context"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/libp2p/go-yamux"
+)
+
+func (n *Node) yamuxConfig() *yamux.Config {
+	cfg := yamux.DefaultConfig()
+	cfg.EnableKeepAlive = false
+	cfg.ConnectionWriteTimeout = time.Second * 15
+	cfg.MaxMessageSize = 65535
+	cfg.ReadBufSize = 655350
+	return cfg
+}
+
+func (n *Node) listenFromYgg() {
+	for {
+		conn, err := n.listener.Accept()
+		if err != nil {
+			n.log.Println("n.listener.Accept:", err)
+			return
+		}
+		var session *yamux.Session
+		// If the remote address is lower than ours then we'll be the
+		// server. Otherwse we'll be the client.
+		if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 {
+			session, err = yamux.Server(conn, n.yamuxConfig())
+		} else {
+			session, err = yamux.Client(conn, n.yamuxConfig())
+		}
+		if err != nil {
+			return
+		}
+		go n.listenFromYggConn(session)
+	}
+}
+
+func (n *Node) listenFromYggConn(session *yamux.Session) {
+	n.sessions.Store(session.RemoteAddr().String(), session)
+	defer n.sessions.Delete(session.RemoteAddr())
+	defer func() {
+		if err := session.Close(); err != nil {
+			n.log.Println("session.Close:", err)
+		}
+	}()
+
+	for {
+		st, err := session.AcceptStream()
+		if err != nil {
+			n.log.Println("session.AcceptStream:", err)
+			return
+		}
+		n.incoming <- st
+	}
+}
+
+// Implements net.Listener
+func (n *Node) Accept() (net.Conn, error) {
+	return <-n.incoming, nil
+}
+
+// Implements net.Listener
+func (n *Node) Close() error {
+	return n.listener.Close()
+}
+
+// Implements net.Listener
+func (n *Node) Addr() net.Addr {
+	return n.listener.Addr()
+}
+
+// Implements http.Transport.Dial
+func (n *Node) Dial(network, address string) (net.Conn, error) {
+	return n.DialContext(context.TODO(), network, address)
+}
+
+// Implements http.Transport.DialContext
+func (n *Node) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	s, ok1 := n.sessions.Load(address)
+	session, ok2 := s.(*yamux.Session)
+	if !ok1 || !ok2 || (ok1 && ok2 && session.IsClosed()) {
+		conn, err := n.dialer.DialContext(ctx, network, address)
+		if err != nil {
+			n.log.Println("n.dialer.DialContext:", err)
+			return nil, err
+		}
+		// If the remote address is lower than ours then we will be the
+		// server. Otherwise we'll be the client.
+		if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 {
+			session, err = yamux.Server(conn, n.yamuxConfig())
+		} else {
+			session, err = yamux.Client(conn, n.yamuxConfig())
+		}
+		if err != nil {
+			return nil, err
+		}
+		go n.listenFromYggConn(session)
+	}
+	st, err := session.OpenStream()
+	if err != nil {
+		n.log.Println("session.OpenStream:", err)
+		return nil, err
+	}
+	return st, nil
+}

cmd/dendrite-edu-server/main.go

@@ -15,22 +15,23 @@ package main
 import (
 	_ "net/http/pprof"
 
-	"github.com/matrix-org/dendrite/common/basecomponent"
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/sirupsen/logrus"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags()
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "EDUServerAPI")
+	base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true)
 	defer func() {
 		if err := base.Close(); err != nil {
 			logrus.WithError(err).Warn("BaseDendrite close failed")
 		}
 	}()
 
-	eduserver.SetupEDUServerComponent(base, cache.New())
+	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
+	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer))
 

cmd/dendrite-federation-api-server/main.go

@@ -15,34 +15,25 @@
 package main
 
 import (
-	"github.com/matrix-org/dendrite/clientapi/producers"
-	"github.com/matrix-org/dendrite/common/basecomponent"
-	"github.com/matrix-org/dendrite/common/keydb"
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/setup"
 )
 
 func main() {
-	cfg := basecomponent.ParseFlags()
+	cfg := setup.ParseFlags(false)
-	base := basecomponent.NewBaseDendrite(cfg, "FederationAPI")
+	base := setup.NewBaseDendrite(cfg, "FederationAPI", true)
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
+	userAPI := base.UserAPIClient()
-	deviceDB := base.CreateDeviceDB()
-	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
-	federationSender := base.CreateHTTPFederationSenderAPIs()
+	serverKeyAPI := base.ServerKeyAPIClient()
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
+	keyRing := serverKeyAPI.KeyRing()
+	fsAPI := base.FederationSenderHTTPClient()
+	rsAPI := base.RoomserverHTTPClient()
 
-	alias, input, query := base.CreateHTTPRoomserverAPIs()
+	federationapi.AddPublicRoutes(
-	asQuery := base.CreateHTTPAppServiceAPIs()
+		base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing,
-	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
+		rsAPI, fsAPI, base.EDUServerClient(),
-	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-
-	federationapi.SetupFederationAPIComponent(
-		base, accountDB, deviceDB, federation, &keyRing,
-		alias, input, query, asQuery, federationSender, eduProducer,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))