Merge branch 'master' into neilalexander/gmestate

appservice/api/query.go

@@ -22,8 +22,8 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 

appservice/appservice.go

@@ -16,7 +16,6 @@ package appservice
 
 import (
 	"context"
-	"errors"
 	"net/http"
 	"sync"
 	"time"
@@ -29,12 +28,10 @@ import (
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/appservice/workers"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
 )
 
@@ -47,8 +44,7 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQuer
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
 	base *setup.BaseDendrite,
-	accountsDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceQueryAPI {
 	// Create a connection to the appservice postgres DB
@@ -70,7 +66,7 @@ func NewInternalAPI(
 		workerStates[i] = ws
 
 		// Create bot account for this AS if it doesn't already exist
-		if err = generateAppServiceAccount(accountsDB, deviceDB, appservice); err != nil {
+		if err = generateAppServiceAccount(userAPI, appservice); err != nil {
 			logrus.WithFields(logrus.Fields{
 				"appservice": appservice.ID,
 			}).WithError(err).Panicf("failed to generate bot account for appservice")
@@ -90,7 +86,7 @@ func NewInternalAPI(
 	// We can't add ASes at runtime so this is safe to do.
 	if len(workerStates) > 0 {
 		consumer := consumers.NewOutputRoomEventConsumer(
-			base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB,
+			base.Cfg, base.KafkaConsumer, appserviceDB,
 			rsAPI, workerStates,
 		)
 		if err := consumer.Start(); err != nil {
@@ -109,22 +105,25 @@ func NewInternalAPI(
 // `sender_localpart` field of each application service if it doesn't
 // exist already
 func generateAppServiceAccount(
-	accountsDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 	as config.ApplicationService,
 ) error {
-	ctx := context.Background()
-
-	// Create an account for the application service
-	_, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID)
+	var accRes userapi.PerformAccountCreationResponse
+	err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{
+		AccountType:  userapi.AccountTypeUser,
+		Localpart:    as.SenderLocalpart,
+		AppServiceID: as.ID,
+		OnConflict:   userapi.ConflictUpdate,
+	}, &accRes)
 	if err != nil {
-		if errors.Is(err, sqlutil.ErrUserExists) { // This account already exists
-			return nil
-		}
 		return err
 	}
-
-	// Create a dummy device with a dummy token for the application service
-	_, err = deviceDB.CreateDevice(ctx, as.SenderLocalpart, nil, as.ASToken, &as.SenderLocalpart)
+	var devRes userapi.PerformDeviceCreationResponse
+	err = userAPI.PerformDeviceCreation(context.Background(), &userapi.PerformDeviceCreationRequest{
+		Localpart:         as.SenderLocalpart,
+		AccessToken:       as.ASToken,
+		DeviceID:          &as.SenderLocalpart,
+		DeviceDisplayName: &as.SenderLocalpart,
+	}, &devRes)
 	return err
 }

appservice/consumers/roomserver.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -33,7 +32,6 @@ import (
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
 	roomServerConsumer *internal.ContinualConsumer
-	db                 accounts.Database
 	asDB               storage.Database
 	rsAPI              api.RoomserverInternalAPI
 	serverName         string
@@ -45,7 +43,6 @@ type OutputRoomEventConsumer struct {
 func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
-	store accounts.Database,
 	appserviceDB storage.Database,
 	rsAPI api.RoomserverInternalAPI,
 	workerStates []types.ApplicationServiceWorkerState,
@@ -53,11 +50,10 @@ func NewOutputRoomEventConsumer(
 	consumer := internal.ContinualConsumer{
 		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
 		Consumer:       kafkaConsumer,
-		PartitionStore: store,
+		PartitionStore: appserviceDB,
 	}
 	s := &OutputRoomEventConsumer{
 		roomServerConsumer: &consumer,
-		db:                 store,
 		asDB:               appserviceDB,
 		rsAPI:              rsAPI,
 		serverName:         string(cfg.Matrix.ServerName),

appservice/storage/interface.go

@@ -17,10 +17,12 @@ package storage
 import (
 	"context"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
+	internal.PartitionStorer
 	StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.HeaderedEvent) error
 	GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error)
 	CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error)

appservice/storage/postgres/storage.go

@@ -27,6 +27,7 @@ import (
 
 // Database stores events intended to be later sent to application services
 type Database struct {
+	sqlutil.PartitionOffsetStatements
 	events eventsStatements
 	txnID  txnStatements
 	db     *sql.DB
@@ -42,6 +43,9 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Dat
 	if err = result.prepare(); err != nil {
 		return nil, err
 	}
+	if err = result.PartitionOffsetStatements.Prepare(result.db, "appservice"); err != nil {
+		return nil, err
+	}
 	return &result, nil
 }
 

appservice/storage/sqlite3/storage.go

@@ -27,6 +27,7 @@ import (
 
 // Database stores events intended to be later sent to application services
 type Database struct {
+	sqlutil.PartitionOffsetStatements
 	events eventsStatements
 	txnID  txnStatements
 	db     *sql.DB
@@ -46,6 +47,9 @@ func NewDatabase(dataSourceName string) (*Database, error) {
 	if err = result.prepare(); err != nil {
 		return nil, err
 	}
+	if err = result.PartitionOffsetStatements.Prepare(result.db, "appservice"); err != nil {
+		return nil, err
+	}
 	return &result, nil
 }
 

are-we-synapse-yet.list

@@ -97,8 +97,8 @@ rst PUT power_levels should not explode if the old power levels were empty
 rst Both GET and PUT work
 rct POST /rooms/:room_id/receipt can create receipts 
 red POST /rooms/:room_id/read_markers can create read marker
-med POST /media/v1/upload can create an upload 
-med GET /media/v1/download can fetch the value again 
+med POST /media/r0/upload can create an upload 
+med GET /media/r0/download can fetch the value again 
 cap GET /capabilities is present and well formed for registered user 
 cap GET /r0/capabilities is not public
 reg Register with a recaptcha 
@@ -853,4 +853,8 @@ fme Outbound federation will ignore a missing event with bad JSON for room versi
 fbk Outbound federation rejects backfill containing invalid JSON for events in room version 6
 jso Invalid JSON integers
 jso Invalid JSON floats
-jso Invalid JSON special values
+jso Invalid JSON special values
+inv Can invite users to invite-only rooms (2 subtests)
+plv setting 'm.room.name' respects room powerlevel (2 subtests)
+psh Messages that notify from another user increment notification_count
+psh Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count

are-we-synapse-yet.py

@@ -33,6 +33,7 @@ import sys
 
 test_mappings = {
     "nsp": "Non-Spec API",
+    "unk": "Unknown API (no group specified)",
     "f": "Federation", # flag to mark test involves federation
 
     "federation_apis": {
@@ -158,6 +159,8 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
     total_tests = 0
     for gid, tests in gid_to_tests.items():
         group_total = len(tests)
+        if group_total == 0:
+            continue
         group_passing = 0
         test_names_and_marks = []
         for name, passing in tests.items():
@@ -214,7 +217,8 @@ def main(results_tap_path, verbose):
             # }
         },
         "nonspec": {
-            "nsp": {}
+            "nsp": {},
+            "unk": {}
         },
     }
     with open(results_tap_path, "r") as f:
@@ -225,7 +229,7 @@ def main(results_tap_path, verbose):
             name = test_result["name"]
             group_id = test_name_to_group_id.get(name)
             if not group_id:
-                raise Exception("The test '%s' doesn't have a group" % (name,))
+                summary["nonspec"]["unk"][name] = test_result["ok"]
             if group_id == "nsp":
                 summary["nonspec"]["nsp"][name] = test_result["ok"]
             elif group_id in test_mappings["federation_apis"]:

build/docker/config/dendrite-config.yaml

@@ -117,6 +117,8 @@ listen:
     federation_sender: "federation_sender:7776"
     edu_server: "edu_server:7777"
     key_server: "key_server:7779"
+    user_api: "user_api:7780"
+    appservice_api: "appservice_api:7781"
 
 # The configuration for tracing the dendrite components.
 tracing:

build/docker/docker-compose.polylith.yml

@@ -152,6 +152,31 @@ services:
     networks:
       - internal
 
+  user_api:
+    hostname: user_api
+    image: matrixdotorg/dendrite:userapi
+    command: [
+        "--config=dendrite.yaml"
+    ]
+    volumes:
+        - ./config:/etc/dendrite
+    networks:
+        - internal
+
+  appservice_api:
+    hostname: appservice_api
+    image: matrixdotorg/dendrite:appservice
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+    depends_on:
+      - room_server
+      - user_api
+
 networks:
   internal:
     attachable: true

build/docker/images-build.sh

@@ -6,6 +6,7 @@ docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest .
 
 docker build -t matrixdotorg/dendrite:monolith          --build-arg component=dendrite-monolith-server          -f build/docker/Dockerfile.component .
 
+docker build -t matrixdotorg/dendrite:appservice        --build-arg component=dendrite-appservice-server        -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:clientapi         --build-arg component=dendrite-client-api-server        -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:clientproxy       --build-arg component=client-api-proxy                  -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:eduserver         --build-arg component=dendrite-edu-server               -f build/docker/Dockerfile.component .
@@ -18,3 +19,4 @@ docker build -t matrixdotorg/dendrite:publicroomsapi    --build-arg component=de
 docker build -t matrixdotorg/dendrite:roomserver        --build-arg component=dendrite-room-server              -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:syncapi           --build-arg component=dendrite-sync-api-server          -f build/docker/Dockerfile.component .
 docker build -t matrixdotorg/dendrite:serverkeyapi      --build-arg component=dendrite-server-key-api-server    -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:userapi           --build-arg component=dendrite-user-api-server          -f build/docker/Dockerfile.component .

build/docker/images-pull.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+docker pull matrixdotorg/dendrite:monolith
+
+docker pull matrixdotorg/dendrite:appservice
+docker pull matrixdotorg/dendrite:clientapi
+docker pull matrixdotorg/dendrite:clientproxy
+docker pull matrixdotorg/dendrite:eduserver
+docker pull matrixdotorg/dendrite:federationapi
+docker pull matrixdotorg/dendrite:federationsender
+docker pull matrixdotorg/dendrite:federationproxy
+docker pull matrixdotorg/dendrite:keyserver
+docker pull matrixdotorg/dendrite:mediaapi
+docker pull matrixdotorg/dendrite:publicroomsapi
+docker pull matrixdotorg/dendrite:roomserver
+docker pull matrixdotorg/dendrite:syncapi
+docker pull matrixdotorg/dendrite:userapi

build/docker/images-push.sh

@@ -2,6 +2,7 @@
 
 docker push matrixdotorg/dendrite:monolith
 
+docker push matrixdotorg/dendrite:appservice
 docker push matrixdotorg/dendrite:clientapi
 docker push matrixdotorg/dendrite:clientproxy
 docker push matrixdotorg/dendrite:eduserver
@@ -13,3 +14,5 @@ docker push matrixdotorg/dendrite:mediaapi
 docker push matrixdotorg/dendrite:publicroomsapi
 docker push matrixdotorg/dendrite:roomserver
 docker push matrixdotorg/dendrite:syncapi
+docker push matrixdotorg/dendrite:serverkeyapi
+docker push matrixdotorg/dendrite:userapi

build/gobind/build.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+gomobile bind -v \
+    -ldflags "-X $github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=riot-ios-p2p" \
+    -target ios \
+    github.com/matrix-org/dendrite/build/gobind

build/gobind/monolith.go

@@ -0,0 +1,161 @@
+package gobind
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/publicroomsapi/storage"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+type DendriteMonolith struct {
+	StorageDirectory string
+	listener         net.Listener
+}
+
+func (m *DendriteMonolith) BaseURL() string {
+	return fmt.Sprintf("http://%s", m.listener.Addr().String())
+}
+
+func (m *DendriteMonolith) Start() {
+	logger := logrus.Logger{
+		Out: BindLogger{},
+	}
+	logrus.SetOutput(BindLogger{})
+
+	var err error
+	m.listener, err = net.Listen("tcp", "localhost:65432")
+	if err != nil {
+		panic(err)
+	}
+
+	ygg, err := yggconn.Setup("dendrite", "", m.StorageDirectory)
+	if err != nil {
+		panic(err)
+	}
+
+	cfg := &config.Dendrite{}
+	cfg.SetDefaults()
+	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
+	cfg.Matrix.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Kafka.UseNaffka = true
+	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput"
+	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory))
+	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory))
+	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory))
+	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory))
+	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory))
+	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory))
+	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory))
+	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory))
+	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory))
+	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory))
+	if err = cfg.Derive(); err != nil {
+		panic(err)
+	}
+
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	deviceDB := base.CreateDeviceDB()
+	federation := ygg.CreateFederationClient(base)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices)
+
+	rsAPI := roomserver.NewInternalAPI(
+		base, keyRing, federation,
+	)
+
+	eduInputAPI := eduserver.NewInternalAPI(
+		base, cache.New(), userAPI,
+	)
+
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing,
+	)
+
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this dependency
+	rsAPI.SetFederationSenderAPI(fsAPI)
+
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to public rooms db")
+	}
+
+	monolith := setup.Monolith{
+		Config:        base.Cfg,
+		AccountDB:     accountDB,
+		DeviceDB:      deviceDB,
+		Client:        ygg.CreateClient(base),
+		FedClient:     federation,
+		KeyRing:       keyRing,
+		KafkaConsumer: base.KafkaConsumer,
+		KafkaProducer: base.KafkaProducer,
+
+		AppserviceAPI:       asAPI,
+		EDUInternalAPI:      eduInputAPI,
+		FederationSenderAPI: fsAPI,
+		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
+		//ServerKeyAPI:        serverKeyAPI,
+
+		PublicRoomsDB: publicRoomsDB,
+	}
+	monolith.AddAllPublicRoutes(base.PublicAPIMux)
+
+	httputil.SetupHTTPAPI(
+		base.BaseMux,
+		base.PublicAPIMux,
+		base.InternalAPIMux,
+		cfg,
+		base.UseHTTPAPIs,
+	)
+
+	// Build both ends of a HTTP multiplex.
+	httpServer := &http.Server{
+		Addr:         ":0",
+		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+		ReadTimeout:  15 * time.Second,
+		WriteTimeout: 45 * time.Second,
+		IdleTimeout:  60 * time.Second,
+		BaseContext: func(_ net.Listener) context.Context {
+			return context.Background()
+		},
+		Handler: base.BaseMux,
+	}
+
+	go func() {
+		logger.Info("Listening on ", ygg.DerivedServerName())
+		logger.Fatal(httpServer.Serve(ygg))
+	}()
+	go func() {
+		logger.Info("Listening on ", m.BaseURL())
+		logger.Fatal(httpServer.Serve(m.listener))
+	}()
+}

build/gobind/platform_ios.go

@@ -0,0 +1,25 @@
+// +build ios
+
+package gobind
+
+/*
+#cgo CFLAGS: -x objective-c
+#cgo LDFLAGS: -framework Foundation
+#import <Foundation/Foundation.h>
+void Log(const char *text) {
+  NSString *nss = [NSString stringWithUTF8String:text];
+  NSLog(@"%@", nss);
+}
+*/
+import "C"
+import "unsafe"
+
+type BindLogger struct {
+}
+
+func (nsl BindLogger) Write(p []byte) (n int, err error) {
+	p = append(p, 0)
+	cstr := (*C.char)(unsafe.Pointer(&p[0]))
+	C.Log(cstr)
+	return len(p), nil
+}

build/gobind/platform_other.go

@@ -0,0 +1,12 @@
+// +build !ios
+
+package gobind
+
+import "log"
+
+type BindLogger struct{}
+
+func (nsl BindLogger) Write(p []byte) (n int, err error) {
+	log.Println(string(p))
+	return len(p), nil
+}

build/scripts/build-test-lint.sh

@@ -10,7 +10,7 @@ set -eu
 echo "Checking that it builds..."
 go build ./cmd/...
 
-./scripts/find-lint.sh
+./build/scripts/find-lint.sh
 
 echo "Testing..."
 go test -v ./...

clientapi/auth/auth.go

@@ -18,17 +18,13 @@ package auth
 import (
 	"context"
 	"crypto/rand"
-	"database/sql"
 	"encoding/base64"
 	"fmt"
 	"net/http"
 	"strings"
 
-	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/userutil"
-	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
@@ -39,21 +35,13 @@ var tokenByteLength = 32
 // DeviceDatabase represents a device database.
 type DeviceDatabase interface {
 	// Look up the device matching the given access token.
-	GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error)
+	GetDeviceByAccessToken(ctx context.Context, token string) (*api.Device, error)
 }
 
 // AccountDatabase represents an account database.
 type AccountDatabase interface {
 	// Look up the account matching the given localpart.
-	GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error)
-}
-
-// Data contains information required to authenticate a request.
-type Data struct {
-	AccountDB AccountDatabase
-	DeviceDB  DeviceDatabase
-	// AppServices is the list of all registered AS
-	AppServices []config.ApplicationService
+	GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error)
 }
 
 // VerifyUserFromRequest authenticates the HTTP request,
@@ -62,8 +50,8 @@ type Data struct {
 // Note: For an AS user, AS dummy device is returned.
 // On failure returns an JSON error response which can be sent to the client.
 func VerifyUserFromRequest(
-	req *http.Request, data Data,
-) (*authtypes.Device, *util.JSONResponse) {
+	req *http.Request, userAPI api.UserInternalAPI,
+) (*api.Device, *util.JSONResponse) {
 	// Try to find the Application Service user
 	token, err := ExtractAccessToken(req)
 	if err != nil {
@@ -72,105 +60,31 @@ func VerifyUserFromRequest(
 			JSON: jsonerror.MissingToken(err.Error()),
 		}
 	}
-
-	// Search for app service with given access_token
-	var appService *config.ApplicationService
-	for _, as := range data.AppServices {
-		if as.ASToken == token {
-			appService = &as
-			break
-		}
+	var res api.QueryAccessTokenResponse
+	err = userAPI.QueryAccessToken(req.Context(), &api.QueryAccessTokenRequest{
+		AccessToken:      token,
+		AppServiceUserID: req.URL.Query().Get("user_id"),
+	}, &res)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccessToken failed")
+		jsonErr := jsonerror.InternalServerError()
+		return nil, &jsonErr
 	}
-
-	if appService != nil {
-		// Create a dummy device for AS user
-		dev := authtypes.Device{
-			// Use AS dummy device ID
-			ID: types.AppServiceDeviceID,
-			// AS dummy device has AS's token.
-			AccessToken: token,
-		}
-
-		userID := req.URL.Query().Get("user_id")
-		localpart, err := userutil.ParseUsernameParam(userID, nil)
-		if err != nil {
-			return nil, &util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.InvalidUsername(err.Error()),
-			}
-		}
-
-		if localpart != "" { // AS is masquerading as another user
-			// Verify that the user is registered
-			account, err := data.AccountDB.GetAccountByLocalpart(req.Context(), localpart)
-			// Verify that account exists & appServiceID matches
-			if err == nil && account.AppServiceID == appService.ID {
-				// Set the userID of dummy device
-				dev.UserID = userID
-				return &dev, nil
-			}
-
+	if res.Err != nil {
+		if forbidden, ok := res.Err.(*api.ErrorForbidden); ok {
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("Application service has not registered this user"),
+				JSON: jsonerror.Forbidden(forbidden.Message),
 			}
 		}
-
-		// AS is not masquerading as any user, so use AS's sender_localpart
-		dev.UserID = appService.SenderLocalpart
-		return &dev, nil
 	}
-
-	// Try to find local user from device database
-	dev, devErr := verifyAccessToken(req, data.DeviceDB)
-	if devErr == nil {
-		return dev, verifyUserParameters(req)
-	}
-
-	return nil, &util.JSONResponse{
-		Code: http.StatusUnauthorized,
-		JSON: jsonerror.UnknownToken("Unrecognized access token"), // nolint: misspell
-	}
-}
-
-// verifyUserParameters ensures that a request coming from a regular user is not
-// using any query parameters reserved for an application service
-func verifyUserParameters(req *http.Request) *util.JSONResponse {
-	if req.URL.Query().Get("ts") != "" {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown("parameter 'ts' not allowed without valid parameter 'access_token'"),
-		}
-	}
-	return nil
-}
-
-// verifyAccessToken verifies that an access token was supplied in the given HTTP request
-// and returns the device it corresponds to. Returns resErr (an error response which can be
-// sent to the client) if the token is invalid or there was a problem querying the database.
-func verifyAccessToken(req *http.Request, deviceDB DeviceDatabase) (device *authtypes.Device, resErr *util.JSONResponse) {
-	token, err := ExtractAccessToken(req)
-	if err != nil {
-		resErr = &util.JSONResponse{
+	if res.Device == nil {
+		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
-			JSON: jsonerror.MissingToken(err.Error()),
-		}
-		return
-	}
-	device, err = deviceDB.GetDeviceByAccessToken(req.Context(), token)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			resErr = &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: jsonerror.UnknownToken("Unknown token"),
-			}
-		} else {
-			util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDeviceByAccessToken failed")
-			jsonErr := jsonerror.InternalServerError()
-			resErr = &jsonErr
+			JSON: jsonerror.UnknownToken("Unknown token"),
 		}
 	}
-	return
+	return res.Device, nil
 }
 
 // GenerateAccessToken creates a new access token. Returns an error if failed to generate

clientapi/clientapi.go

@@ -18,8 +18,6 @@ import (
 	"github.com/Shopify/sarama"
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/consumers"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/routing"
@@ -28,6 +26,9 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
@@ -41,12 +42,12 @@ func AddPublicRoutes(
 	deviceDB devices.Database,
 	accountsDB accounts.Database,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing *gomatrixserverlib.KeyRing,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	eduInputAPI eduServerAPI.EDUServerInputAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	transactionsCache *transactions.Cache,
 	fsAPI federationSenderAPI.FederationSenderInternalAPI,
+	userAPI userapi.UserInternalAPI,
 ) {
 	syncProducer := &producers.SyncAPIProducer{
 		Producer: producer,
@@ -62,7 +63,7 @@ func AddPublicRoutes(
 
 	routing.Setup(
 		router, cfg, eduInputAPI, rsAPI, asAPI,
-		accountsDB, deviceDB, federation, *keyRing,
+		accountsDB, deviceDB, userAPI, federation,
 		syncProducer, transactionsCache, fsAPI,
 	)
 }

clientapi/consumers/roomserver.go

@@ -18,10 +18,10 @@ import (
 	"context"
 	"encoding/json"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/Shopify/sarama"

clientapi/jsonerror/jsonerror.go

@@ -125,10 +125,20 @@ func GuestAccessForbidden(msg string) *MatrixError {
 	return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg}
 }
 
+type IncompatibleRoomVersionError struct {
+	RoomVersion string `json:"room_version"`
+	Error       string `json:"error"`
+	Code        string `json:"errcode"`
+}
+
 // IncompatibleRoomVersion is an error which is returned when the client
 // requests a room with a version that is unsupported.
-func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *MatrixError {
-	return &MatrixError{"M_INCOMPATIBLE_ROOM_VERSION", string(roomVersion)}
+func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *IncompatibleRoomVersionError {
+	return &IncompatibleRoomVersionError{
+		Code:        "M_INCOMPATIBLE_ROOM_VERSION",
+		RoomVersion: string(roomVersion),
+		Error:       "Your homeserver does not support the features required to join this room",
+	}
 }
 
 // UnsupportedRoomVersion is an error which is returned when the client

clientapi/routing/account_data.go

@@ -16,21 +16,20 @@ package routing
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/util"
 )
 
 // GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type}
 func GetAccountData(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
 	userID string, roomID string, dataType string,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -40,15 +39,25 @@ func GetAccountData(
 		}
 	}
 
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+	dataReq := api.QueryAccountDataRequest{
+		UserID:   userID,
+		DataType: dataType,
+		RoomID:   roomID,
+	}
+	dataRes := api.QueryAccountDataResponse{}
+	if err := userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed")
+		return util.ErrorResponse(fmt.Errorf("userAPI.QueryAccountData: %w", err))
 	}
 
-	if data, err := accountDB.GetAccountDataByType(
-		req.Context(), localpart, roomID, dataType,
-	); err == nil {
+	var data json.RawMessage
+	var ok bool
+	if roomID != "" {
+		data, ok = dataRes.RoomAccountData[roomID][dataType]
+	} else {
+		data, ok = dataRes.GlobalAccountData[dataType]
+	}
+	if ok {
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: data,
@@ -63,7 +72,7 @@ func GetAccountData(
 
 // SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type}
 func SaveAccountData(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
 	userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -73,12 +82,6 @@ func SaveAccountData(
 		}
 	}
 
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
-	}
-
 	defer req.Body.Close() // nolint: errcheck
 
 	if req.Body == http.NoBody {
@@ -101,13 +104,19 @@ func SaveAccountData(
 		}
 	}
 
-	if err := accountDB.SaveAccountData(
-		req.Context(), localpart, roomID, dataType, string(body),
-	); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("accountDB.SaveAccountData failed")
-		return jsonerror.InternalServerError()
+	dataReq := api.InputAccountDataRequest{
+		UserID:      userID,
+		DataType:    dataType,
+		RoomID:      roomID,
+		AccountData: json.RawMessage(body),
+	}
+	dataRes := api.InputAccountDataResponse{}
+	if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed")
+		return util.ErrorResponse(err)
 	}
 
+	// TODO: user API should do this since it's account data
 	if err := syncProducer.SendData(userID, roomID, dataType); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
 		return jsonerror.InternalServerError()

clientapi/routing/createroom.go

@@ -24,14 +24,13 @@ import (
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/dendrite/userapi/api"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	log "github.com/sirupsen/logrus"
@@ -135,7 +134,7 @@ type fledglingEvent struct {
 
 // CreateRoom implements /createRoom
 func CreateRoom(
-	req *http.Request, device *authtypes.Device,
+	req *http.Request, device *api.Device,
 	cfg *config.Dendrite,
 	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
@@ -149,7 +148,7 @@ func CreateRoom(
 // createRoom implements /createRoom
 // nolint: gocyclo
 func createRoom(
-	req *http.Request, device *authtypes.Device,
+	req *http.Request, device *api.Device,
 	cfg *config.Dendrite, roomID string,
 	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
@@ -373,13 +372,9 @@ func createRoom(
 
 	// If this is a direct message then we should invite the participants.
 	for _, invitee := range r.Invite {
-		// Build the membership request.
-		body := threepid.MembershipRequest{
-			UserID: invitee,
-		}
 		// Build the invite event.
 		inviteEvent, err := buildMembershipEvent(
-			req.Context(), body, accountDB, device, gomatrixserverlib.Invite,
+			req.Context(), invitee, "", accountDB, device, gomatrixserverlib.Invite,
 			roomID, true, cfg, evTime, rsAPI, asAPI,
 		)
 		if err != nil {
@@ -403,15 +398,15 @@ func createRoom(
 			}
 		}
 		// Send the invite event to the roomserver.
-		if err = roomserverAPI.SendInvite(
+		if perr := roomserverAPI.SendInvite(
 			req.Context(), rsAPI,
 			inviteEvent.Headered(roomVersion),
 			strippedState,         // invite room state
 			cfg.Matrix.ServerName, // send as server
 			nil,                   // transaction ID
-		); err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("SendInvite failed")
-			return jsonerror.InternalServerError()
+		); perr != nil {
+			util.GetLogger(req.Context()).WithError(perr).Error("SendInvite failed")
+			return perr.JSONResponse()
 		}
 	}
 

clientapi/routing/device.go

@@ -19,16 +19,19 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
+// https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-devices
 type deviceJSON struct {
-	DeviceID string `json:"device_id"`
-	UserID   string `json:"user_id"`
+	DeviceID    string `json:"device_id"`
+	DisplayName string `json:"display_name"`
+	LastSeenIP  string `json:"last_seen_ip"`
+	LastSeenTS  uint64 `json:"last_seen_ts"`
 }
 
 type devicesJSON struct {
@@ -45,7 +48,7 @@ type devicesDeleteJSON struct {
 
 // GetDeviceByID handles /devices/{deviceID}
 func GetDeviceByID(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -70,14 +73,13 @@ func GetDeviceByID(
 		Code: http.StatusOK,
 		JSON: deviceJSON{
 			DeviceID: dev.ID,
-			UserID:   dev.UserID,
 		},
 	}
 }
 
 // GetDevicesByLocalpart handles /devices
 func GetDevicesByLocalpart(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
@@ -98,7 +100,6 @@ func GetDevicesByLocalpart(
 	for _, dev := range deviceList {
 		res.Devices = append(res.Devices, deviceJSON{
 			DeviceID: dev.ID,
-			UserID:   dev.UserID,
 		})
 	}
 
@@ -110,7 +111,7 @@ func GetDevicesByLocalpart(
 
 // UpdateDeviceByID handles PUT on /devices/{deviceID}
 func UpdateDeviceByID(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -160,7 +161,7 @@ func UpdateDeviceByID(
 
 // DeleteDeviceById handles DELETE requests to /devices/{deviceId}
 func DeleteDeviceById(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -185,7 +186,7 @@ func DeleteDeviceById(
 
 // DeleteDevices handles POST requests to /delete_devices
 func DeleteDevices(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {

clientapi/routing/directory.go

@@ -18,12 +18,12 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -112,7 +112,7 @@ func DirectoryRoom(
 // TODO: Check if the user has the power level to set an alias
 func SetLocalAlias(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	alias string,
 	cfg *config.Dendrite,
 	aliasAPI roomserverAPI.RoomserverInternalAPI,
@@ -188,7 +188,7 @@ func SetLocalAlias(
 // RemoveLocalAlias implements DELETE /directory/room/{roomAlias}
 func RemoveLocalAlias(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	alias string,
 	aliasAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {

clientapi/routing/getevent.go

@@ -17,22 +17,21 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 type getEventRequest struct {
 	req            *http.Request
-	device         *authtypes.Device
+	device         *userapi.Device
 	roomID         string
 	eventID        string
 	cfg            *config.Dendrite
 	federation     *gomatrixserverlib.FederationClient
-	keyRing        gomatrixserverlib.KeyRing
 	requestedEvent gomatrixserverlib.Event
 }
 
@@ -40,13 +39,12 @@ type getEventRequest struct {
 // https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid
 func GetEvent(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	roomID string,
 	eventID string,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing gomatrixserverlib.KeyRing,
 ) util.JSONResponse {
 	eventsReq := api.QueryEventsByIDRequest{
 		EventIDs: []string{eventID},
@@ -75,7 +73,6 @@ func GetEvent(
 		eventID:        eventID,
 		cfg:            cfg,
 		federation:     federation,
-		keyRing:        keyRing,
 		requestedEvent: requestedEvent,
 	}
 

clientapi/routing/joinroom.go

@@ -18,17 +18,17 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 func JoinRoomByIDOrAlias(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	accountDB accounts.Database,
 	roomIDOrAlias string,
@@ -37,15 +37,14 @@ func JoinRoomByIDOrAlias(
 	joinReq := roomserverAPI.PerformJoinRequest{
 		RoomIDOrAlias: roomIDOrAlias,
 		UserID:        device.UserID,
+		Content:       map[string]interface{}{},
 	}
 	joinRes := roomserverAPI.PerformJoinResponse{}
 
 	// If content was provided in the request then incude that
 	// in the request. It'll get used as a part of the membership
 	// event content.
-	if err := httputil.UnmarshalJSONRequest(req, &joinReq.Content); err != nil {
-		return *err
-	}
+	_ = httputil.UnmarshalJSONRequest(req, &joinReq.Content)
 
 	// Work out our localpart for the client profile request.
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -53,7 +52,8 @@ func JoinRoomByIDOrAlias(
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
 	} else {
 		// Request our profile content to populate the request content with.
-		profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
+		var profile *authtypes.Profile
+		profile, err = accountDB.GetProfileByLocalpart(req.Context(), localpart)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed")
 		} else {
@@ -63,11 +63,9 @@ func JoinRoomByIDOrAlias(
 	}
 
 	// Ask the roomserver to perform the join.
-	if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown(err.Error()),
-		}
+	rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes)
+	if joinRes.Error != nil {
+		return joinRes.Error.JSONResponse()
 	}
 
 	return util.JSONResponse{

clientapi/routing/leaveroom.go

@@ -17,15 +17,15 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
 func LeaveRoomByID(
 	req *http.Request,
-	device *authtypes.Device,
+	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	roomID string,
 ) util.JSONResponse {

clientapi/routing/login.go

@@ -20,13 +20,13 @@ import (
 	"context"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -47,6 +47,7 @@ type loginIdentifier struct {
 
 type passwordRequest struct {
 	Identifier loginIdentifier `json:"identifier"`
+	User       string          `json:"user"` // deprecated in favour of identifier
 	Password   string          `json:"password"`
 	// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
 	// Thus a pointer is needed to differentiate between the two
@@ -80,7 +81,8 @@ func Login(
 		}
 	} else if req.Method == http.MethodPost {
 		var r passwordRequest
-		var acc *authtypes.Account
+		var acc *api.Account
+		var errJSON *util.JSONResponse
 		resErr := httputil.UnmarshalJSONRequest(req, &r)
 		if resErr != nil {
 			return *resErr
@@ -93,30 +95,22 @@ func Login(
 					JSON: jsonerror.BadJSON("'user' must be supplied."),
 				}
 			}
-
-			util.GetLogger(req.Context()).WithField("user", r.Identifier.User).Info("Processing login request")
-
-			localpart, err := userutil.ParseUsernameParam(r.Identifier.User, &cfg.Matrix.ServerName)
-			if err != nil {
-				return util.JSONResponse{
-					Code: http.StatusBadRequest,
-					JSON: jsonerror.InvalidUsername(err.Error()),
-				}
-			}
-
-			acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password)
-			if err != nil {
-				// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
-				// but that would leak the existence of the user.
-				return util.JSONResponse{
-					Code: http.StatusForbidden,
-					JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"),
-				}
+			acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.Identifier.User)
+			if errJSON != nil {
+				return *errJSON
 			}
 		default:
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.BadJSON("login identifier '" + r.Identifier.Type + "' not supported"),
+			// TODO: The below behaviour is deprecated but without it Riot iOS won't log in
+			if r.User != "" {
+				acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.User)
+				if errJSON != nil {
+					return *errJSON
+				}
+			} else {
+				return util.JSONResponse{
+					Code: http.StatusBadRequest,
+					JSON: jsonerror.BadJSON("login identifier '" + r.Identifier.Type + "' not supported"),
+				}
 			}
 		}
 
@@ -155,11 +149,40 @@ func getDevice(
 	ctx context.Context,
 	r passwordRequest,
 	deviceDB devices.Database,
-	acc *authtypes.Account,
+	acc *api.Account,
 	token string,
-) (dev *authtypes.Device, err error) {
+) (dev *api.Device, err error) {
 	dev, err = deviceDB.CreateDevice(
 		ctx, acc.Localpart, r.DeviceID, token, r.InitialDisplayName,
 	)
 	return
 }
+
+func (r *passwordRequest) processUsernamePasswordLoginRequest(
+	req *http.Request, accountDB accounts.Database,
+	cfg *config.Dendrite, username string,
+) (acc *api.Account, errJSON *util.JSONResponse) {
+	util.GetLogger(req.Context()).WithField("user", username).Info("Processing login request")
+
+	localpart, err := userutil.ParseUsernameParam(username, &cfg.Matrix.ServerName)
+	if err != nil {
+		errJSON = &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidUsername(err.Error()),
+		}
+		return
+	}
+
+	acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password)
+	if err != nil {
+		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
+		// but that would leak the existence of the user.
+		errJSON = &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"),
+		}
+		return
+	}
+
+	return
+}

clientapi/routing/logout.go

@@ -17,16 +17,16 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 // Logout handles POST /logout
 func Logout(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
@@ -47,7 +47,7 @@ func Logout(
 
 // LogoutAll handles POST /logout/all
 func LogoutAll(
-	req *http.Request, deviceDB devices.Database, device *authtypes.Device,
+	req *http.Request, deviceDB devices.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {

clientapi/routing/membership.go

@@ -22,7 +22,6 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
@@ -30,6 +29,8 @@ import (
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/util"
@@ -37,40 +38,141 @@ import (
 
 var errMissingUserID = errors.New("'user_id' must be supplied")
 
-// SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
-// by building a m.room.member event then sending it to the room server
-// TODO: Can we improve the cyclo count here? Separate code paths for invites?
-// nolint:gocyclo
-func SendMembership(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
-	roomID string, membership string, cfg *config.Dendrite,
+func SendBan(
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	roomID string, cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
-	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
-	verRes := api.QueryRoomVersionForRoomResponse{}
-	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	if reqErr != nil {
+		return *reqErr
+	}
+	return sendMembership(req.Context(), accountDB, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+}
+
+func sendMembership(ctx context.Context, accountDB accounts.Database, device *userapi.Device,
+	roomID, membership, reason string, cfg *config.Dendrite, targetUserID string, evTime time.Time,
+	roomVer gomatrixserverlib.RoomVersion,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
+
+	event, err := buildMembershipEvent(
+		ctx, targetUserID, reason, accountDB, device, membership,
+		roomID, false, cfg, evTime, rsAPI, asAPI,
+	)
+	if err == errMissingUserID {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	} else if err == eventutil.ErrRoomNoExists {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound(err.Error()),
+		}
+	} else if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
+		return jsonerror.InternalServerError()
+	}
+
+	_, err = roomserverAPI.SendEvents(
+		ctx, rsAPI,
+		[]gomatrixserverlib.HeaderedEvent{event.Headered(roomVer)},
+		cfg.Matrix.ServerName,
+		nil,
+	)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
+		return jsonerror.InternalServerError()
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func SendKick(
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	roomID string, cfg *config.Dendrite,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	if reqErr != nil {
+		return *reqErr
+	}
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("missing user_id"),
 		}
 	}
 
-	var body threepid.MembershipRequest
-	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
+	var queryRes roomserverAPI.QueryMembershipForUserResponse
+	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+		RoomID: roomID,
+		UserID: body.UserID,
+	}, &queryRes)
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	// kick is only valid if the user is not currently banned
+	if queryRes.Membership == "ban" {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Unknown("cannot /kick banned users"),
+		}
+	}
+	// TODO: should we be using SendLeave instead?
+	return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+}
+
+func SendUnban(
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	roomID string, cfg *config.Dendrite,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	if reqErr != nil {
+		return *reqErr
+	}
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("missing user_id"),
+		}
+	}
+
+	var queryRes roomserverAPI.QueryMembershipForUserResponse
+	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+		RoomID: roomID,
+		UserID: body.UserID,
+	}, &queryRes)
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	// unban is only valid if the user is currently banned
+	if queryRes.Membership != "ban" {
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.Unknown("can only /unban users that are banned"),
+		}
+	}
+	// TODO: should we be using SendLeave instead?
+	return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+}
+
+func SendInvite(
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	roomID string, cfg *config.Dendrite,
+	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	if reqErr != nil {
 		return *reqErr
 	}
 
-	evTime, err := httputil.ParseTSParam(req)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
-		}
-	}
-
 	inviteStored, jsonErrResp := checkAndProcessThreepid(
-		req, device, &body, cfg, rsAPI, accountDB,
-		membership, roomID, evTime,
+		req, device, body, cfg, rsAPI, accountDB, roomID, evTime,
 	)
 	if jsonErrResp != nil {
 		return *jsonErrResp
@@ -87,7 +189,7 @@ func SendMembership(
 	}
 
 	event, err := buildMembershipEvent(
-		req.Context(), body, accountDB, device, membership,
+		req.Context(), body.UserID, body.Reason, accountDB, device, "invite",
 		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
 	if err == errMissingUserID {
@@ -105,61 +207,32 @@ func SendMembership(
 		return jsonerror.InternalServerError()
 	}
 
-	var returnData interface{} = struct{}{}
-
-	switch membership {
-	case gomatrixserverlib.Invite:
-		// Invites need to be handled specially
-		err = roomserverAPI.SendInvite(
-			req.Context(), rsAPI,
-			event.Headered(verRes.RoomVersion),
-			nil, // ask the roomserver to draw up invite room state for us
-			cfg.Matrix.ServerName,
-			nil,
-		)
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("producer.SendInvite failed")
-			return jsonerror.InternalServerError()
-		}
-	case gomatrixserverlib.Join:
-		// The join membership requires the room id to be sent in the response
-		returnData = struct {
-			RoomID string `json:"room_id"`
-		}{roomID}
-		fallthrough
-	default:
-		_, err = roomserverAPI.SendEvents(
-			req.Context(), rsAPI,
-			[]gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)},
-			cfg.Matrix.ServerName,
-			nil,
-		)
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
-			return jsonerror.InternalServerError()
-		}
+	perr := roomserverAPI.SendInvite(
+		req.Context(), rsAPI,
+		event.Headered(roomVer),
+		nil, // ask the roomserver to draw up invite room state for us
+		cfg.Matrix.ServerName,
+		nil,
+	)
+	if perr != nil {
+		util.GetLogger(req.Context()).WithError(perr).Error("producer.SendInvite failed")
+		return perr.JSONResponse()
 	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: returnData,
+		JSON: struct{}{},
 	}
 }
 
 func buildMembershipEvent(
 	ctx context.Context,
-	body threepid.MembershipRequest, accountDB accounts.Database,
-	device *authtypes.Device,
+	targetUserID, reason string, accountDB accounts.Database,
+	device *userapi.Device,
 	membership, roomID string, isDirect bool,
 	cfg *config.Dendrite, evTime time.Time,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*gomatrixserverlib.Event, error) {
-	stateKey, reason, err := getMembershipStateKey(body, device, membership)
-	if err != nil {
-		return nil, err
-	}
-
-	profile, err := loadProfile(ctx, stateKey, cfg, accountDB, asAPI)
+	profile, err := loadProfile(ctx, targetUserID, cfg, accountDB, asAPI)
 	if err != nil {
 		return nil, err
 	}
@@ -168,12 +241,7 @@ func buildMembershipEvent(
 		Sender:   device.UserID,
 		RoomID:   roomID,
 		Type:     "m.room.member",
-		StateKey: &stateKey,
-	}
-
-	// "unban" or "kick" isn't a valid membership value, change it to "leave"
-	if membership == "unban" || membership == "kick" {
-		membership = gomatrixserverlib.Leave
+		StateKey: &targetUserID,
 	}
 
 	content := gomatrixserverlib.MemberContent{
@@ -217,46 +285,50 @@ func loadProfile(
 	return profile, err
 }
 
-// getMembershipStateKey extracts the target user ID of a membership change.
-// For "join" and "leave" this will be the ID of the user making the change.
-// For "ban", "unban", "kick" and "invite" the target user ID will be in the JSON request body.
-// In the latter case, if there was an issue retrieving the user ID from the request body,
-// returns a JSONResponse with a corresponding error code and message.
-func getMembershipStateKey(
-	body threepid.MembershipRequest, device *authtypes.Device, membership string,
-) (stateKey string, reason string, err error) {
-	if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite {
-		// If we're in this case, the state key is contained in the request body,
-		// possibly along with a reason (for "kick" and "ban") so we need to parse
-		// it
-		if body.UserID == "" {
-			err = errMissingUserID
-			return
+func extractRequestData(req *http.Request, roomID string, rsAPI api.RoomserverInternalAPI) (
+	body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse,
+) {
+	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
+	verRes := api.QueryRoomVersionForRoomResponse{}
+	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
+		resErr = &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
 		}
+		return
+	}
+	roomVer = verRes.RoomVersion
 
-		stateKey = body.UserID
-		reason = body.Reason
-	} else {
-		stateKey = device.UserID
+	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
+		resErr = reqErr
+		return
 	}
 
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		resErr = &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+		return
+	}
 	return
 }
 
 func checkAndProcessThreepid(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	body *threepid.MembershipRequest,
 	cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	accountDB accounts.Database,
-	membership, roomID string,
+	roomID string,
 	evTime time.Time,
 ) (inviteStored bool, errRes *util.JSONResponse) {
 
 	inviteStored, err := threepid.CheckAndProcessInvite(
 		req.Context(), device, body, cfg, rsAPI, accountDB,
-		membership, roomID, evTime,
+		roomID, evTime,
 	)
 	if err == threepid.ErrMissingParameter {
 		return inviteStored, &util.JSONResponse{

clientapi/routing/memberships.go

@@ -15,14 +15,15 @@
 package routing
 
 import (
+	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -35,9 +36,19 @@ type getJoinedRoomsResponse struct {
 	JoinedRooms []string `json:"joined_rooms"`
 }
 
+// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
+type getJoinedMembersResponse struct {
+	Joined map[string]joinedMember `json:"joined"`
+}
+
+type joinedMember struct {
+	DisplayName string `json:"display_name"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
 // GetMemberships implements GET /rooms/{roomId}/members
 func GetMemberships(
-	req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool,
+	req *http.Request, device *userapi.Device, roomID string, joinedOnly bool,
 	_ *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
@@ -59,6 +70,22 @@ func GetMemberships(
 		}
 	}
 
+	if joinedOnly {
+		var res getJoinedMembersResponse
+		res.Joined = make(map[string]joinedMember)
+		for _, ev := range queryRes.JoinEvents {
+			var content joinedMember
+			if err := json.Unmarshal(ev.Content, &content); err != nil {
+				util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
+				return jsonerror.InternalServerError()
+			}
+			res.Joined[ev.Sender] = content
+		}
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: res,
+		}
+	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: getMembershipResponse{queryRes.JoinEvents},
@@ -67,7 +94,7 @@ func GetMemberships(
 
 func GetJoinedRooms(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	accountsDB accounts.Database,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)

clientapi/routing/profile.go

@@ -21,12 +21,13 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/gomatrix"
@@ -92,7 +93,7 @@ func GetAvatarURL(
 // SetAvatarURL implements PUT /profile/{userID}/avatar_url
 // nolint:gocyclo
 func SetAvatarURL(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
 	userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -206,7 +207,7 @@ func GetDisplayName(
 // SetDisplayName implements PUT /profile/{userID}/displayname
 // nolint:gocyclo
 func SetDisplayName(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *userapi.Device,
 	userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {

clientapi/routing/register.go

@@ -34,15 +34,14 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/tokens"
 	"github.com/matrix-org/util"
@@ -441,8 +440,8 @@ func validateApplicationService(
 // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
 func Register(
 	req *http.Request,
+	userAPI userapi.UserInternalAPI,
 	accountDB accounts.Database,
-	deviceDB devices.Database,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var r registerRequest
@@ -451,7 +450,7 @@ func Register(
 		return *resErr
 	}
 	if req.URL.Query().Get("kind") == "guest" {
-		return handleGuestRegistration(req, r, cfg, accountDB, deviceDB)
+		return handleGuestRegistration(req, r, cfg, userAPI)
 	}
 
 	// Retrieve or generate the sessionID
@@ -507,17 +506,19 @@ func Register(
 		"session_id": r.Auth.Session,
 	}).Info("Processing registration request")
 
-	return handleRegistrationFlow(req, r, sessionID, cfg, accountDB, deviceDB)
+	return handleRegistrationFlow(req, r, sessionID, cfg, userAPI)
 }
 
 func handleGuestRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
-	acc, err := accountDB.CreateGuestAccount(req.Context())
+	var res userapi.PerformAccountCreationResponse
+	err := userAPI.PerformAccountCreation(req.Context(), &userapi.PerformAccountCreationRequest{
+		AccountType: userapi.AccountTypeGuest,
+	}, &res)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -526,8 +527,8 @@ func handleGuestRegistration(
 	}
 	token, err := tokens.GenerateLoginToken(tokens.TokenOptions{
 		ServerPrivateKey: cfg.Matrix.PrivateKey.Seed(),
-		ServerName:       string(acc.ServerName),
-		UserID:           acc.UserID,
+		ServerName:       string(res.Account.ServerName),
+		UserID:           res.Account.UserID,
 	})
 
 	if err != nil {
@@ -537,7 +538,12 @@ func handleGuestRegistration(
 		}
 	}
 	//we don't allow guests to specify their own device_id
-	dev, err := deviceDB.CreateDevice(req.Context(), acc.Localpart, nil, token, r.InitialDisplayName)
+	var devRes userapi.PerformDeviceCreationResponse
+	err = userAPI.PerformDeviceCreation(req.Context(), &userapi.PerformDeviceCreationRequest{
+		Localpart:         res.Account.Localpart,
+		DeviceDisplayName: r.InitialDisplayName,
+		AccessToken:       token,
+	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -547,10 +553,10 @@ func handleGuestRegistration(
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: registerResponse{
-			UserID:      dev.UserID,
-			AccessToken: dev.AccessToken,
-			HomeServer:  acc.ServerName,
-			DeviceID:    dev.ID,
+			UserID:      devRes.Device.UserID,
+			AccessToken: devRes.Device.AccessToken,
+			HomeServer:  res.Account.ServerName,
+			DeviceID:    devRes.Device.ID,
 		},
 	}
 }
@@ -563,8 +569,7 @@ func handleRegistrationFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
 	// TODO: Shared secret registration (create new user scripts)
 	// TODO: Enable registration config flag
@@ -615,7 +620,7 @@ func handleRegistrationFlow(
 		// by whether the request contains an access token.
 		if err == nil {
 			return handleApplicationServiceRegistration(
-				accessToken, err, req, r, cfg, accountDB, deviceDB,
+				accessToken, err, req, r, cfg, userAPI,
 			)
 		}
 
@@ -626,7 +631,7 @@ func handleRegistrationFlow(
 		// don't need a condition on that call since the registration is clearly
 		// stated as being AS-related.
 		return handleApplicationServiceRegistration(
-			accessToken, err, req, r, cfg, accountDB, deviceDB,
+			accessToken, err, req, r, cfg, userAPI,
 		)
 
 	case authtypes.LoginTypeDummy:
@@ -645,7 +650,7 @@ func handleRegistrationFlow(
 	// A response with current registration flow and remaining available methods
 	// will be returned if a flow has not been successfully completed yet
 	return checkAndCompleteFlow(sessions.GetCompletedStages(sessionID),
-		req, r, sessionID, cfg, accountDB, deviceDB)
+		req, r, sessionID, cfg, userAPI)
 }
 
 // handleApplicationServiceRegistration handles the registration of an
@@ -662,8 +667,7 @@ func handleApplicationServiceRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
 	// Check if we previously had issues extracting the access token from the
 	// request.
@@ -687,7 +691,7 @@ func handleApplicationServiceRegistration(
 	// Don't need to worry about appending to registration stages as
 	// application service registration is entirely separate.
 	return completeRegistration(
-		req.Context(), accountDB, deviceDB, r.Username, "", appserviceID,
+		req.Context(), userAPI, r.Username, "", appserviceID,
 		r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 	)
 }
@@ -701,13 +705,12 @@ func checkAndCompleteFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.Dendrite,
-	accountDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
 		return completeRegistration(
-			req.Context(), accountDB, deviceDB, r.Username, r.Password, "",
+			req.Context(), userAPI, r.Username, r.Password, "",
 			r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 		)
 	}
@@ -724,8 +727,7 @@ func checkAndCompleteFlow(
 // LegacyRegister process register requests from the legacy v1 API
 func LegacyRegister(
 	req *http.Request,
-	accountDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var r legacyRegisterRequest
@@ -760,10 +762,10 @@ func LegacyRegister(
 			return util.MessageResponse(http.StatusForbidden, "HMAC incorrect")
 		}
 
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
+		return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil)
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
+		return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil)
 	default:
 		return util.JSONResponse{
 			Code: http.StatusNotImplemented,
@@ -809,8 +811,7 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u
 // not all
 func completeRegistration(
 	ctx context.Context,
-	accountDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 	username, password, appserviceID string,
 	inhibitLogin eventutil.WeakBoolean,
 	displayName, deviceID *string,
@@ -829,9 +830,16 @@ func completeRegistration(
 		}
 	}
 
-	acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID)
+	var accRes userapi.PerformAccountCreationResponse
+	err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{
+		AppServiceID: appserviceID,
+		Localpart:    username,
+		Password:     password,
+		AccountType:  userapi.AccountTypeUser,
+		OnConflict:   userapi.ConflictAbort,
+	}, &accRes)
 	if err != nil {
-		if errors.Is(err, sqlutil.ErrUserExists) { // user already exists
+		if _, ok := err.(*userapi.ErrorConflict); ok { // user already exists
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: jsonerror.UserInUse("Desired user ID is already taken."),
@@ -852,8 +860,8 @@ func completeRegistration(
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: registerResponse{
-				UserID:     userutil.MakeUserID(username, acc.ServerName),
-				HomeServer: acc.ServerName,
+				UserID:     userutil.MakeUserID(username, accRes.Account.ServerName),
+				HomeServer: accRes.Account.ServerName,
 			},
 		}
 	}
@@ -866,7 +874,13 @@ func completeRegistration(
 		}
 	}
 
-	dev, err := deviceDB.CreateDevice(ctx, username, deviceID, token, displayName)
+	var devRes userapi.PerformDeviceCreationResponse
+	err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
+		Localpart:         username,
+		AccessToken:       token,
+		DeviceDisplayName: displayName,
+		DeviceID:          deviceID,
+	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -877,10 +891,10 @@ func completeRegistration(
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: registerResponse{
-			UserID:      dev.UserID,
-			AccessToken: dev.AccessToken,
-			HomeServer:  acc.ServerName,
-			DeviceID:    dev.ID,
+			UserID:      devRes.Device.UserID,
+			AccessToken: devRes.Device.AccessToken,
+			HomeServer:  accRes.Account.ServerName,
+			DeviceID:    devRes.Device.ID,
 		},
 	}
 }

clientapi/routing/room_tagging.go

@@ -20,28 +20,19 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
-// newTag creates and returns a new gomatrix.TagContent
-func newTag() gomatrix.TagContent {
-	return gomatrix.TagContent{
-		Tags: make(map[string]gomatrix.TagProperties),
-	}
-}
-
 // GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags
 func GetTags(
 	req *http.Request,
-	accountDB accounts.Database,
-	device *authtypes.Device,
+	userAPI api.UserInternalAPI,
+	device *api.Device,
 	userID string,
 	roomID string,
 	syncProducer *producers.SyncAPIProducer,
@@ -54,22 +45,15 @@ func GetTags(
 		}
 	}
 
-	_, data, err := obtainSavedTags(req, userID, roomID, accountDB)
+	tagContent, err := obtainSavedTags(req, userID, roomID, userAPI)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed")
 		return jsonerror.InternalServerError()
 	}
 
-	if data == nil {
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: struct{}{},
-		}
-	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: data.Content,
+		JSON: tagContent,
 	}
 }
 
@@ -78,8 +62,8 @@ func GetTags(
 // the tag to the "map" and saving the new "map" to the DB
 func PutTag(
 	req *http.Request,
-	accountDB accounts.Database,
-	device *authtypes.Device,
+	userAPI api.UserInternalAPI,
+	device *api.Device,
 	userID string,
 	roomID string,
 	tag string,
@@ -98,34 +82,25 @@ func PutTag(
 		return *reqErr
 	}
 
-	localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB)
+	tagContent, err := obtainSavedTags(req, userID, roomID, userAPI)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed")
 		return jsonerror.InternalServerError()
 	}
 
-	var tagContent gomatrix.TagContent
-	if data != nil {
-		if err = json.Unmarshal(data.Content, &tagContent); err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed")
-			return jsonerror.InternalServerError()
-		}
-	} else {
-		tagContent = newTag()
+	if tagContent.Tags == nil {
+		tagContent.Tags = make(map[string]gomatrix.TagProperties)
 	}
 	tagContent.Tags[tag] = properties
-	if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil {
+
+	if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed")
 		return jsonerror.InternalServerError()
 	}
 
-	// Send data to syncProducer in order to inform clients of changes
-	// Run in a goroutine in order to prevent blocking the tag request response
-	go func() {
-		if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
-			logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-		}
-	}()
+	if err = syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
+		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
+	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -138,8 +113,8 @@ func PutTag(
 // the "map" and then saving the new "map" in the DB
 func DeleteTag(
 	req *http.Request,
-	accountDB accounts.Database,
-	device *authtypes.Device,
+	userAPI api.UserInternalAPI,
+	device *api.Device,
 	userID string,
 	roomID string,
 	tag string,
@@ -153,28 +128,12 @@ func DeleteTag(
 		}
 	}
 
-	localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB)
+	tagContent, err := obtainSavedTags(req, userID, roomID, userAPI)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed")
 		return jsonerror.InternalServerError()
 	}
 
-	// If there are no tags in the database, exit
-	if data == nil {
-		// Spec only defines 200 responses for this endpoint so we don't return anything else.
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: struct{}{},
-		}
-	}
-
-	var tagContent gomatrix.TagContent
-	err = json.Unmarshal(data.Content, &tagContent)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed")
-		return jsonerror.InternalServerError()
-	}
-
 	// Check whether the tag to be deleted exists
 	if _, ok := tagContent.Tags[tag]; ok {
 		delete(tagContent.Tags, tag)
@@ -185,18 +144,16 @@ func DeleteTag(
 			JSON: struct{}{},
 		}
 	}
-	if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil {
+
+	if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed")
 		return jsonerror.InternalServerError()
 	}
 
-	// Send data to syncProducer in order to inform clients of changes
-	// Run in a goroutine in order to prevent blocking the tag request response
-	go func() {
-		if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
-			logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-		}
-	}()
+	// TODO: user API should do this since it's account data
+	if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
+		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
+	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -210,32 +167,46 @@ func obtainSavedTags(
 	req *http.Request,
 	userID string,
 	roomID string,
-	accountDB accounts.Database,
-) (string, *gomatrixserverlib.ClientEvent, error) {
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
-	if err != nil {
-		return "", nil, err
+	userAPI api.UserInternalAPI,
+) (tags gomatrix.TagContent, err error) {
+	dataReq := api.QueryAccountDataRequest{
+		UserID:   userID,
+		RoomID:   roomID,
+		DataType: "m.tag",
 	}
-
-	data, err := accountDB.GetAccountDataByType(
-		req.Context(), localpart, roomID, "m.tag",
-	)
-
-	return localpart, data, err
+	dataRes := api.QueryAccountDataResponse{}
+	err = userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes)
+	if err != nil {
+		return
+	}
+	data, ok := dataRes.RoomAccountData[roomID]["m.tag"]
+	if !ok {
+		return
+	}
+	if err = json.Unmarshal(data, &tags); err != nil {
+		return
+	}
+	return tags, nil
 }
 
 // saveTagData saves the provided tag data into the database
 func saveTagData(
 	req *http.Request,
-	localpart string,
+	userID string,
 	roomID string,
-	accountDB accounts.Database,
+	userAPI api.UserInternalAPI,
 	Tag gomatrix.TagContent,
 ) error {
 	newTagData, err := json.Marshal(Tag)
 	if err != nil {
 		return err
 	}
-
-	return accountDB.SaveAccountData(req.Context(), localpart, roomID, "m.tag", string(newTagData))
+	dataReq := api.InputAccountDataRequest{
+		UserID:      userID,
+		RoomID:      roomID,
+		DataType:    "m.tag",
+		AccountData: json.RawMessage(newTagData),
+	}
+	dataRes := api.InputAccountDataResponse{}
+	return userAPI.InputAccountData(req.Context(), &dataReq, &dataRes)
 }

clientapi/routing/routing.go

@@ -21,10 +21,6 @@ import (
 
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
@@ -33,6 +29,9 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -54,8 +53,8 @@ func Setup(
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	accountDB accounts.Database,
 	deviceDB devices.Database,
+	userAPI api.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing gomatrixserverlib.KeyRing,
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
 	federationSender federationSenderAPI.FederationSenderInternalAPI,
@@ -81,19 +80,13 @@ func Setup(
 	v1mux := publicAPIMux.PathPrefix(pathPrefixV1).Subrouter()
 	unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter()
 
-	authData := auth.Data{
-		AccountDB:   accountDB,
-		DeviceDB:    deviceDB,
-		AppServices: cfg.Derived.ApplicationServices,
-	}
-
 	r0mux.Handle("/createRoom",
-		httputil.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
-		httputil.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -104,12 +97,23 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/joined_rooms",
-		httputil.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetJoinedRooms(req, device, accountDB)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/join",
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return JoinRoomByIDOrAlias(
+				req, device, rsAPI, accountDB, vars["roomID"],
+			)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/leave",
-		httputil.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -119,17 +123,44 @@ func Setup(
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}",
-		httputil.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/ban",
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI)
+			return SendBan(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/invite",
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendInvite(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/kick",
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendKick(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/unban",
+		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendUnban(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
-		httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -138,7 +169,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
-		httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -149,16 +180,16 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/event/{eventID}",
-		httputil.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing)
+			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
@@ -166,24 +197,31 @@ func Setup(
 		return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type}", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "")
+		// If there's a trailing slash, remove it
+		eventType := vars["type"]
+		if strings.HasSuffix(eventType, "/") {
+			eventType = eventType[:len(eventType)-1]
+		}
+		eventFormat := req.URL.Query().Get("format") == "event"
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], eventType, "", eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"])
+		eventFormat := req.URL.Query().Get("format") == "event"
+		return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"], eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
-		httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -199,7 +237,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
-		httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -210,11 +248,11 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
-		return Register(req, accountDB, deviceDB, cfg)
+		return Register(req, userAPI, accountDB, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
 	v1mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
-		return LegacyRegister(req, accountDB, deviceDB, cfg)
+		return LegacyRegister(req, userAPI, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
@@ -232,7 +270,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		httputil.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -242,7 +280,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		httputil.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -252,19 +290,19 @@ func Setup(
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/logout",
-		httputil.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return Logout(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/logout/all",
-		httputil.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return LogoutAll(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
-		httputil.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -274,7 +312,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/sendToDevice/{eventType}/{txnID}",
-		httputil.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -288,7 +326,7 @@ func Setup(
 	// rather than r0. It's an exact duplicate of the above handler.
 	// TODO: Remove this if/when sytest is fixed!
 	unstableMux.Handle("/sendToDevice/{eventType}/{txnID}",
-		httputil.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -299,7 +337,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/account/whoami",
-		httputil.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return Whoami(req, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -338,26 +376,6 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/user/{userId}/filter",
-		httputil.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return PutFilter(req, device, accountDB, vars["userId"])
-		}),
-	).Methods(http.MethodPost, http.MethodOptions)
-
-	r0mux.Handle("/user/{userId}/filter/{filterId}",
-		httputil.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"])
-		}),
-	).Methods(http.MethodGet, http.MethodOptions)
-
 	// Riot user settings
 
 	r0mux.Handle("/profile/{userID}",
@@ -381,7 +399,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
-		httputil.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -403,7 +421,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/displayname",
-		httputil.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -415,19 +433,19 @@ func Setup(
 	// PUT requests, so we need to allow this method
 
 	r0mux.Handle("/account/3pid",
-		httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetAssociated3PIDs(req, accountDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/account/3pid",
-		httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return CheckAndSave3PIDAssociation(req, accountDB, device, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	unstableMux.Handle("/account/3pid/delete",
-		httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return Forget3PID(req, accountDB)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -450,7 +468,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/voip/turnServer",
-		httputil.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return RequestTurnServer(req, device, cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -476,47 +494,47 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
-		httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer)
+			return SaveAccountData(req, userAPI, device, vars["userID"], "", vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
-		httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer)
+			return SaveAccountData(req, userAPI, device, vars["userID"], vars["roomID"], vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
-		httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetAccountData(req, accountDB, device, vars["userID"], "", vars["type"])
+			return GetAccountData(req, userAPI, device, vars["userID"], "", vars["type"])
 		}),
 	).Methods(http.MethodGet)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
-		httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"])
+			return GetAccountData(req, userAPI, device, vars["userID"], vars["roomID"], vars["type"])
 		}),
 	).Methods(http.MethodGet)
 
 	r0mux.Handle("/rooms/{roomID}/members",
-		httputil.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -526,7 +544,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/joined_members",
-		httputil.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -543,13 +561,13 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/devices",
-		httputil.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetDevicesByLocalpart(req, deviceDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		httputil.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -559,7 +577,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		httputil.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -569,7 +587,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		httputil.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
@@ -579,7 +597,7 @@ func Setup(
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/delete_devices",
-		httputil.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return DeleteDevices(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -604,37 +622,37 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags",
-		httputil.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetTags(req, accountDB, device, vars["userId"], vars["roomId"], syncProducer)
+			return GetTags(req, userAPI, device, vars["userId"], vars["roomId"], syncProducer)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
-		httputil.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return PutTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
+			return PutTag(req, userAPI, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
-		httputil.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return DeleteTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
+			return DeleteTag(req, userAPI, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/capabilities",
-		httputil.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
 			return GetCapabilities(req, rsAPI)
 		}),
 	).Methods(http.MethodGet)

clientapi/routing/sendevent.go

@@ -17,13 +17,13 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
@@ -41,7 +41,7 @@ type sendEventResponse struct {
 //   /rooms/{roomID}/state/{eventType}/{stateKey}
 func SendEvent(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	roomID, eventType string, txnID, stateKey *string,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
@@ -110,7 +110,7 @@ func SendEvent(
 
 func generateSendEvent(
 	req *http.Request,
-	device *authtypes.Device,
+	device *userapi.Device,
 	roomID, eventType string, stateKey *string,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
@@ -157,6 +157,17 @@ func generateSendEvent(
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.BadJSON(e.Error()),
 		}
+	} else if e, ok := err.(gomatrixserverlib.EventValidationError); ok {
+		if e.Code == gomatrixserverlib.EventValidationTooLarge {
+			return nil, &util.JSONResponse{
+				Code: http.StatusRequestEntityTooLarge,
+				JSON: jsonerror.BadJSON(e.Error()),
+			}
+		}
+		return nil, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(e.Error()),
+		}
 	} else if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("eventutil.BuildEvent failed")
 		resErr := jsonerror.InternalServerError()

clientapi/routing/sendtodevice.go

@@ -16,18 +16,18 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/internal/transactions"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
 // SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}
 // sends the device events to the EDU Server
 func SendToDevice(
-	req *http.Request, device *authtypes.Device,
+	req *http.Request, device *userapi.Device,
 	eduAPI api.EDUServerInputAPI,
 	txnCache *transactions.Cache,
 	eventType string, txnID *string,

clientapi/routing/sendtyping.go

@@ -16,12 +16,12 @@ import (
 	"database/sql"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/eduserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/util"
 )
 
@@ -33,7 +33,7 @@ type typingContentJSON struct {
 // SendTyping handles PUT /rooms/{roomID}/typing/{userID}
 // sends the typing events to client API typingProducer
 func SendTyping(
-	req *http.Request, device *authtypes.Device, roomID string,
+	req *http.Request, device *userapi.Device, roomID string,
 	userID string, accountDB accounts.Database,
 	eduAPI api.EDUServerInputAPI,
 ) util.JSONResponse {

clientapi/routing/state.go

@@ -98,7 +98,8 @@ func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI
 // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current
 // state to see if there is an event with that type and state key, if there
 // is then (by default) we return the content, otherwise a 404.
-func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string, evType, stateKey string) util.JSONResponse {
+// If eventFormat=true, sends the whole event else just the content.
+func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID, evType, stateKey string, eventFormat bool) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 	util.GetLogger(ctx).WithFields(log.Fields{
@@ -134,8 +135,15 @@ func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInterna
 		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(stateRes.StateEvents[0], gomatrixserverlib.FormatAll),
 	}
 
+	var res interface{}
+	if eventFormat {
+		res = stateEvent
+	} else {
+		res = stateEvent.Content
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: stateEvent.Content,
+		JSON: res,
 	}
 }

clientapi/routing/threepid.go

@@ -18,11 +18,12 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -84,7 +85,7 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf
 
 // CheckAndSave3PIDAssociation implements POST /account/3pid
 func CheckAndSave3PIDAssociation(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *api.Device,
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var body threepid.EmailAssociationCheckRequest
@@ -148,7 +149,7 @@ func CheckAndSave3PIDAssociation(
 
 // GetAssociated3PIDs implements GET /account/3pid
 func GetAssociated3PIDs(
-	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
+	req *http.Request, accountDB accounts.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {

clientapi/routing/voip.go

@@ -22,16 +22,16 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/util"
 )
 
 // RequestTurnServer implements:
 //     GET /voip/turnServer
-func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg *config.Dendrite) util.JSONResponse {
+func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Dendrite) util.JSONResponse {
 	turnConfig := cfg.TURN
 
 	// TODO Guest Support

clientapi/routing/whoami.go

@@ -15,7 +15,7 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
@@ -26,7 +26,7 @@ type whoamiResponse struct {
 
 // Whoami implements `/account/whoami` which enables client to query their account user id.
 // https://matrix.org/docs/spec/client_server/r0.3.0.html#get-matrix-client-r0-account-whoami
-func Whoami(req *http.Request, device *authtypes.Device) util.JSONResponse {
+func Whoami(req *http.Request, device *api.Device) util.JSONResponse {
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: whoamiResponse{UserID: device.UserID},

clientapi/threepid/invites.go

@@ -25,10 +25,11 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -85,12 +86,12 @@ var (
 // can be emitted.
 func CheckAndProcessInvite(
 	ctx context.Context,
-	device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite,
+	device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI, db accounts.Database,
-	membership string, roomID string,
+	roomID string,
 	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
-	if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") {
+	if body.Address == "" && body.IDServer == "" && body.Medium == "" {
 		// If none of the 3PID-specific fields are supplied, it's a standard invite
 		// so return nil for it to be processed as such
 		return
@@ -136,7 +137,7 @@ func CheckAndProcessInvite(
 // Returns an error if a check or a request failed.
 func queryIDServer(
 	ctx context.Context,
-	db accounts.Database, cfg *config.Dendrite, device *authtypes.Device,
+	db accounts.Database, cfg *config.Dendrite, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) {
 	if err = isTrusted(body.IDServer, cfg); err != nil {
@@ -205,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe
 // Returns an error if the request failed to send or if the response couldn't be parsed.
 func queryIDServerStoreInvite(
 	ctx context.Context,
-	db accounts.Database, cfg *config.Dendrite, device *authtypes.Device,
+	db accounts.Database, cfg *config.Dendrite, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (*idServerStoreInviteResponse, error) {
 	// Retrieve the sender's profile to get their display name
@@ -329,7 +330,7 @@ func checkIDServerSignatures(
 func emit3PIDInviteEvent(
 	ctx context.Context,
 	body *MembershipRequest, res *idServerStoreInviteResponse,
-	device *authtypes.Device, roomID string, cfg *config.Dendrite,
+	device *userapi.Device, roomID string, cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
 	evTime time.Time,
 ) error {

cmd/client-api-proxy/main.go

@@ -58,9 +58,8 @@ var (
 )
 
 func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
-	if !strings.HasSuffix(targetURL, "/") {
-		targetURL += "/"
-	}
+	targetURL = strings.TrimSuffix(targetURL, "/")
+
 	// Check that we can parse the URL.
 	_, err := url.Parse(targetURL)
 	if err != nil {

cmd/create-account/main.go

@@ -20,8 +20,8 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 

cmd/create-room-events/main.go

@@ -47,6 +47,7 @@ var (
 	userID           = flag.String("user-id", "@userid:$SERVER_NAME", "The user ID to use as the event sender")
 	messageCount     = flag.Int("message-count", 10, "The number of m.room.messsage events to generate")
 	format           = flag.String("Format", "InputRoomEvent", "The output format to use for the messages: InputRoomEvent or Event")
+	ver              = flag.String("version", string(gomatrixserverlib.RoomVersionV1), "Room version to generate events as")
 )
 
 // By default we use a private key of 0.
@@ -109,7 +110,7 @@ func buildAndOutput() gomatrixserverlib.EventReference {
 
 	event, err := b.Build(
 		now, name, key, privateKey,
-		gomatrixserverlib.RoomVersionV1,
+		gomatrixserverlib.RoomVersion(*ver),
 	)
 	if err != nil {
 		panic(err)
@@ -127,7 +128,7 @@ func writeEvent(event gomatrixserverlib.Event) {
 	if *format == "InputRoomEvent" {
 		var ire api.InputRoomEvent
 		ire.Kind = api.KindNew
-		ire.Event = event.Headered(gomatrixserverlib.RoomVersionV1)
+		ire.Event = event.Headered(gomatrixserverlib.RoomVersion(*ver))
 		authEventIDs := []string{}
 		for _, ref := range b.AuthEvents.([]gomatrixserverlib.EventReference) {
 			authEventIDs = append(authEventIDs, ref.EventID)

cmd/dendrite-appservice-server/main.go

@@ -24,11 +24,10 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true)
 
 	defer base.Close() // nolint: errcheck
-	accountDB := base.CreateAccountsDB()
-	deviceDB := base.CreateDeviceDB()
+	userAPI := base.UserAPIClient()
 	rsAPI := base.RoomserverHTTPClient()
 
-	intAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI)
+	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI))

cmd/dendrite-client-api-server/main.go

@@ -30,17 +30,15 @@ func main() {
 	deviceDB := base.CreateDeviceDB()
 	federation := base.CreateFederationClient()
 
-	serverKeyAPI := base.ServerKeyAPIClient()
-	keyRing := serverKeyAPI.KeyRing()
-
 	asQuery := base.AppserviceHTTPClient()
 	rsAPI := base.RoomserverHTTPClient()
 	fsAPI := base.FederationSenderHTTPClient()
 	eduInputAPI := base.EDUServerClient()
+	userAPI := base.UserAPIClient()
 
 	clientapi.AddPublicRoutes(
-		base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation, keyRing,
-		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI,
+		base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation,
+		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))

cmd/dendrite-demo-libp2p/main.go

@@ -37,6 +37,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/serverkeyapi"
+	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/eduserver/cache"
@@ -79,6 +80,17 @@ func createFederationClient(
 	)
 }
 
+func createClient(
+	base *P2PDendrite,
+) *gomatrixserverlib.Client {
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix",
+		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
+	)
+	return gomatrixserverlib.NewClientWithTransport(tr)
+}
+
 func main() {
 	instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance")
 	instancePort := flag.Int("port", 8080, "the port that the client API will listen on")
@@ -101,6 +113,7 @@ func main() {
 	}
 
 	cfg := config.Dendrite{}
+	cfg.SetDefaults()
 	cfg.Matrix.ServerName = "p2p"
 	cfg.Matrix.PrivateKey = privKey
 	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
@@ -128,6 +141,7 @@ func main() {
 	accountDB := base.Base.CreateAccountsDB()
 	deviceDB := base.Base.CreateDeviceDB()
 	federation := createFederationClient(base)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil)
 
 	serverKeyAPI := serverkeyapi.NewInternalAPI(
 		base.Base.Cfg, federation, base.Base.Caches,
@@ -141,9 +155,9 @@ func main() {
 		&base.Base, keyRing, federation,
 	)
 	eduInputAPI := eduserver.NewInternalAPI(
-		&base.Base, cache.New(), deviceDB,
+		&base.Base, cache.New(), userAPI,
 	)
-	asAPI := appservice.NewInternalAPI(&base.Base, accountDB, deviceDB, rsAPI)
+	asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
 	fsAPI := federationsender.NewInternalAPI(
 		&base.Base, federation, rsAPI, keyRing,
 	)
@@ -157,6 +171,7 @@ func main() {
 		Config:        base.Base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
+		Client:        createClient(base),
 		FedClient:     federation,
 		KeyRing:       keyRing,
 		KafkaConsumer: base.Base.KafkaConsumer,
@@ -167,13 +182,14 @@ func main() {
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
 		ServerKeyAPI:        serverKeyAPI,
+		UserAPI:             userAPI,
 
 		PublicRoomsDB: publicRoomsDB,
 	}
 	monolith.AddAllPublicRoutes(base.Base.PublicAPIMux)
 
 	httputil.SetupHTTPAPI(
-		http.DefaultServeMux,
+		base.Base.BaseMux,
 		base.Base.PublicAPIMux,
 		base.Base.InternalAPIMux,
 		&cfg,
@@ -184,7 +200,7 @@ func main() {
 	go func() {
 		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
 		logrus.Info("Listening on ", httpBindAddr)
-		logrus.Fatal(http.ListenAndServe(httpBindAddr, nil))
+		logrus.Fatal(http.ListenAndServe(httpBindAddr, base.Base.BaseMux))
 	}()
 	// Expose the matrix APIs also via libp2p
 	if base.LibP2P != nil {
@@ -197,7 +213,7 @@ func main() {
 			defer func() {
 				logrus.Fatal(listener.Close())
 			}()
-			logrus.Fatal(http.Serve(listener, nil))
+			logrus.Fatal(http.Serve(listener, base.Base.BaseMux))
 		}()
 	}
 

cmd/dendrite-demo-yggdrasil/embed/embed_other.go

@@ -2,6 +2,8 @@
 
 package embed
 
-func Embed(_ int, _ string) {
+import "github.com/gorilla/mux"
+
+func Embed(_ *mux.Router, _ int, _ string) {
 
 }

cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go

@@ -7,19 +7,20 @@ import (
 	"io"
 	"net/http"
 
+	"github.com/gorilla/mux"
 	"github.com/tidwall/sjson"
 )
 
 // From within the Riot Web directory:
 // go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_riotweb.go -private -pkg embed .
 
-func Embed(listenPort int, serverName string) {
+func Embed(rootMux *mux.Router, listenPort int, serverName string) {
 	url := fmt.Sprintf("http://localhost:%d", listenPort)
 	embeddedFS := _escFS(false)
 	embeddedServ := http.FileServer(embeddedFS)
 
-	http.DefaultServeMux.Handle("/", embeddedServ)
-	http.DefaultServeMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) {
+	rootMux.Handle("/", embeddedServ)
+	rootMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) {
 		configFile, err := embeddedFS.Open("/config.sample.json")
 		if err != nil {
 			w.WriteHeader(500)

cmd/dendrite-demo-yggdrasil/main.go

@@ -16,18 +16,14 @@ package main
 
 import (
 	"context"
-	"crypto/ed25519"
 	"crypto/tls"
-	"encoding/hex"
 	"flag"
 	"fmt"
 	"net"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
@@ -39,6 +35,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/sirupsen/logrus"
@@ -50,63 +47,11 @@ var (
 	instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to")
 )
 
-type yggroundtripper struct {
-	inner *http.Transport
-}
-
-func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.URL.Scheme = "http"
-	return y.inner.RoundTrip(req)
-}
-
-func createFederationClient(
-	base *setup.BaseDendrite, n *yggconn.Node,
-) *gomatrixserverlib.FederationClient {
-	yggdialer := func(_, address string) (net.Conn, error) {
-		tokens := strings.Split(address, ":")
-		raw, err := hex.DecodeString(tokens[0])
-		if err != nil {
-			return nil, fmt.Errorf("hex.DecodeString: %w", err)
-		}
-		converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw))
-		convhex := hex.EncodeToString(converted)
-		return n.Dial("curve25519", convhex)
-	}
-	yggdialerctx := func(ctx context.Context, network, address string) (net.Conn, error) {
-		return yggdialer(network, address)
-	}
-	tr := &http.Transport{}
-	tr.RegisterProtocol(
-		"matrix", &yggroundtripper{
-			inner: &http.Transport{
-				ResponseHeaderTimeout: 15 * time.Second,
-				IdleConnTimeout:       60 * time.Second,
-				DialContext:           yggdialerctx,
-			},
-		},
-	)
-	return gomatrixserverlib.NewFederationClientWithTransport(
-		base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr,
-	)
-}
-
 // nolint:gocyclo
 func main() {
 	flag.Parse()
 
-	// Build both ends of a HTTP multiplex.
-	httpServer := &http.Server{
-		Addr:         ":0",
-		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
-		ReadTimeout:  15 * time.Second,
-		WriteTimeout: 45 * time.Second,
-		IdleTimeout:  60 * time.Second,
-		BaseContext: func(_ net.Listener) context.Context {
-			return context.Background()
-		},
-	}
-
-	ygg, err := yggconn.Setup(*instanceName, *instancePeer)
+	ygg, err := yggconn.Setup(*instanceName, *instancePeer, ".")
 	if err != nil {
 		panic(err)
 	}
@@ -139,21 +84,23 @@ func main() {
 
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
-	federation := createFederationClient(base, ygg)
+	federation := ygg.CreateFederationClient(base)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil)
+
 	rsComponent := roomserver.NewInternalAPI(
 		base, keyRing, federation,
 	)
 	rsAPI := rsComponent
 
 	eduInputAPI := eduserver.NewInternalAPI(
-		base, cache.New(), deviceDB,
+		base, cache.New(), userAPI,
 	)
 
-	asAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI)
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 
 	fsAPI := federationsender.NewInternalAPI(
 		base, federation, rsAPI, keyRing,
@@ -166,12 +113,13 @@ func main() {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
 
-	embed.Embed(*instancePort, "Yggdrasil Demo")
+	embed.Embed(base.BaseMux, *instancePort, "Yggdrasil Demo")
 
 	monolith := setup.Monolith{
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
+		Client:        ygg.CreateClient(base),
 		FedClient:     federation,
 		KeyRing:       keyRing,
 		KafkaConsumer: base.KafkaConsumer,
@@ -181,6 +129,7 @@ func main() {
 		EDUInternalAPI:      eduInputAPI,
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 
 		PublicRoomsDB: publicRoomsDB,
@@ -188,21 +137,34 @@ func main() {
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
 
 	httputil.SetupHTTPAPI(
-		http.DefaultServeMux,
+		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
 		cfg,
 		base.UseHTTPAPIs,
 	)
 
+	// Build both ends of a HTTP multiplex.
+	httpServer := &http.Server{
+		Addr:         ":0",
+		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+		ReadTimeout:  15 * time.Second,
+		WriteTimeout: 45 * time.Second,
+		IdleTimeout:  60 * time.Second,
+		BaseContext: func(_ net.Listener) context.Context {
+			return context.Background()
+		},
+		Handler: base.BaseMux,
+	}
+
 	go func() {
 		logrus.Info("Listening on ", ygg.DerivedServerName())
 		logrus.Fatal(httpServer.Serve(ygg))
 	}()
 	go func() {
-		httpBindAddr := fmt.Sprintf("localhost:%d", *instancePort)
+		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
 		logrus.Info("Listening on ", httpBindAddr)
-		logrus.Fatal(http.ListenAndServe(httpBindAddr, nil))
+		logrus.Fatal(http.ListenAndServe(httpBindAddr, base.BaseMux))
 	}()
 
 	select {}

cmd/dendrite-demo-yggdrasil/yggconn/client.go

@@ -0,0 +1,74 @@
+package yggconn
+
+import (
+	"context"
+	"crypto/ed25519"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func (n *Node) yggdialer(_, address string) (net.Conn, error) {
+	tokens := strings.Split(address, ":")
+	raw, err := hex.DecodeString(tokens[0])
+	if err != nil {
+		return nil, fmt.Errorf("hex.DecodeString: %w", err)
+	}
+	converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw))
+	convhex := hex.EncodeToString(converted)
+	return n.Dial("curve25519", convhex)
+}
+
+func (n *Node) yggdialerctx(ctx context.Context, network, address string) (net.Conn, error) {
+	return n.yggdialer(network, address)
+}
+
+type yggroundtripper struct {
+	inner *http.Transport
+}
+
+func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.URL.Scheme = "http"
+	return y.inner.RoundTrip(req)
+}
+
+func (n *Node) CreateClient(
+	base *setup.BaseDendrite,
+) *gomatrixserverlib.Client {
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix", &yggroundtripper{
+			inner: &http.Transport{
+				ResponseHeaderTimeout: 15 * time.Second,
+				IdleConnTimeout:       60 * time.Second,
+				DialContext:           n.yggdialerctx,
+			},
+		},
+	)
+	return gomatrixserverlib.NewClientWithTransport(tr)
+}
+
+func (n *Node) CreateFederationClient(
+	base *setup.BaseDendrite,
+) *gomatrixserverlib.FederationClient {
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix", &yggroundtripper{
+			inner: &http.Transport{
+				ResponseHeaderTimeout: 15 * time.Second,
+				IdleConnTimeout:       60 * time.Second,
+				DialContext:           n.yggdialerctx,
+			},
+		},
+	)
+	return gomatrixserverlib.NewFederationClientWithTransport(
+		base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr,
+	)
+}

cmd/dendrite-demo-yggdrasil/yggconn/node.go

@@ -15,13 +15,16 @@
 package yggconn
 
 import (
+	"context"
 	"crypto/ed25519"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
@@ -48,8 +51,23 @@ type Node struct {
 	incoming  chan *yamux.Stream
 }
 
+func (n *Node) Dialer(_, address string) (net.Conn, error) {
+	tokens := strings.Split(address, ":")
+	raw, err := hex.DecodeString(tokens[0])
+	if err != nil {
+		return nil, fmt.Errorf("hex.DecodeString: %w", err)
+	}
+	converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw))
+	convhex := hex.EncodeToString(converted)
+	return n.Dial("curve25519", convhex)
+}
+
+func (n *Node) DialerContext(ctx context.Context, network, address string) (net.Conn, error) {
+	return n.Dialer(network, address)
+}
+
 // nolint:gocyclo
-func Setup(instanceName, instancePeer string) (*Node, error) {
+func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) {
 	n := &Node{
 		core:      &yggdrasil.Core{},
 		config:    yggdrasilconfig.GenerateConfig(),
@@ -59,7 +77,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) {
 		incoming:  make(chan *yamux.Stream),
 	}
 
-	yggfile := fmt.Sprintf("%s-yggdrasil.conf", instanceName)
+	yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
 	if _, err := os.Stat(yggfile); !os.IsNotExist(err) {
 		yggconf, e := ioutil.ReadFile(yggfile)
 		if e != nil {
@@ -69,7 +87,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) {
 			panic(err)
 		}
 	} else {
-		n.config.AdminListen = fmt.Sprintf("unix://./%s-yggdrasil.sock", instanceName)
+		n.config.AdminListen = "none" // fmt.Sprintf("unix://%s/%s-yggdrasil.sock", storageDirectory, instanceName)
 		n.config.MulticastInterfaces = []string{".*"}
 		n.config.EncryptionPrivateKey = hex.EncodeToString(n.EncryptionPrivateKey())
 		n.config.EncryptionPublicKey = hex.EncodeToString(n.EncryptionPublicKey())
@@ -96,20 +114,22 @@ func Setup(instanceName, instancePeer string) (*Node, error) {
 			panic(err)
 		}
 	}
-	if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil {
-		panic(err)
-	}
-	if err = n.admin.Start(); err != nil {
-		panic(err)
-	}
+	/*
+		if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil {
+			panic(err)
+		}
+		if err = n.admin.Start(); err != nil {
+			panic(err)
+		}
+	*/
 	if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil {
 		panic(err)
 	}
 	if err = n.multicast.Start(); err != nil {
 		panic(err)
 	}
-	n.admin.SetupAdminHandlers(n.admin)
-	n.multicast.SetupAdminHandlers(n.admin)
+	//n.admin.SetupAdminHandlers(n.admin)
+	//n.multicast.SetupAdminHandlers(n.admin)
 	n.listener, err = n.core.ConnListen()
 	if err != nil {
 		panic(err)
@@ -119,6 +139,9 @@ func Setup(instanceName, instancePeer string) (*Node, error) {
 		panic(err)
 	}
 
+	n.log.Println("Public curve25519:", n.core.EncryptionPublicKey())
+	n.log.Println("Public ed25519:", n.core.SigningPublicKey())
+
 	go n.listenFromYgg()
 
 	return n, nil

cmd/dendrite-edu-server/main.go

@@ -29,9 +29,8 @@ func main() {
 			logrus.WithError(err).Warn("BaseDendrite close failed")
 		}
 	}()
-	deviceDB := base.CreateDeviceDB()
 
-	intAPI := eduserver.NewInternalAPI(base, cache.New(), deviceDB)
+	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
 	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer))

cmd/dendrite-federation-api-server/main.go

@@ -24,18 +24,16 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "FederationAPI", true)
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
-	deviceDB := base.CreateDeviceDB()
+	userAPI := base.UserAPIClient()
 	federation := base.CreateFederationClient()
 	serverKeyAPI := base.ServerKeyAPIClient()
 	keyRing := serverKeyAPI.KeyRing()
 	fsAPI := base.FederationSenderHTTPClient()
 	rsAPI := base.RoomserverHTTPClient()
-	asAPI := base.AppserviceHTTPClient()
 
 	federationapi.AddPublicRoutes(
-		base.PublicAPIMux, base.Cfg, accountDB, deviceDB, federation, keyRing,
-		rsAPI, asAPI, fsAPI, base.EDUServerClient(),
+		base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing,
+		rsAPI, fsAPI, base.EDUServerClient(),
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))

cmd/dendrite-key-server/main.go

@@ -24,10 +24,9 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "KeyServer", true)
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
-	deviceDB := base.CreateDeviceDB()
+	userAPI := base.UserAPIClient()
 
-	keyserver.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB, accountDB)
+	keyserver.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer))
 

cmd/dendrite-media-api-server/main.go

@@ -17,6 +17,7 @@ package main
 import (
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/mediaapi"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 func main() {
@@ -24,9 +25,10 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "MediaAPI", true)
 	defer base.Close() // nolint: errcheck
 
-	deviceDB := base.CreateDeviceDB()
+	userAPI := base.UserAPIClient()
+	client := gomatrixserverlib.NewClient()
 
-	mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB)
+	mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI, client)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI))
 

cmd/dendrite-monolith-server/main.go

@@ -30,6 +30,8 @@ import (
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/serverkeyapi"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/sirupsen/logrus"
 )
@@ -73,12 +75,15 @@ func main() {
 		serverKeyAPI = base.ServerKeyAPIClient()
 	}
 	keyRing := serverKeyAPI.KeyRing()
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices)
 
-	rsAPI := roomserver.NewInternalAPI(
+	rsImpl := roomserver.NewInternalAPI(
 		base, keyRing, federation,
 	)
+	// call functions directly on the impl unless running in HTTP mode
+	rsAPI := rsImpl
 	if base.UseHTTPAPIs {
-		roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
+		roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl)
 		rsAPI = base.RoomserverHTTPClient()
 	}
 	if traceInternal {
@@ -88,14 +93,14 @@ func main() {
 	}
 
 	eduInputAPI := eduserver.NewInternalAPI(
-		base, cache.New(), deviceDB,
+		base, cache.New(), userAPI,
 	)
 	if base.UseHTTPAPIs {
 		eduserver.AddInternalRoutes(base.InternalAPIMux, eduInputAPI)
 		eduInputAPI = base.EDUServerClient()
 	}
 
-	asAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI)
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	if base.UseHTTPAPIs {
 		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
 		asAPI = base.AppserviceHTTPClient()
@@ -108,7 +113,9 @@ func main() {
 		federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI)
 		fsAPI = base.FederationSenderHTTPClient()
 	}
-	rsAPI.SetFederationSenderAPI(fsAPI)
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this dependency
+	rsImpl.SetFederationSenderAPI(fsAPI)
 
 	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
 	if err != nil {
@@ -119,6 +126,7 @@ func main() {
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
+		Client:        gomatrixserverlib.NewClient(),
 		FedClient:     federation,
 		KeyRing:       keyRing,
 		KafkaConsumer: base.KafkaConsumer,
@@ -129,13 +137,14 @@ func main() {
 		FederationSenderAPI: fsAPI,
 		RoomserverAPI:       rsAPI,
 		ServerKeyAPI:        serverKeyAPI,
+		UserAPI:             userAPI,
 
 		PublicRoomsDB: publicRoomsDB,
 	}
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
 
 	httputil.SetupHTTPAPI(
-		http.DefaultServeMux,
+		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
 		cfg,
@@ -147,6 +156,7 @@ func main() {
 		serv := http.Server{
 			Addr:         *httpBindAddr,
 			WriteTimeout: setup.HTTPServerTimeout,
+			Handler:      base.BaseMux,
 		}
 
 		logrus.Info("Listening on ", serv.Addr)
@@ -158,6 +168,7 @@ func main() {
 			serv := http.Server{
 				Addr:         *httpsBindAddr,
 				WriteTimeout: setup.HTTPServerTimeout,
+				Handler:      base.BaseMux,
 			}
 
 			logrus.Info("Listening on ", serv.Addr)

cmd/dendrite-monolith-server/main_test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+// This is an instrumented main, used when running integration tests (sytest) with code coverage.
+// Compile: go test -c -race -cover -covermode=atomic -o monolith.debug -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server
+// Run the monolith: ./monolith.debug -test.coverprofile=/somewhere/to/dump/integrationcover.out DEVEL --config dendrite.yaml
+// Generate HTML with coverage: go tool cover -html=/somewhere/where/there/is/integrationcover.out -o cover.html
+// Source: https://dzone.com/articles/measuring-integration-test-coverage-rate-in-pouchc
+func TestMain(_ *testing.T) {
+	var (
+		args []string
+	)
+
+	for _, arg := range os.Args {
+		switch {
+		case strings.HasPrefix(arg, "DEVEL"):
+		case strings.HasPrefix(arg, "-test"):
+		default:
+			args = append(args, arg)
+		}
+	}
+	// only run the tests if there are args to be passed
+	if len(args) <= 1 {
+		return
+	}
+
+	waitCh := make(chan int, 1)
+	os.Args = args
+	go func() {
+		main()
+		close(waitCh)
+	}()
+
+	signalCh := make(chan os.Signal, 1)
+	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGHUP)
+
+	select {
+	case <-signalCh:
+		return
+	case <-waitCh:
+		return
+	}
+}

cmd/dendrite-public-rooms-api-server/main.go

@@ -26,7 +26,7 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "PublicRoomsAPI", true)
 	defer base.Close() // nolint: errcheck
 
-	deviceDB := base.CreateDeviceDB()
+	userAPI := base.UserAPIClient()
 
 	rsAPI := base.RoomserverHTTPClient()
 
@@ -34,7 +34,7 @@ func main() {
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
-	publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, deviceDB, publicRoomsDB, rsAPI, nil, nil)
+	publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, userAPI, publicRoomsDB, rsAPI, nil, nil)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI))
 

cmd/dendrite-sync-api-server/main.go

@@ -24,13 +24,12 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "SyncAPI", true)
 	defer base.Close() // nolint: errcheck
 
-	deviceDB := base.CreateDeviceDB()
-	accountDB := base.CreateAccountsDB()
+	userAPI := base.UserAPIClient()
 	federation := base.CreateFederationClient()
 
 	rsAPI := base.RoomserverHTTPClient()
 
-	syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, deviceDB, accountDB, rsAPI, federation, cfg)
+	syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, userAPI, rsAPI, federation, cfg)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI))
 

cmd/dendrite-user-api-server/main.go

@@ -12,19 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package authtypes
+package main
 
-// Device represents a client's device (mobile, web, etc)
-type Device struct {
-	ID     string
-	UserID string
-	// The access_token granted to this device.
-	// This uniquely identifies the device from all other devices and clients.
-	AccessToken string
-	// The unique ID of the session identified by the access token.
-	// Can be used as a secure substitution in places where data needs to be
-	// associated with access tokens.
-	SessionID int64
-	// TODO: display name, last used timestamp, keys, etc
-	DisplayName string
+import (
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/userapi"
+)
+
+func main() {
+	cfg := setup.ParseFlags(false)
+	base := setup.NewBaseDendrite(cfg, "UserAPI", true)
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	deviceDB := base.CreateDeviceDB()
+
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices)
+
+	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
+
+	base.SetupAndServeHTTP(string(base.Cfg.Bind.UserAPI), string(base.Cfg.Listen.UserAPI))
 }

cmd/dendritejs/jsServer.go

@@ -28,7 +28,7 @@ import (
 // JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it.
 type JSServer struct {
 	// The router which will service requests
-	Mux *http.ServeMux
+	Mux http.Handler
 }
 
 // OnRequestFromJS is the function that JS will invoke when there is a new request.

cmd/dendritejs/main.go

@@ -19,7 +19,6 @@ package main
 import (
 	"crypto/ed25519"
 	"fmt"
-	"net/http"
 	"syscall/js"
 
 	"github.com/matrix-org/dendrite/appservice"
@@ -31,6 +30,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/userapi"
 	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -145,6 +145,11 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc
 	return fed
 }
 
+func createClient(node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.Client {
+	tr := go_http_js_libp2p.NewP2pTransport(node)
+	return gomatrixserverlib.NewClientWithTransport(tr)
+}
+
 func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) {
 	hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/"
 	node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}, "p2p")
@@ -189,6 +194,7 @@ func main() {
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
 	federation := createFederationClient(cfg, node)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil)
 
 	fetcher := &libp2pKeyFetcher{}
 	keyRing := gomatrixserverlib.KeyRing{
@@ -199,9 +205,9 @@ func main() {
 	}
 
 	rsAPI := roomserver.NewInternalAPI(base, keyRing, federation)
-	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), deviceDB)
+	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI)
 	asQuery := appservice.NewInternalAPI(
-		base, accountDB, deviceDB, rsAPI,
+		base, userAPI, rsAPI,
 	)
 	fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing)
 	rsAPI.SetFederationSenderAPI(fedSenderAPI)
@@ -216,6 +222,7 @@ func main() {
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
+		Client:        createClient(node),
 		FedClient:     federation,
 		KeyRing:       &keyRing,
 		KafkaConsumer: base.KafkaConsumer,
@@ -225,6 +232,7 @@ func main() {
 		EDUInternalAPI:      eduInputAPI,
 		FederationSenderAPI: fedSenderAPI,
 		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 
 		PublicRoomsDB:          publicRoomsDB,
@@ -233,7 +241,7 @@ func main() {
 	monolith.AddAllPublicRoutes(base.PublicAPIMux)
 
 	httputil.SetupHTTPAPI(
-		http.DefaultServeMux,
+		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
 		cfg,
@@ -245,7 +253,7 @@ func main() {
 		go func() {
 			logrus.Info("Listening on libp2p-js host ID ", node.Id)
 			s := JSServer{
-				Mux: http.DefaultServeMux,
+				Mux: base.BaseMux,
 			}
 			s.ListenAndServe("p2p")
 		}()
@@ -255,7 +263,7 @@ func main() {
 	go func() {
 		logrus.Info("Listening for service-worker fetch traffic")
 		s := JSServer{
-			Mux: http.DefaultServeMux,
+			Mux: base.BaseMux,
 		}
 		s.ListenAndServe("fetch")
 	}()

cmd/roomserver-integration-tests/main.go

@@ -255,7 +255,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 		panic(err)
 	}
 
-	cache, err := caching.NewInMemoryLRUCache()
+	cache, err := caching.NewInMemoryLRUCache(false)
 	if err != nil {
 		panic(err)
 	}

dendrite-config.yaml

@@ -108,7 +108,9 @@ kafka:
         output_send_to_device_event: eduServerSendToDeviceOutput
         user_updates: userUpdates
 
-# The postgres connection configs for connecting to the databases e.g a postgres:// URI
+# The postgres connection configs for connecting to the databases, e.g.
+# for Postgres: postgres://username:password@hostname/database
+# for SQLite: file:filename.db or file:///path/to/filename.db
 database:
     account: "postgres://dendrite:itsasecret@localhost/dendrite_account?sslmode=disable"
     device: "postgres://dendrite:itsasecret@localhost/dendrite_device?sslmode=disable"
@@ -122,7 +124,7 @@ database:
     max_open_conns: 100
     max_idle_conns: 2
     conn_max_lifetime: -1
-    # If using naffka you need to specify a naffka database
+    # If 'use_naffka: true' set above then you need to specify a naffka database
     # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable"
 
 # The TCP host:port pairs to bind the internal HTTP APIs to.
@@ -140,6 +142,7 @@ listen:
     edu_server: "localhost:7778"
     key_server: "localhost:7779"
     server_key_api: "localhost:7780"
+    user_api: "localhost:7781"
 
 # The configuration for tracing the dendrite components.
 tracing:

docs/INSTALL.md

@@ -329,3 +329,13 @@ finished).
 ```bash
 ./bin/dendrite-key-server --config dendrite.yaml
 ```
+
+### User server
+
+This manages user accounts, device access tokens and user account data,
+amongst other things.
+
+```bash
+./bin/dendrite-user-api-server --config dendrite.yaml
+```
+

docs/WIRING-Current.md

@@ -0,0 +1,73 @@
+This document details how various components communicate with each other. There are two kinds of components:
+ - Public-facing: exposes CS/SS API endpoints and need to be routed to via client-api-proxy or equivalent.
+ - Internal-only: exposes internal APIs and produces Kafka events.
+
+## Internal HTTP APIs
+
+Not everything can be done using Kafka logs. For example, requesting the latest events in a room is much better suited to
+a request/response model like HTTP or RPC. Therefore, components can expose "internal APIs" which sit outside of Kafka logs.
+Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests.
+
+```
+   Tier 1            Sync        PublicRooms      FederationAPI      ClientAPI    MediaAPI
+Public Facing        | .-----1------`              | | |    | |        | | | |
+                     2 | .-------3-----------------` | |    | `--------|-|-|-|--11--------------------.
+                     | | | .--------4----------------------------------` | | |                        |
+                     | | | |         .---5-----------` |    |            | | |                        |
+                     | | | |         |  .---6----------------------------` | |                        |
+                     | | | |         |  |              |  .-----7----------` |                        |
+                     | | | |         |  |              8  | |                10                       |
+                     | | | |         |  |              |  | `---9----.       |                        |
+                     V V V V         V  V              V  V          V       V                        V
+   Tier 2           Roomserver     EDUServer         FedSender       AppService    KeyServer   ServerKeyAPI
+Internal only               |                               `------------------------12----------^   ^
+                            `------------------------------------------------------------13----------`
+
+ Client ---> Server
+```
+- 1 (PublicRooms -> Roomserver): Calculating current auth for changing visibility
+- 2 (Sync -> Roomserver): When making backfill requests
+- 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests
+- 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests
+- 5 (FedAPI -> EDUServer): Sending typing/send-to-device events
+- 6 (ClientAPI -> EDUServer): Sending typing/send-to-device events
+- 7 (ClientAPI -> FedSender): Handling directory lookups
+- 8 (FedAPI -> FedSender): Resetting backoffs when receiving traffic from a server. Querying joined hosts when handling alias lookup requests
+- 9 (FedAPI -> AppService): Working out if the client is an appservice user
+- 10 (ClientAPI -> AppService): Working out if the client is an appservice user
+- 11 (FedAPI -> ServerKeyAPI): Verifying incoming event signatures
+- 12 (FedSender -> ServerKeyAPI): Verifying event signatures of responses (e.g from send_join)
+- 13 (Roomserver -> ServerKeyAPI): Verifying event signatures of backfilled events
+
+In addition to this, all public facing components (Tier 1) talk to the `UserAPI` to verify access tokens and extract profile information where needed.
+
+## Kafka logs
+
+```
+                       .----1--------------------------------------------.
+                       V                                                 |
+   Tier 1            Sync        PublicRooms      FederationAPI      ClientAPI    MediaAPI
+Public Facing        ^   ^           ^                                 ^  
+                     |   |           |                                 |
+                     2   |           |                                 |
+                     |   `-3------------.                              |
+                     |               |  |                              |
+                     |               |  |                              |
+                     | .------4------`  |                              |
+                     | | .--------5-----|------------------------------`             
+                     | | |              |           
+   Tier 2           Roomserver     EDUServer         FedSender       AppService    KeyServer   ServerKeyAPI
+Internal only              |          |                ^ ^
+                           |          `-----6----------` |
+                           `--------------------7--------`
+
+
+Producer ----> Consumer
+```
+- 1 (ClientAPI -> Sync): For tracking account data
+- 2 (Roomserver -> Sync): For all data to send to clients
+- 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients
+- 4 (Roomserver -> PublicRooms): For tracking the current room name/topic/joined count/etc.
+- 5 (Roomserver -> ClientAPI): For tracking memberships for profile updates.
+- 6 (EDUServer -> FedSender): For sending EDUs over federation
+- 7 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts.

eduserver/eduserver.go

@@ -18,12 +18,12 @@ package eduserver
 
 import (
 	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/eduserver/input"
 	"github.com/matrix-org/dendrite/eduserver/inthttp"
 	"github.com/matrix-org/dendrite/internal/setup"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
@@ -37,11 +37,11 @@ func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI)
 func NewInternalAPI(
 	base *setup.BaseDendrite,
 	eduCache *cache.EDUCache,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 ) api.EDUServerInputAPI {
 	return &input.EDUServerInputAPI{
 		Cache:                        eduCache,
-		DeviceDB:                     deviceDB,
+		UserAPI:                      userAPI,
 		Producer:                     base.KafkaProducer,
 		OutputTypingEventTopic:       string(base.Cfg.Kafka.Topics.OutputTypingEvent),
 		OutputSendToDeviceEventTopic: string(base.Cfg.Kafka.Topics.OutputSendToDeviceEvent),

eduserver/input/input.go

@@ -22,9 +22,9 @@ import (
 	"time"
 
 	"github.com/Shopify/sarama"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/eduserver/cache"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
@@ -39,8 +39,8 @@ type EDUServerInputAPI struct {
 	OutputSendToDeviceEventTopic string
 	// kafka producer
 	Producer sarama.SyncProducer
-	// device database
-	DeviceDB devices.Database
+	// Internal user query API
+	UserAPI userapi.UserInternalAPI
 	// our server name
 	ServerName gomatrixserverlib.ServerName
 }
@@ -115,7 +115,7 @@ func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error {
 
 func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) error {
 	devices := []string{}
-	localpart, domain, err := gomatrixserverlib.SplitID('@', ise.UserID)
+	_, domain, err := gomatrixserverlib.SplitID('@', ise.UserID)
 	if err != nil {
 		return err
 	}
@@ -126,11 +126,14 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e
 	// wildcard as we don't know about the remote devices, so instead we leave it
 	// as-is, so that the federation sender can send it on with the wildcard intact.
 	if domain == t.ServerName && ise.DeviceID == "*" {
-		devs, err := t.DeviceDB.GetDevicesByLocalpart(context.TODO(), localpart)
+		var res userapi.QueryDevicesResponse
+		err = t.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{
+			UserID: ise.UserID,
+		}, &res)
 		if err != nil {
 			return err
 		}
-		for _, dev := range devs {
+		for _, dev := range res.Devices {
 			devices = append(devices, dev.ID)
 		}
 	} else {

federationapi/federationapi.go

@@ -16,13 +16,11 @@ package federationapi
 
 import (
 	"github.com/gorilla/mux"
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -32,19 +30,17 @@ import (
 func AddPublicRoutes(
 	router *mux.Router,
 	cfg *config.Dendrite,
-	accountsDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
-	keyRing *gomatrixserverlib.KeyRing,
+	keyRing gomatrixserverlib.JSONVerifier,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
 	federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
 ) {
 
 	routing.Setup(
-		router, cfg, rsAPI, asAPI,
-		eduAPI, federationSenderAPI, *keyRing,
-		federation, accountsDB, deviceDB,
+		router, cfg, rsAPI,
+		eduAPI, federationSenderAPI, keyRing,
+		federation, userAPI,
 	)
 }

federationapi/federationapi_test.go

@@ -0,0 +1,102 @@
+package federationapi_test
+
+import (
+	"context"
+	"crypto/ed25519"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/internal/test"
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404.
+// Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated.
+func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
+	_, privKey, _ := ed25519.GenerateKey(nil)
+	cfg := &config.Dendrite{}
+	cfg.Matrix.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
+	cfg.Matrix.ServerName = gomatrixserverlib.ServerName("localhost")
+	cfg.Matrix.PrivateKey = privKey
+	cfg.Kafka.UseNaffka = true
+	cfg.Database.Naffka = "file::memory:"
+	cfg.SetDefaults()
+	base := setup.NewBaseDendrite(cfg, "Test", false)
+	keyRing := &test.NopJSONVerifier{}
+	fsAPI := base.FederationSenderHTTPClient()
+	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
+	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
+	federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil)
+	httputil.SetupHTTPAPI(
+		base.BaseMux,
+		base.PublicAPIMux,
+		base.InternalAPIMux,
+		cfg,
+		base.UseHTTPAPIs,
+	)
+	baseURL, cancel := test.ListenAndServe(t, base.BaseMux, true)
+	defer cancel()
+	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
+
+	fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey)
+
+	testCases := []struct {
+		roomVer   gomatrixserverlib.RoomVersion
+		eventJSON string
+	}{
+		{
+			eventJSON: `{"auth_events":[["$Nzfbrhc3oaYVKzGM:localhost",{"sha256":"BCBHOgB4qxLPQkBd6th8ydFSyqjth/LF99VNjYffOQ0"}],["$EZzkD2BH1Gtm5v1D:localhost",{"sha256":"3dLUnDBs8/iC5DMw/ydKtmAqVZtzqqtHpsjsQPk7GJA"}]],"content":{"body":"Test Message"},"depth":11,"event_id":"$mGiPO3oGjQfCkIUw:localhost","hashes":{"sha256":"h+t+4DwIBC9UNyJ3jzyAQAAl4H3yQHVuHrm2S1JZizU"},"origin":"localhost","origin_server_ts":0,"prev_events":[["$tFr64vpiSHdLU0Qr:localhost",{"sha256":"+R07ZrIs4c4tjPFE+tmcYIGUfeLGFI/4e0OITb9uEcM"}]],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"LYFr/rW9m5/7UKBQMF5qWnG82He4VGsRESUgDmvkn5DrJRyS4TLL/7zl0Lymn3pa3q2yaTO74LQX/CRotqG1BA"}},"type":"m.room.message"}`,
+			roomVer:   gomatrixserverlib.RoomVersionV1,
+		},
+		// single / (handlers which do not UseEncodedPath will fail this test)
+		// EventID: $0SFh2WJbjBs3OT+E0yl95giDKo/3Zp52HsHUUk4uPyg
+		{
+			eventJSON: `{"auth_events":["$x4MKEPRSF6OGlo0qpnsP3BfSmYX5HhVlykOsQH3ECyg","$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"content":{"body":"Test Message"},"depth":8,"hashes":{"sha256":"dfK0MBn1RZZqCVJqWsn/MGY7QJHjQcwqF0unOonLCTU"},"origin":"localhost","origin_server_ts":0,"prev_events":["$1SwcZ1XY/Y8yKLjP4DzAOHN5WFBcDAZxb5vFDnW2ubA"],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"INOjuWMg+GmFkUpmzhMB0bqLNs73mSvwldY1ftYIQ/B3lD9soD2OMG3AF+wgZW/I8xqzY4DOHfbnbUeYPf67BA"}},"type":"m.room.message"}`,
+			roomVer:   gomatrixserverlib.RoomVersionV3,
+		},
+		// multiple /
+		// EventID: $OzENBCuVv/fnRAYCeQudIon/84/V5pxtEjQMTgi3emk
+		{
+			eventJSON: `{"auth_events":["$x4MKEPRSF6OGlo0qpnsP3BfSmYX5HhVlykOsQH3ECyg","$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"content":{"body":"Test Message"},"depth":2,"hashes":{"sha256":"U5+WsiJAhiEM88J8HTjuUjPImVGVzDFD3v/WS+jb2f0"},"origin":"localhost","origin_server_ts":0,"prev_events":["$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"tKS469e9+wdWPEKB/LbBJWQ8vfOOdKgTWER5IwbSAH1CxmLvkCziUsgVu85zfzDSLoUi5mU5FHLiMTC6P/qICw"}},"type":"m.room.message"}`,
+			roomVer:   gomatrixserverlib.RoomVersionV3,
+		},
+		// two slashes (handlers which clean paths before UseEncodedPath will fail this test)
+		// EventID: $EmwNBlHoSOVmCZ1cM//yv/OvxB6r4OFEIGSJea7+Amk
+		{
+			eventJSON: `{"auth_events":["$x4MKEPRSF6OGlo0qpnsP3BfSmYX5HhVlykOsQH3ECyg","$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"content":{"body":"Test Message"},"depth":3917,"hashes":{"sha256":"cNAWtlHIegrji0mMA6x1rhpYCccY8W1NsWZqSpJFhjs"},"origin":"localhost","origin_server_ts":0,"prev_events":["$4GDB0bVjkWwS3G4noUZCq5oLWzpBYpwzdMcf7gj24CI"],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"NKym6Kcy3u9mGUr21Hjfe3h7DfDilDhN5PqztT0QZ4NTZ+8Y7owseLolQVXp+TvNjecvzdDywsXXVvGiuQiWAQ"}},"type":"m.room.message"}`,
+			roomVer:   gomatrixserverlib.RoomVersionV3,
+		},
+	}
+
+	for _, tc := range testCases {
+		ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(tc.eventJSON), false, tc.roomVer)
+		if err != nil {
+			t.Errorf("failed to parse event: %s", err)
+		}
+		he := ev.Headered(tc.roomVer)
+		invReq, err := gomatrixserverlib.NewInviteV2Request(&he, nil)
+		if err != nil {
+			t.Errorf("failed to create invite v2 request: %s", err)
+			continue
+		}
+		_, err = fedCli.SendInviteV2(context.Background(), serverName, invReq)
+		if err == nil {
+			t.Errorf("expected an error, got none")
+			continue
+		}
+		gerr, ok := err.(gomatrix.HTTPError)
+		if !ok {
+			t.Errorf("failed to cast response error as gomatrix.HTTPError")
+			continue
+		}
+		t.Logf("Error: %+v", gerr)
+		if gerr.Code == 404 {
+			t.Errorf("invite event resulted in a 404")
+		}
+	}
+}

federationapi/routing/devices.go

@@ -15,9 +15,8 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/clientapi/userutil"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -25,17 +24,9 @@ import (
 // GetUserDevices for the given user id
 func GetUserDevices(
 	req *http.Request,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 	userID string,
 ) util.JSONResponse {
-	localpart, err := userutil.ParseUsernameParam(userID, nil)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue("Invalid user ID"),
-		}
-	}
-
 	response := gomatrixserverlib.RespUserDevices{
 		UserID: userID,
 		// TODO: we should return an incrementing stream ID each time the device
@@ -43,13 +34,16 @@ func GetUserDevices(
 		StreamID: 0,
 	}
 
-	devs, err := deviceDB.GetDevicesByLocalpart(req.Context(), localpart)
+	var res userapi.QueryDevicesResponse
+	err := userAPI.QueryDevices(req.Context(), &userapi.QueryDevicesRequest{
+		UserID: userID,
+	}, &res)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDevicesByLocalPart failed")
+		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryDevices failed")
 		return jsonerror.InternalServerError()
 	}
 
-	for _, dev := range devs {
+	for _, dev := range res.Devices {
 		device := gomatrixserverlib.RespUserDevice{
 			DeviceID:    dev.ID,
 			DisplayName: dev.DisplayName,

federationapi/routing/events.go

@@ -33,7 +33,11 @@ func GetEvent(
 	eventID string,
 	origin gomatrixserverlib.ServerName,
 ) util.JSONResponse {
-	event, err := getEvent(ctx, request, rsAPI, eventID)
+	err := allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID)
+	if err != nil {
+		return *err
+	}
+	event, err := fetchEvent(ctx, rsAPI, eventID)
 	if err != nil {
 		return *err
 	}
@@ -47,35 +51,40 @@ func GetEvent(
 	}}
 }
 
-// getEvent returns the requested event,
+// allowedToSeeEvent returns no error if the server is allowed to see this event,
 // otherwise it returns an error response which can be sent to the client.
-func getEvent(
+func allowedToSeeEvent(
 	ctx context.Context,
-	request *gomatrixserverlib.FederationRequest,
+	origin gomatrixserverlib.ServerName,
 	rsAPI api.RoomserverInternalAPI,
 	eventID string,
-) (*gomatrixserverlib.Event, *util.JSONResponse) {
+) *util.JSONResponse {
 	var authResponse api.QueryServerAllowedToSeeEventResponse
 	err := rsAPI.QueryServerAllowedToSeeEvent(
 		ctx,
 		&api.QueryServerAllowedToSeeEventRequest{
 			EventID:    eventID,
-			ServerName: request.Origin(),
+			ServerName: origin,
 		},
 		&authResponse,
 	)
 	if err != nil {
 		resErr := util.ErrorResponse(err)
-		return nil, &resErr
+		return &resErr
 	}
 
 	if !authResponse.AllowedToSeeEvent {
 		resErr := util.MessageResponse(http.StatusForbidden, "server not allowed to see event")
-		return nil, &resErr
+		return &resErr
 	}
 
+	return nil
+}
+
+// fetchEvent fetches the event without auth checks. Returns an error if the event cannot be found.
+func fetchEvent(ctx context.Context, rsAPI api.RoomserverInternalAPI, eventID string) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	var eventsResponse api.QueryEventsByIDResponse
-	err = rsAPI.QueryEventsByID(
+	err := rsAPI.QueryEventsByID(
 		ctx,
 		&api.QueryEventsByIDRequest{EventIDs: []string{eventID}},
 		&eventsResponse,

federationapi/routing/invite.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -27,15 +28,15 @@ import (
 	"github.com/matrix-org/util"
 )
 
-// Invite implements /_matrix/federation/v2/invite/{roomID}/{eventID}
-func Invite(
+// InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID}
+func InviteV2(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
 	eventID string,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
 	inviteReq := gomatrixserverlib.InviteV2Request{}
 	if err := json.Unmarshal(request.Content(), &inviteReq); err != nil {
@@ -44,14 +45,58 @@ func Invite(
 			JSON: jsonerror.NotJSON("The request body could not be decoded into an invite request. " + err.Error()),
 		}
 	}
-	event := inviteReq.Event()
+	return processInvite(
+		httpReq.Context(), inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys,
+	)
+}
+
+// InviteV1 implements /_matrix/federation/v1/invite/{roomID}/{eventID}
+func InviteV1(
+	httpReq *http.Request,
+	request *gomatrixserverlib.FederationRequest,
+	roomID string,
+	eventID string,
+	cfg *config.Dendrite,
+	rsAPI api.RoomserverInternalAPI,
+	keys gomatrixserverlib.JSONVerifier,
+) util.JSONResponse {
+	roomVer := gomatrixserverlib.RoomVersionV1
+	body := request.Content()
+	event, err := gomatrixserverlib.NewEventFromTrustedJSON(body, false, roomVer)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotJSON("The request body could not be decoded into an invite v1 request: " + err.Error()),
+		}
+	}
+	var strippedState []gomatrixserverlib.InviteV2StrippedState
+	if err := json.Unmarshal(event.Unsigned(), &strippedState); err != nil {
+		// just warn, they may not have added any.
+		util.GetLogger(httpReq.Context()).Warnf("failed to extract stripped state from invite event")
+	}
+	return processInvite(
+		httpReq.Context(), event, roomVer, strippedState, roomID, eventID, cfg, rsAPI, keys,
+	)
+}
+
+func processInvite(
+	ctx context.Context,
+	event gomatrixserverlib.Event,
+	roomVer gomatrixserverlib.RoomVersion,
+	strippedState []gomatrixserverlib.InviteV2StrippedState,
+	roomID string,
+	eventID string,
+	cfg *config.Dendrite,
+	rsAPI api.RoomserverInternalAPI,
+	keys gomatrixserverlib.JSONVerifier,
+) util.JSONResponse {
 
 	// Check that we can accept invites for this room version.
-	if _, err := roomserverVersion.SupportedRoomVersion(inviteReq.RoomVersion()); err != nil {
+	if _, err := roomserverVersion.SupportedRoomVersion(roomVer); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.UnsupportedRoomVersion(
-				fmt.Sprintf("Room version %q is not supported by this server.", inviteReq.RoomVersion()),
+				fmt.Sprintf("Room version %q is not supported by this server.", roomVer),
 			),
 		}
 	}
@@ -80,9 +125,9 @@ func Invite(
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}
-	verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests)
+	verifyResults, err := keys.VerifyJSONs(ctx, verifyRequests)
 	if err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("keys.VerifyJSONs failed")
+		util.GetLogger(ctx).WithError(err).Error("keys.VerifyJSONs failed")
 		return jsonerror.InternalServerError()
 	}
 	if verifyResults[0].Error != nil {
@@ -98,15 +143,11 @@ func Invite(
 	)
 
 	// Add the invite event to the roomserver.
-	if err = api.SendInvite(
-		httpReq.Context(), rsAPI,
-		signedEvent.Headered(inviteReq.RoomVersion()),
-		inviteReq.InviteRoomState(),
-		event.Origin(),
-		nil,
-	); err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed")
-		return jsonerror.InternalServerError()
+	if perr := api.SendInvite(
+		ctx, rsAPI, signedEvent.Headered(roomVer), strippedState, event.Origin(), nil,
+	); perr != nil {
+		util.GetLogger(ctx).WithError(err).Error("producer.SendInvite failed")
+		return perr.JSONResponse()
 	}
 
 	// Return the signed event to the originating server, it should then tell

federationapi/routing/join.go

@@ -143,7 +143,7 @@ func SendJoin(
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}

federationapi/routing/leave.go

@@ -113,7 +113,7 @@ func SendLeave(
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}

federationapi/routing/profile.go

@@ -18,11 +18,10 @@ import (
 	"fmt"
 	"net/http"
 
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -30,9 +29,8 @@ import (
 // GetProfile implements GET /_matrix/federation/v1/query/profile
 func GetProfile(
 	httpReq *http.Request,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
 	cfg *config.Dendrite,
-	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field")
 
@@ -60,9 +58,12 @@ func GetProfile(
 		}
 	}
 
-	profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB)
+	var profileRes userapi.QueryProfileResponse
+	err = userAPI.QueryProfile(httpReq.Context(), &userapi.QueryProfileRequest{
+		UserID: userID,
+	}, &profileRes)
 	if err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed")
+		util.GetLogger(httpReq.Context()).WithError(err).Error("userAPI.QueryProfile failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -73,11 +74,11 @@ func GetProfile(
 		switch field {
 		case "displayname":
 			res = eventutil.DisplayName{
-				DisplayName: profile.DisplayName,
+				DisplayName: profileRes.DisplayName,
 			}
 		case "avatar_url":
 			res = eventutil.AvatarURL{
-				AvatarURL: profile.AvatarURL,
+				AvatarURL: profileRes.AvatarURL,
 			}
 		default:
 			code = http.StatusBadRequest
@@ -85,8 +86,8 @@ func GetProfile(
 		}
 	} else {
 		res = eventutil.ProfileResponse{
-			AvatarURL:   profile.AvatarURL,
-			DisplayName: profile.DisplayName,
+			AvatarURL:   profileRes.AvatarURL,
+			DisplayName: profileRes.DisplayName,
 		}
 	}
 

federationapi/routing/routing.go

@@ -18,14 +18,13 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -48,13 +47,11 @@ func Setup(
 	publicAPIMux *mux.Router,
 	cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
 	fsAPI federationSenderAPI.FederationSenderInternalAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
-	accountDB accounts.Database,
-	deviceDB devices.Database,
+	userAPI userapi.UserInternalAPI,
 ) {
 	v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter()
 	v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter()
@@ -86,10 +83,26 @@ func Setup(
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
 
+	v1fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
+		"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
+		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
+			res := InviteV1(
+				httpReq, request, vars["roomID"], vars["eventID"],
+				cfg, rsAPI, keys,
+			)
+			return util.JSONResponse{
+				Code: res.Code,
+				JSON: []interface{}{
+					res.Code, res.JSON,
+				},
+			}
+		},
+	)).Methods(http.MethodPut, http.MethodOptions)
+
 	v2fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
 		"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
-			return Invite(
+			return InviteV2(
 				httpReq, request, vars["roomID"], vars["eventID"],
 				cfg, rsAPI, keys,
 			)
@@ -98,7 +111,7 @@ func Setup(
 
 	v1fedmux.Handle("/3pid/onbind", httputil.MakeExternalAPI("3pid_onbind",
 		func(req *http.Request) util.JSONResponse {
-			return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, federation, accountDB)
+			return CreateInvitesFrom3PIDInvites(req, rsAPI, cfg, federation, userAPI)
 		},
 	)).Methods(http.MethodPost, http.MethodOptions)
 
@@ -160,7 +173,7 @@ func Setup(
 		"federation_query_profile", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetProfile(
-				httpReq, accountDB, cfg, asAPI,
+				httpReq, userAPI, cfg,
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -169,7 +182,7 @@ func Setup(
 		"federation_user_devices", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetUserDevices(
-				httpReq, deviceDB, vars["userID"],
+				httpReq, userAPI, vars["userID"],
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -207,12 +220,20 @@ func Setup(
 			res := SendJoin(
 				httpReq, request, cfg, rsAPI, keys, roomID, eventID,
 			)
+			// not all responses get wrapped in [code, body]
+			var body interface{}
+			body = []interface{}{
+				res.Code, res.JSON,
+			}
+			jerr, ok := res.JSON.(*jsonerror.MatrixError)
+			if ok {
+				body = jerr
+			}
+
 			return util.JSONResponse{
 				Headers: res.Headers,
 				Code:    res.Code,
-				JSON: []interface{}{
-					res.Code, res.JSON,
-				},
+				JSON:    body,
 			}
 		},
 	)).Methods(http.MethodPut)

federationapi/routing/send.go

@@ -37,7 +37,7 @@ func Send(
 	cfg *config.Dendrite,
 	rsAPI api.RoomserverInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
-	keys gomatrixserverlib.KeyRing,
+	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
 	t := txnReq{
@@ -61,6 +61,14 @@ func Send(
 			JSON: jsonerror.NotJSON("The request body could not be decoded into valid JSON. " + err.Error()),
 		}
 	}
+	// Transactions are limited in size; they can have at most 50 PDUs and 100 EDUs.
+	// https://matrix.org/docs/spec/server_server/latest#transactions
+	if len(txnEvents.PDUs) > 50 || len(txnEvents.EDUs) > 100 {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("max 50 pdus / 100 edus"),
+		}
+	}
 
 	// TODO: Really we should have a function to convert FederationRequest to txnReq
 	t.PDUs = txnEvents.PDUs
@@ -71,10 +79,10 @@ func Send(
 
 	util.GetLogger(httpReq.Context()).Infof("Received transaction %q containing %d PDUs, %d EDUs", txnID, len(t.PDUs), len(t.EDUs))
 
-	resp, err := t.processTransaction()
-	if err != nil {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed")
-		return util.ErrorResponse(err)
+	resp, jsonErr := t.processTransaction()
+	if jsonErr != nil {
+		util.GetLogger(httpReq.Context()).WithField("jsonErr", jsonErr).Error("t.processTransaction failed")
+		return *jsonErr
 	}
 
 	// https://matrix.org/docs/spec/server_server/r0.1.3#put-matrix-federation-v1-send-txnid
@@ -112,7 +120,7 @@ type txnFederationClient interface {
 		roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
-func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
+func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, *util.JSONResponse) {
 	results := make(map[string]gomatrixserverlib.PDUResult)
 
 	pdus := []gomatrixserverlib.HeaderedEvent{}
@@ -136,10 +144,20 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
 		}
 		event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, verRes.RoomVersion)
 		if err != nil {
-			util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID())
-			results[event.EventID()] = gomatrixserverlib.PDUResult{
-				Error: err.Error(),
+			if _, ok := err.(gomatrixserverlib.BadJSONError); ok {
+				// Room version 6 states that homeservers should strictly enforce canonical JSON
+				// on PDUs.
+				//
+				// This enforces that the entire transaction is rejected if a single bad PDU is
+				// sent. It is unclear if this is the correct behaviour or not.
+				//
+				// See https://github.com/matrix-org/synapse/issues/7543
+				return nil, &util.JSONResponse{
+					Code: 400,
+					JSON: jsonerror.BadJSON("PDU contains bad JSON"),
+				}
 			}
+			util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %s", string(pdu))
 			continue
 		}
 		if err = gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil {
@@ -174,11 +192,20 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
 				// Any other error should be the result of a temporary error in
 				// our server so we should bail processing the transaction entirely.
 				util.GetLogger(t.context).Warnf("Processing %s failed fatally: %s", e.EventID(), err)
-				return nil, err
+				jsonErr := util.ErrorResponse(err)
+				return nil, &jsonErr
 			} else {
-				util.GetLogger(t.context).WithError(err).WithField("event_id", e.EventID()).Warn("Failed to process incoming federation event, skipping")
+				// Auth errors mean the event is 'rejected' which have to be silent to appease sytest
+				_, rejected := err.(*gomatrixserverlib.NotAllowed)
+				errMsg := err.Error()
+				if rejected {
+					errMsg = ""
+				}
+				util.GetLogger(t.context).WithError(err).WithField("event_id", e.EventID()).WithField("rejected", rejected).Warn(
+					"Failed to process incoming federation event, skipping",
+				)
 				results[e.EventID()] = gomatrixserverlib.PDUResult{
-					Error: err.Error(),
+					Error: errMsg,
 				}
 			}
 		} else {

federationapi/routing/send_test.go

@@ -10,6 +10,7 @@ import (
 
 	eduAPI "github.com/matrix-org/dendrite/eduserver/api"
 	fsAPI "github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -53,15 +54,6 @@ func init() {
 	}
 }
 
-type testNopJSONVerifier struct {
-	// this verifier verifies nothing
-}
-
-func (t *testNopJSONVerifier) VerifyJSONs(ctx context.Context, requests []gomatrixserverlib.VerifyJSONRequest) ([]gomatrixserverlib.VerifyJSONResult, error) {
-	result := make([]gomatrixserverlib.VerifyJSONResult, len(requests))
-	return result, nil
-}
-
 type testEDUProducer struct {
 	// this producer keeps track of calls to InputTypingEvent
 	invocations []eduAPI.InputTypingEventRequest
@@ -105,12 +97,18 @@ func (t *testRoomserverAPI) InputRoomEvents(
 	return nil
 }
 
+func (t *testRoomserverAPI) PerformInvite(
+	ctx context.Context,
+	req *api.PerformInviteRequest,
+	res *api.PerformInviteResponse,
+) {
+}
+
 func (t *testRoomserverAPI) PerformJoin(
 	ctx context.Context,
 	req *api.PerformJoinRequest,
 	res *api.PerformJoinResponse,
-) error {
-	return nil
+) {
 }
 
 func (t *testRoomserverAPI) PerformLeave(
@@ -330,7 +328,7 @@ func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederat
 		context:    context.Background(),
 		rsAPI:      rsAPI,
 		eduAPI:     &testEDUProducer{},
-		keys:       &testNopJSONVerifier{},
+		keys:       &test.NopJSONVerifier{},
 		federation: fedClient,
 		haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent),
 		newEvents:  make(map[string]bool),
@@ -345,7 +343,7 @@ func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederat
 func mustProcessTransaction(t *testing.T, txn *txnReq, pdusWithErrors []string) {
 	res, err := txn.processTransaction()
 	if err != nil {
-		t.Errorf("txn.processTransaction returned an error: %s", err)
+		t.Errorf("txn.processTransaction returned an error: %v", err)
 		return
 	}
 	if len(res.PDUs) != len(txn.PDUs) {

federationapi/routing/state.go

@@ -98,13 +98,17 @@ func getState(
 	roomID string,
 	eventID string,
 ) (*gomatrixserverlib.RespState, *util.JSONResponse) {
-	event, resErr := getEvent(ctx, request, rsAPI, eventID)
+	event, resErr := fetchEvent(ctx, rsAPI, eventID)
 	if resErr != nil {
 		return nil, resErr
 	}
 
 	if event.RoomID() != roomID {
-		return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
+		return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: jsonerror.NotFound("event does not belong to this room")}
+	}
+	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID)
+	if resErr != nil {
+		return nil, resErr
 	}
 
 	var response api.QueryStateAndAuthChainResponse

federationapi/routing/threepid.go

@@ -21,13 +21,11 @@ import (
 	"net/http"
 	"time"
 
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/api"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -57,10 +55,10 @@ var (
 
 // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind
 func CreateInvitesFrom3PIDInvites(
-	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite,
+	req *http.Request, rsAPI api.RoomserverInternalAPI,
+	cfg *config.Dendrite,
 	federation *gomatrixserverlib.FederationClient,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
 	var body invites
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
@@ -79,7 +77,7 @@ func CreateInvitesFrom3PIDInvites(
 		}
 
 		event, err := createInviteFrom3PIDInvite(
-			req.Context(), rsAPI, asAPI, cfg, inv, federation, accountDB,
+			req.Context(), rsAPI, cfg, inv, federation, userAPI,
 		)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed")
@@ -107,7 +105,7 @@ func ExchangeThirdPartyInvite(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI api.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
@@ -197,10 +195,10 @@ func ExchangeThirdPartyInvite(
 // Returns an error if there was a problem building the event or fetching the
 // necessary data to do so.
 func createInviteFrom3PIDInvite(
-	ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite,
+	ctx context.Context, rsAPI api.RoomserverInternalAPI,
+	cfg *config.Dendrite,
 	inv invite, federation *gomatrixserverlib.FederationClient,
-	accountDB accounts.Database,
+	userAPI userapi.UserInternalAPI,
 ) (*gomatrixserverlib.Event, error) {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
@@ -225,14 +223,17 @@ func createInviteFrom3PIDInvite(
 		StateKey: &inv.MXID,
 	}
 
-	profile, err := appserviceAPI.RetrieveUserProfile(ctx, inv.MXID, asAPI, accountDB)
+	var res userapi.QueryProfileResponse
+	err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{
+		UserID: inv.MXID,
+	}, &res)
 	if err != nil {
 		return nil, err
 	}
 
 	content := gomatrixserverlib.MemberContent{
-		AvatarURL:   profile.AvatarURL,
-		DisplayName: profile.DisplayName,
+		AvatarURL:   res.AvatarURL,
+		DisplayName: res.DisplayName,
 		Membership:  gomatrixserverlib.Invite,
 		ThirdPartyInvite: &gomatrixserverlib.MemberThirdPartyInvite{
 			Signed: inv.Signed,
@@ -261,7 +262,7 @@ func createInviteFrom3PIDInvite(
 // Returns an error if something failed during the process.
 func buildMembershipEvent(
 	ctx context.Context,
-	builder *gomatrixserverlib.EventBuilder, rsAPI roomserverAPI.RoomserverInternalAPI,
+	builder *gomatrixserverlib.EventBuilder, rsAPI api.RoomserverInternalAPI,
 	cfg *config.Dendrite,
 ) (*gomatrixserverlib.Event, error) {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
@@ -274,11 +275,11 @@ func buildMembershipEvent(
 	}
 
 	// Ask the roomserver for information about this room
-	queryReq := roomserverAPI.QueryLatestEventsAndStateRequest{
+	queryReq := api.QueryLatestEventsAndStateRequest{
 		RoomID:       builder.RoomID,
 		StateToFetch: eventsNeeded.Tuples(),
 	}
-	var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
+	var queryRes api.QueryLatestEventsAndStateResponse
 	if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil {
 		return nil, err
 	}

federationsender/api/api.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/matrix-org/dendrite/federationsender/types"
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -28,7 +29,7 @@ type FederationSenderInternalAPI interface {
 		ctx context.Context,
 		request *PerformJoinRequest,
 		response *PerformJoinResponse,
-	) error
+	)
 	// Handle an instruction to make_leave & send_leave with a remote server.
 	PerformLeave(
 		ctx context.Context,
@@ -62,6 +63,7 @@ type PerformJoinRequest struct {
 }
 
 type PerformJoinResponse struct {
+	LastError *gomatrix.HTTPError
 }
 
 type PerformLeaveRequest struct {

federationsender/internal/perform.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -9,6 +10,7 @@ import (
 	"github.com/matrix-org/dendrite/federationsender/internal/perform"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
@@ -40,7 +42,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 	ctx context.Context,
 	request *api.PerformJoinRequest,
 	response *api.PerformJoinResponse,
-) (err error) {
+) {
 	// Look up the supported room versions.
 	var supportedVersions []gomatrixserverlib.RoomVersion
 	for version := range version.SupportedRoomVersions() {
@@ -63,6 +65,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 
 	// Try each server that we were provided until we land on one that
 	// successfully completes the make-join send-join dance.
+	var lastErr error
 	for _, serverName := range request.ServerNames {
 		if err := r.performJoinUsingServer(
 			ctx,
@@ -76,17 +79,32 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 				"server_name": serverName,
 				"room_id":     request.RoomID,
 			}).Warnf("Failed to join room through server")
+			lastErr = err
 			continue
 		}
 
 		// We're all good.
-		return nil
+		return
 	}
 
 	// If we reach here then we didn't complete a join for some reason.
-	return fmt.Errorf(
-		"failed to join user %q to room %q through %d server(s)",
-		request.UserID, request.RoomID, len(request.ServerNames),
+	var httpErr gomatrix.HTTPError
+	if ok := errors.As(lastErr, &httpErr); ok {
+		httpErr.Message = string(httpErr.Contents)
+		// Clear the wrapped error, else serialising to JSON (in polylith mode) will fail
+		httpErr.WrappedError = nil
+		response.LastError = &httpErr
+	} else {
+		response.LastError = &gomatrix.HTTPError{
+			Code:         0,
+			WrappedError: nil,
+			Message:      lastErr.Error(),
+		}
+	}
+
+	logrus.Errorf(
+		"failed to join user %q to room %q through %d server(s): last error %s",
+		request.UserID, request.RoomID, len(request.ServerNames), lastErr,
 	)
 }
 

federationsender/inthttp/client.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/gomatrix"
 	"github.com/opentracing/opentracing-go"
 )
 
@@ -77,12 +78,19 @@ func (h *httpFederationSenderInternalAPI) PerformJoin(
 	ctx context.Context,
 	request *api.PerformJoinRequest,
 	response *api.PerformJoinResponse,
-) error {
+) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest")
 	defer span.Finish()
 
 	apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	if err != nil {
+		response.LastError = &gomatrix.HTTPError{
+			Message:      err.Error(),
+			Code:         0,
+			WrappedError: err,
+		}
+	}
 }
 
 // Handle an instruction to make_join & send_join with a remote server.

federationsender/inthttp/server.go

@@ -33,9 +33,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
 				return util.MessageResponse(http.StatusBadRequest, err.Error())
 			}
-			if err := intAPI.PerformJoin(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
+			intAPI.PerformJoin(req.Context(), &request, &response)
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible
@@ -38,7 +38,6 @@ require (
 	github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b
 	go.uber.org/atomic v1.4.0
 	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
-	golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2 // indirect
 	gopkg.in/h2non/bimg.v1 v1.0.18
 	gopkg.in/yaml.v2 v2.2.8
 )

go.sum

@@ -131,7 +131,6 @@ github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hjson/hjson-go v3.0.2-0.20200316202735-d5d0e8b0617d+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -372,8 +371,14 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1 h1:BfrvDrbjoPBvYua/3F/FmrqiZTRGrvtoMRgCVnrufMI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1 h1:3yS6hw01X72jpJuAPGVOY+QFD9cpAETR/6Hq2WYKbpU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625121044-e5d892cd30c1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1 h1:QDOdGCfrzuVLEess3id2a2B29oVZ9JXgJmUfwE7r/iI=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625153204-0f1026cd05d1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d h1:v1JS+JZWwAsqAc22TGWPbRDc6O5D6geSfV5Bb5wvYIs=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328 h1:rz6aiTpUyNPRcWZBWUGDkQjI7lfeLdhzy+x/Pw2jha8=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
@@ -566,11 +571,8 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yggdrasil-network/yggdrasil-extras v0.0.0-20200525205615-6c8a4a2e8855/go.mod h1:xQdsh08Io6nV4WRnOVTe6gI8/2iTvfLDQ0CYa5aMt+I=
-github.com/yggdrasil-network/yggdrasil-go v0.3.14 h1:vWzYzCQxOruS+J5FkLfXOS0JhCJx1yI9Erj/h2wfZ/E=
-github.com/yggdrasil-network/yggdrasil-go v0.3.14/go.mod h1:rkQzLzVHlFdzsEMG+bDdTI+KeWPCZq1HpXRFzwinf6M=
 github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b h1:ELOisSxFXCcptRs4LFub+Hz5fYUvV12wZrTps99Eb3E=
 github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -604,8 +606,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -675,9 +675,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2 h1:DVqHa33CzfnTKwUV6be+I4hp31W6iXn3ZiEcdKGzLyI=
-golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

internal/caching/cache_serverkeys.go

@@ -2,7 +2,6 @@ package caching
 
 import (
 	"fmt"
-	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -16,22 +15,29 @@ const (
 // ServerKeyCache contains the subset of functions needed for
 // a server key cache.
 type ServerKeyCache interface {
-	GetServerKey(request gomatrixserverlib.PublicKeyLookupRequest) (response gomatrixserverlib.PublicKeyLookupResult, ok bool)
+	// request -> timestamp is emulating gomatrixserverlib.FetchKeys:
+	// https://github.com/matrix-org/gomatrixserverlib/blob/f69539c86ea55d1e2cc76fd8e944e2d82d30397c/keyring.go#L95
+	// The timestamp should be the timestamp of the event that is being
+	// verified. We will not return keys from the cache that are not valid
+	// at this timestamp.
+	GetServerKey(request gomatrixserverlib.PublicKeyLookupRequest, timestamp gomatrixserverlib.Timestamp) (response gomatrixserverlib.PublicKeyLookupResult, ok bool)
+
+	// request -> result is emulating gomatrixserverlib.StoreKeys:
+	// https://github.com/matrix-org/gomatrixserverlib/blob/f69539c86ea55d1e2cc76fd8e944e2d82d30397c/keyring.go#L112
 	StoreServerKey(request gomatrixserverlib.PublicKeyLookupRequest, response gomatrixserverlib.PublicKeyLookupResult)
 }
 
 func (c Caches) GetServerKey(
 	request gomatrixserverlib.PublicKeyLookupRequest,
+	timestamp gomatrixserverlib.Timestamp,
 ) (gomatrixserverlib.PublicKeyLookupResult, bool) {
 	key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID)
-	now := gomatrixserverlib.AsTimestamp(time.Now())
 	val, found := c.ServerKeys.Get(key)
 	if found && val != nil {
 		if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok {
-			if !keyLookupResult.WasValidAt(now, true) {
-				// We appear to be past the key validity so don't return this
-				// with the results. This ensures that the cache doesn't return
-				// values that are not useful to us.
+			if !keyLookupResult.WasValidAt(timestamp, true) {
+				// The key wasn't valid at the requested timestamp so don't
+				// return it. The caller will have to work out what to do.
 				c.ServerKeys.Unset(key)
 				return gomatrixserverlib.PublicKeyLookupResult{}, false
 			}

internal/caching/impl_inmemorylru.go

@@ -8,11 +8,12 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
-func NewInMemoryLRUCache() (*Caches, error) {
+func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) {
 	roomVersions, err := NewInMemoryLRUCachePartition(
 		RoomVersionCacheName,
 		RoomVersionCacheMutable,
 		RoomVersionCacheMaxEntries,
+		enablePrometheus,
 	)
 	if err != nil {
 		return nil, err
@@ -21,6 +22,7 @@ func NewInMemoryLRUCache() (*Caches, error) {
 		ServerKeyCacheName,
 		ServerKeyCacheMutable,
 		ServerKeyCacheMaxEntries,
+		enablePrometheus,
 	)
 	if err != nil {
 		return nil, err
@@ -38,7 +40,7 @@ type InMemoryLRUCachePartition struct {
 	lru        *lru.Cache
 }
 
-func NewInMemoryLRUCachePartition(name string, mutable bool, maxEntries int) (*InMemoryLRUCachePartition, error) {
+func NewInMemoryLRUCachePartition(name string, mutable bool, maxEntries int, enablePrometheus bool) (*InMemoryLRUCachePartition, error) {
 	var err error
 	cache := InMemoryLRUCachePartition{
 		name:       name,
@@ -49,13 +51,15 @@ func NewInMemoryLRUCachePartition(name string, mutable bool, maxEntries int) (*I
 	if err != nil {
 		return nil, err
 	}
-	promauto.NewGaugeFunc(prometheus.GaugeOpts{
-		Namespace: "dendrite",
-		Subsystem: "caching_in_memory_lru",
-		Name:      name,
-	}, func() float64 {
-		return float64(cache.lru.Len())
-	})
+	if enablePrometheus {
+		promauto.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace: "dendrite",
+			Subsystem: "caching_in_memory_lru",
+			Name:      name,
+		}, func() float64 {
+			return float64(cache.lru.Len())
+		})
+	}
 	return &cache, nil
 }
 

internal/config/config.go

@@ -226,6 +226,7 @@ type Dendrite struct {
 		ServerKeyAPI     Address `yaml:"server_key_api"`
 		AppServiceAPI    Address `yaml:"appservice_api"`
 		SyncAPI          Address `yaml:"sync_api"`
+		UserAPI          Address `yaml:"user_api"`
 		RoomServer       Address `yaml:"room_server"`
 		FederationSender Address `yaml:"federation_sender"`
 		PublicRoomsAPI   Address `yaml:"public_rooms_api"`
@@ -241,6 +242,7 @@ type Dendrite struct {
 		ServerKeyAPI     Address `yaml:"server_key_api"`
 		AppServiceAPI    Address `yaml:"appservice_api"`
 		SyncAPI          Address `yaml:"sync_api"`
+		UserAPI          Address `yaml:"user_api"`
 		RoomServer       Address `yaml:"room_server"`
 		FederationSender Address `yaml:"federation_sender"`
 		PublicRoomsAPI   Address `yaml:"public_rooms_api"`
@@ -610,6 +612,7 @@ func (config *Dendrite) checkListen(configErrs *configErrors) {
 	checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer))
 	checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer))
 	checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer))
+	checkNotEmpty(configErrs, "listen.user_api", string(config.Listen.UserAPI))
 }
 
 // checkLogging verifies the parameters logging.* are valid.
@@ -723,6 +726,15 @@ func (config *Dendrite) RoomServerURL() string {
 	return "http://" + string(config.Listen.RoomServer)
 }
 
+// UserAPIURL returns an HTTP URL for where the userapi is listening.
+func (config *Dendrite) UserAPIURL() string {
+	// Hard code the userapi to talk HTTP for now.
+	// If we support HTTPS we need to think of a practical way to do certificate validation.
+	// People setting up servers shouldn't need to get a certificate valid for the public
+	// internet for an internal API.
+	return "http://" + string(config.Listen.UserAPI)
+}
+
 // EDUServerURL returns an HTTP URL for where the EDU server is listening.
 func (config *Dendrite) EDUServerURL() string {
 	// Hard code the EDU server to talk HTTP for now.

internal/config/config_test.go

@@ -63,6 +63,7 @@ listen:
   media_api: "localhost:7774"
   appservice_api: "localhost:7777"
   edu_server: "localhost:7778"
+  user_api: "localhost:7779"
 logging:
   - type: "file"
     level: "info"

internal/httputil/httpapi.go

@@ -27,9 +27,9 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	federationsenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -48,11 +48,11 @@ type BasicAuth struct {
 
 // MakeAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request.
 func MakeAuthAPI(
-	metricsName string, data auth.Data,
-	f func(*http.Request, *authtypes.Device) util.JSONResponse,
+	metricsName string, userAPI userapi.UserInternalAPI,
+	f func(*http.Request, *userapi.Device) util.JSONResponse,
 ) http.Handler {
 	h := func(req *http.Request) util.JSONResponse {
-		device, err := auth.VerifyUserFromRequest(req, data)
+		device, err := auth.VerifyUserFromRequest(req, userAPI)
 		if err != nil {
 			return *err
 		}
@@ -185,7 +185,7 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse
 func MakeFedAPI(
 	metricsName string,
 	serverName gomatrixserverlib.ServerName,
-	keyRing gomatrixserverlib.KeyRing,
+	keyRing gomatrixserverlib.JSONVerifier,
 	wakeup *FederationWakeups,
 	f func(*http.Request, *gomatrixserverlib.FederationRequest, map[string]string) util.JSONResponse,
 ) http.Handler {
@@ -233,9 +233,8 @@ func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib
 	}
 }
 
-// SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics
-// listener.
-func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) {
+// SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics listener
+func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) {
 	if cfg.Metrics.Enabled {
 		servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth))
 	}

internal/setup/base.go

@@ -28,9 +28,9 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/naffka"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 
 	"github.com/Shopify/sarama"
 	"github.com/gorilla/mux"
@@ -46,6 +46,8 @@ import (
 	rsinthttp "github.com/matrix-org/dendrite/roomserver/inthttp"
 	serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api"
 	skinthttp "github.com/matrix-org/dendrite/serverkeyapi/inthttp"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	userapiinthttp "github.com/matrix-org/dendrite/userapi/inthttp"
 	"github.com/sirupsen/logrus"
 
 	_ "net/http/pprof"
@@ -63,6 +65,7 @@ type BaseDendrite struct {
 	// PublicAPIMux should be used to register new public matrix api endpoints
 	PublicAPIMux   *mux.Router
 	InternalAPIMux *mux.Router
+	BaseMux        *mux.Router // base router which created public/internal subrouters
 	UseHTTPAPIs    bool
 	httpClient     *http.Client
 	Cfg            *config.Dendrite
@@ -95,7 +98,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 		kafkaConsumer, kafkaProducer = setupKafka(cfg)
 	}
 
-	cache, err := caching.NewInMemoryLRUCache()
+	cache, err := caching.NewInMemoryLRUCache(true)
 	if err != nil {
 		logrus.WithError(err).Warnf("Failed to create cache")
 	}
@@ -127,6 +130,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 		tracerCloser:   closer,
 		Cfg:            cfg,
 		Caches:         cache,
+		BaseMux:        httpmux,
 		PublicAPIMux:   httpmux.PathPrefix(httputil.PublicPathPrefix).Subrouter().UseEncodedPath(),
 		InternalAPIMux: httpmux.PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(),
 		httpClient:     &client,
@@ -158,6 +162,15 @@ func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAP
 	return rsAPI
 }
 
+// UserAPIClient returns UserInternalAPI for hitting the userapi over HTTP.
+func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI {
+	userAPI, err := userapiinthttp.NewUserAPIClient(b.Cfg.UserAPIURL(), b.httpClient)
+	if err != nil {
+		logrus.WithError(err).Panic("UserAPIClient failed", b.httpClient)
+	}
+	return userAPI
+}
+
 // EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP
 func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI {
 	e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient)
@@ -238,12 +251,13 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) {
 	}
 
 	httputil.SetupHTTPAPI(
-		http.DefaultServeMux,
+		b.BaseMux,
 		b.PublicAPIMux,
 		b.InternalAPIMux,
 		b.Cfg,
 		b.UseHTTPAPIs,
 	)
+	serv.Handler = b.BaseMux
 	logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr)
 
 	err := serv.ListenAndServe()

internal/setup/monolith.go

@@ -19,8 +19,6 @@ import (
 	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/federationapi"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
@@ -34,6 +32,9 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api"
 	"github.com/matrix-org/dendrite/syncapi"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -44,6 +45,7 @@ type Monolith struct {
 	DeviceDB      devices.Database
 	AccountDB     accounts.Database
 	KeyRing       *gomatrixserverlib.KeyRing
+	Client        *gomatrixserverlib.Client
 	FedClient     *gomatrixserverlib.FederationClient
 	KafkaConsumer sarama.Consumer
 	KafkaProducer sarama.SyncProducer
@@ -53,6 +55,7 @@ type Monolith struct {
 	FederationSenderAPI federationSenderAPI.FederationSenderInternalAPI
 	RoomserverAPI       roomserverAPI.RoomserverInternalAPI
 	ServerKeyAPI        serverKeyAPI.ServerKeyInternalAPI
+	UserAPI             userapi.UserInternalAPI
 
 	// TODO: can we remove this? It's weird that we are required the database
 	// yet every other component can do that on its own. libp2p-demo uses a custom
@@ -67,23 +70,23 @@ type Monolith struct {
 func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) {
 	clientapi.AddPublicRoutes(
 		publicMux, m.Config, m.KafkaConsumer, m.KafkaProducer, m.DeviceDB, m.AccountDB,
-		m.FedClient, m.KeyRing, m.RoomserverAPI,
+		m.FedClient, m.RoomserverAPI,
 		m.EDUInternalAPI, m.AppserviceAPI, transactions.New(),
-		m.FederationSenderAPI,
+		m.FederationSenderAPI, m.UserAPI,
 	)
 
-	keyserver.AddPublicRoutes(publicMux, m.Config, m.DeviceDB, m.AccountDB)
+	keyserver.AddPublicRoutes(publicMux, m.Config, m.UserAPI)
 	federationapi.AddPublicRoutes(
-		publicMux, m.Config, m.AccountDB, m.DeviceDB, m.FedClient,
-		m.KeyRing, m.RoomserverAPI, m.AppserviceAPI, m.FederationSenderAPI,
+		publicMux, m.Config, m.UserAPI, m.FedClient,
+		m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI,
 		m.EDUInternalAPI,
 	)
-	mediaapi.AddPublicRoutes(publicMux, m.Config, m.DeviceDB)
+	mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client)
 	publicroomsapi.AddPublicRoutes(
-		publicMux, m.Config, m.KafkaConsumer, m.DeviceDB, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient,
+		publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient,
 		m.ExtPublicRoomsProvider,
 	)
 	syncapi.AddPublicRoutes(
-		publicMux, m.KafkaConsumer, m.DeviceDB, m.AccountDB, m.RoomserverAPI, m.FedClient, m.Config,
+		publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, m.Config,
 	)
 }

internal/test/keyring.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,20 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package authtypes
+package test
 
 import (
+	"context"
+
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// Account represents a Matrix account on this home server.
-type Account struct {
-	UserID       string
-	Localpart    string
-	ServerName   gomatrixserverlib.ServerName
-	Profile      *Profile
-	AppServiceID string
-	// TODO: Other flags like IsAdmin, IsGuest
-	// TODO: Devices
-	// TODO: Associations (e.g. with application services)
+// NopJSONVerifier is a JSONVerifier that verifies nothing and returns no errors.
+type NopJSONVerifier struct {
+	// this verifier verifies nothing
+}
+
+func (t *NopJSONVerifier) VerifyJSONs(ctx context.Context, requests []gomatrixserverlib.VerifyJSONRequest) ([]gomatrixserverlib.VerifyJSONResult, error) {
+	result := make([]gomatrixserverlib.VerifyJSONResult, len(requests))
+	return result, nil
 }