Merge branch 'master' into add-presence

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @matrix-org/dendrite-core

appservice/appservice.go

@@ -131,10 +131,11 @@ func generateAppServiceAccount(
 	}
 	var devRes userapi.PerformDeviceCreationResponse
 	err = userAPI.PerformDeviceCreation(context.Background(), &userapi.PerformDeviceCreationRequest{
-		Localpart:         as.SenderLocalpart,
+		Localpart:          as.SenderLocalpart,
-		AccessToken:       as.ASToken,
+		AccessToken:        as.ASToken,
-		DeviceID:          &as.SenderLocalpart,
+		DeviceID:           &as.SenderLocalpart,
-		DeviceDisplayName: &as.SenderLocalpart,
+		DeviceDisplayName:  &as.SenderLocalpart,
+		NoDeviceListUpdate: true,
 	}, &devRes)
 	return err
 }

are-we-synapse-yet.py

@@ -177,6 +177,10 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
         line = "%s: %s (%d/%d tests)" % (gid_to_name[gid].ljust(25, ' '), pct.rjust(4, ' '), group_passing, group_total)
         subsections.append(line)
         subsection_test_names[line] = test_names_and_marks
+
+    # avoid errors when trying to divide by 0
+    if total_tests == 0:
+        return
     
     pct = "{0:.0f}%".format(total_passing/total_tests * 100)
     print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests))

build/docker/docker-compose.deps.yml

@@ -3,7 +3,7 @@ services:
   # PostgreSQL is needed for both polylith and monolith modes.
   postgres:
     hostname: postgres
-    image: postgres:11
+    image: postgres:14
     restart: always
     volumes:
       - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh

build/gobind-pinecone/monolith.go

@@ -310,7 +310,7 @@ func (m *DendriteMonolith) Start() {
 	rsAPI := roomserver.NewInternalAPI(base)
 
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, m.userAPI, base.Caches, true,
+		base, federation, rsAPI, m.userAPI, base.Caches, keyRing, true,
 	)
 
 	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
@@ -325,8 +325,7 @@ func (m *DendriteMonolith) Start() {
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
-	rsAPI.SetFederationAPI(fsAPI)
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
-	rsAPI.SetKeyring(keyRing)
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,

build/gobind-yggdrasil/monolith.go

@@ -118,7 +118,7 @@ func (m *DendriteMonolith) Start() {
 	keyAPI.SetUserAPI(userAPI)
 
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, userAPI, base.Caches, true,
+		base, federation, rsAPI, userAPI, base.Caches, keyRing, true,
 	)
 
 	eduInputAPI := eduserver.NewInternalAPI(
@@ -130,8 +130,7 @@ func (m *DendriteMonolith) Start() {
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
-	rsAPI.SetFederationAPI(fsAPI)
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
-	rsAPI.SetKeyring(keyRing)
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,

clientapi/auth/auth.go

@@ -70,11 +70,11 @@ func VerifyUserFromRequest(
 		jsonErr := jsonerror.InternalServerError()
 		return nil, &jsonErr
 	}
-	if res.Err != nil {
+	if res.Err != "" {
-		if forbidden, ok := res.Err.(*api.ErrorForbidden); ok {
+		if strings.HasPrefix(strings.ToLower(res.Err), "forbidden:") { // TODO: use actual error and no string comparison
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden(forbidden.Message),
+				JSON: jsonerror.Forbidden(res.Err),
 			}
 		}
 	}

clientapi/auth/password.go

@@ -17,6 +17,7 @@ package auth
 import (
 	"context"
 	"net/http"
+	"strings"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
@@ -48,7 +49,8 @@ func (t *LoginTypePassword) Request() interface{} {
 
 func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
 	r := req.(*PasswordRequest)
-	username := r.Username()
+	// Squash username to all lowercase letters
+	username := strings.ToLower(r.Username())
 	if username == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,

clientapi/routing/key_backup.go

@@ -62,12 +62,14 @@ func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI,
 		return *resErr
 	}
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
-	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
 		UserID:    device.UserID,
 		Version:   "",
 		AuthData:  kb.AuthData,
 		Algorithm: kb.Algorithm,
-	}, &performKeyBackupResp)
+	}, &performKeyBackupResp); err != nil {
+		return jsonerror.InternalServerError()
+	}
 	if performKeyBackupResp.Error != "" {
 		if performKeyBackupResp.BadInput {
 			return util.JSONResponse{
@@ -123,12 +125,14 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInter
 		return *resErr
 	}
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
-	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
 		UserID:    device.UserID,
 		Version:   version,
 		AuthData:  kb.AuthData,
 		Algorithm: kb.Algorithm,
-	}, &performKeyBackupResp)
+	}, &performKeyBackupResp); err != nil {
+		return jsonerror.InternalServerError()
+	}
 	if performKeyBackupResp.Error != "" {
 		if performKeyBackupResp.BadInput {
 			return util.JSONResponse{
@@ -157,11 +161,13 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInter
 // Implements DELETE  /_matrix/client/r0/room_keys/version/{version}
 func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
-	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
 		UserID:       device.UserID,
 		Version:      version,
 		DeleteBackup: true,
-	}, &performKeyBackupResp)
+	}, &performKeyBackupResp); err != nil {
+		return jsonerror.InternalServerError()
+	}
 	if performKeyBackupResp.Error != "" {
 		if performKeyBackupResp.BadInput {
 			return util.JSONResponse{
@@ -191,11 +197,13 @@ func UploadBackupKeys(
 	req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string, keys *keyBackupSessionRequest,
 ) util.JSONResponse {
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
-	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
 		UserID:  device.UserID,
 		Version: version,
 		Keys:    *keys,
-	}, &performKeyBackupResp)
+	}, &performKeyBackupResp); err != nil && performKeyBackupResp.Error == "" {
+		return jsonerror.InternalServerError()
+	}
 	if performKeyBackupResp.Error != "" {
 		if performKeyBackupResp.BadInput {
 			return util.JSONResponse{

cmd/dendrite-demo-libp2p/main.go

@@ -158,10 +158,10 @@ func main() {
 	asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationapi.NewInternalAPI(
-		&base.Base, federation, rsAPI, userAPI, base.Base.Caches, true,
+		&base.Base, federation, rsAPI, userAPI, base.Base.Caches, nil, true,
 	)
 	keyRing := fsAPI.KeyRing()
-	rsAPI.SetFederationAPI(fsAPI)
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
 	provider := newPublicRoomsProvider(base.LibP2PPubsub, rsAPI)
 	err = provider.Start()
 	if err != nil {

cmd/dendrite-demo-pinecone/main.go

@@ -185,7 +185,7 @@ func main() {
 	rsComponent := roomserver.NewInternalAPI(base)
 	rsAPI := rsComponent
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, nil, base.Caches, true,
+		base, federation, rsAPI, nil, base.Caches, keyRing, true,
 	)
 
 	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
@@ -199,8 +199,7 @@ func main() {
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 
-	rsComponent.SetFederationAPI(fsAPI)
+	rsComponent.SetFederationAPI(fsAPI, keyRing)
-	rsComponent.SetKeyring(keyRing)
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,

cmd/dendrite-demo-yggdrasil/main.go

@@ -118,11 +118,10 @@ func main() {
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, userAPI, base.Caches, true,
+		base, federation, rsAPI, userAPI, base.Caches, keyRing, true,
 	)
 
-	rsComponent.SetFederationAPI(fsAPI)
+	rsComponent.SetFederationAPI(fsAPI, keyRing)
-	rsComponent.SetKeyring(keyRing)
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,

cmd/dendrite-monolith-server/main.go

@@ -67,6 +67,7 @@ func main() {
 		cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr
 		cfg.RoomServer.InternalAPI.Connect = httpAPIAddr
 		cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr
+		cfg.UserAPI.InternalAPI.Connect = httpAPIAddr
 		options = append(options, basepkg.UseHTTPAPIs)
 	}
 
@@ -90,7 +91,7 @@ func main() {
 	}
 
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, nil, base.Caches, false,
+		base, federation, rsAPI, nil, base.Caches, nil, false,
 	)
 	if base.UseHTTPAPIs {
 		federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
@@ -100,24 +101,44 @@ func main() {
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
-	rsImpl.SetFederationAPI(fsAPI)
+	rsImpl.SetFederationAPI(fsAPI, keyRing)
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
+	keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
-	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
+	keyAPI := keyImpl
-	keyAPI.SetUserAPI(userAPI)
+	if base.UseHTTPAPIs {
-	fsAPI.SetUserAPI(userAPI)
+		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
+		keyAPI = base.KeyServerHTTPClient()
+	}
 
+	userImpl := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
+	userAPI := userImpl
+	if base.UseHTTPAPIs {
+		userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
+		userAPI = base.UserAPIClient()
+	}
 	if traceInternal {
 		userAPI = &uapi.UserInternalAPITrace{
 			Impl: userAPI,
 		}
 	}
-	// needs to be after the SetUserAPI call above
+
+	// TODO: This should use userAPI, not userImpl, but the appservice setup races with
+	// the listeners and panics at startup if it tries to create appservice accounts
+	// before the listeners are up.
+	asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI)
 	if base.UseHTTPAPIs {
-		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
+		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
-		keyAPI = base.KeyServerHTTPClient()
+		asAPI = base.AppserviceHTTPClient()
 	}
 
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this
+	// dependency. Other components also need updating after their dependencies are up.
+	rsImpl.SetFederationAPI(fsAPI, keyRing)
+	rsImpl.SetAppserviceAPI(asAPI)
+	keyImpl.SetUserAPI(userAPI)
+	fsAPI.SetUserAPI(userAPI)
+
 	eduInputAPI := eduserver.NewInternalAPI(
 		base, cache.New(), userAPI,
 	)
@@ -126,13 +147,6 @@ func main() {
 		eduInputAPI = base.EDUServerClient()
 	}
 
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	if base.UseHTTPAPIs {
-		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
-		asAPI = base.AppserviceHTTPClient()
-	}
-	rsAPI.SetAppserviceAPI(asAPI)
-
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
 		AccountDB: accountDB,

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -23,10 +23,10 @@ import (
 func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	federation := base.CreateFederationClient()
-	fsAPI := base.FederationAPIHTTPClient()
-	keyRing := fsAPI.KeyRing()
 	rsAPI := base.RoomserverHTTPClient()
 	keyAPI := base.KeyServerHTTPClient()
+	fsAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, nil, true)
+	keyRing := fsAPI.KeyRing()
 
 	federationapi.AddPublicRoutes(
 		base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux,
@@ -35,6 +35,8 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 		&base.Cfg.MSCs, nil,
 	)
 
+	federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
+
 	base.SetupAndServeHTTP(
 		base.Cfg.FederationAPI.InternalAPI.Listen,
 		base.Cfg.FederationAPI.ExternalAPI.Listen,

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -24,7 +24,7 @@ func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	asAPI := base.AppserviceHTTPClient()
 	fsAPI := base.FederationAPIHTTPClient()
 	rsAPI := roomserver.NewInternalAPI(base)
-	rsAPI.SetFederationAPI(fsAPI)
+	rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing())
 	rsAPI.SetAppserviceAPI(asAPI)
 	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
 

cmd/dendritejs-pinecone/main.go

@@ -198,9 +198,9 @@ func startup() {
 		base, userAPI, rsAPI,
 	)
 	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, userAPI, base.Caches, true)
+
-	rsAPI.SetFederationAPI(fedSenderAPI)
+	fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, userAPI, base.Caches, keyRing, true)
-	rsAPI.SetKeyring(keyRing)
+	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,

cmd/dendritejs/main.go

@@ -210,9 +210,8 @@ func main() {
 		base, userAPI, rsAPI,
 	)
 	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, true)
+	fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, keyRing, true)
-	rsAPI.SetFederationAPI(fedSenderAPI)
+	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
-	rsAPI.SetKeyring(keyRing)
 	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation)
 
 	monolith := setup.Monolith{

dendrite-config.yaml

@@ -378,9 +378,10 @@ tracing:
     baggage_restrictions: null
     throttler: null
 
-# Logging configuration, in addition to the standard logging that is sent to
+# Logging configuration
-# stdout by Dendrite.
 logging:
+- type: std
+  level: info
 - type: file
   level: info
   params:

federationapi/federationapi.go

@@ -74,6 +74,7 @@ func NewInternalAPI(
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	userAPI userAPI.UserInternalAPI,
 	caches *caching.Caches,
+	keyRing *gomatrixserverlib.KeyRing,
 	resetBlacklist bool,
 ) api.FederationInternalAPI {
 	cfg := &base.Cfg.FederationAPI
@@ -126,5 +127,5 @@ func NewInternalAPI(
 		logrus.WithError(err).Panic("failed to start key server consumer")
 	}
 
-	return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, userAPI, federation, stats, caches, queues)
+	return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, userAPI, federation, stats, caches, queues, keyRing)
 }

federationapi/federationapi_keys_test.go

@@ -94,7 +94,7 @@ func TestMain(m *testing.M) {
 
 		// Finally, build the server key APIs.
 		sbase := base.NewBaseDendrite(cfg, "Monolith", base.NoCacheMetrics)
-		s.api = NewInternalAPI(sbase, s.fedclient, nil, nil, s.cache, true)
+		s.api = NewInternalAPI(sbase, s.fedclient, nil, nil, s.cache, nil, true)
 	}
 
 	// Now that we have built our server key APIs, start the

federationapi/internal/api.go

@@ -42,58 +42,61 @@ func NewFederationInternalAPI(
 	statistics *statistics.Statistics,
 	caches *caching.Caches,
 	queues *queue.OutgoingQueues,
+	keyRing *gomatrixserverlib.KeyRing,
 ) *FederationInternalAPI {
 	serverKeyDB, err := cache.NewKeyDatabase(db, caches)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to set up caching wrapper for server key database")
 	}
 
-	keyRing := &gomatrixserverlib.KeyRing{
+	if keyRing == nil {
-		KeyFetchers: []gomatrixserverlib.KeyFetcher{},
+		keyRing = &gomatrixserverlib.KeyRing{
-		KeyDatabase: serverKeyDB,
+			KeyFetchers: []gomatrixserverlib.KeyFetcher{},
-	}
+			KeyDatabase: serverKeyDB,
-
-	addDirectFetcher := func() {
-		keyRing.KeyFetchers = append(
-			keyRing.KeyFetchers,
-			&gomatrixserverlib.DirectKeyFetcher{
-				Client: federation,
-			},
-		)
-	}
-
-	if cfg.PreferDirectFetch {
-		addDirectFetcher()
-	} else {
-		defer addDirectFetcher()
-	}
-
-	var b64e = base64.StdEncoding.WithPadding(base64.NoPadding)
-	for _, ps := range cfg.KeyPerspectives {
-		perspective := &gomatrixserverlib.PerspectiveKeyFetcher{
-			PerspectiveServerName: ps.ServerName,
-			PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{},
-			Client:                federation,
 		}
 
-		for _, key := range ps.Keys {
+		addDirectFetcher := func() {
-			rawkey, err := b64e.DecodeString(key.PublicKey)
+			keyRing.KeyFetchers = append(
-			if err != nil {
+				keyRing.KeyFetchers,
-				logrus.WithError(err).WithFields(logrus.Fields{
+				&gomatrixserverlib.DirectKeyFetcher{
-					"server_name": ps.ServerName,
+					Client: federation,
-					"public_key":  key.PublicKey,
+				},
-				}).Warn("Couldn't parse perspective key")
+			)
-				continue
+		}
+
+		if cfg.PreferDirectFetch {
+			addDirectFetcher()
+		} else {
+			defer addDirectFetcher()
+		}
+
+		var b64e = base64.StdEncoding.WithPadding(base64.NoPadding)
+		for _, ps := range cfg.KeyPerspectives {
+			perspective := &gomatrixserverlib.PerspectiveKeyFetcher{
+				PerspectiveServerName: ps.ServerName,
+				PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{},
+				Client:                federation,
 			}
-			perspective.PerspectiveServerKeys[key.KeyID] = rawkey
+
+			for _, key := range ps.Keys {
+				rawkey, err := b64e.DecodeString(key.PublicKey)
+				if err != nil {
+					logrus.WithError(err).WithFields(logrus.Fields{
+						"server_name": ps.ServerName,
+						"public_key":  key.PublicKey,
+					}).Warn("Couldn't parse perspective key")
+					continue
+				}
+				perspective.PerspectiveServerKeys[key.KeyID] = rawkey
+			}
+
+			keyRing.KeyFetchers = append(keyRing.KeyFetchers, perspective)
+
+			logrus.WithFields(logrus.Fields{
+				"server_name":     ps.ServerName,
+				"num_public_keys": len(ps.Keys),
+			}).Info("Enabled perspective key fetcher")
 		}
-
-		keyRing.KeyFetchers = append(keyRing.KeyFetchers, perspective)
-
-		logrus.WithFields(logrus.Fields{
-			"server_name":     ps.ServerName,
-			"num_public_keys": len(ps.Keys),
-		}).Info("Enabled perspective key fetcher")
 	}
 
 	return &FederationInternalAPI{

go.mod

@@ -4,6 +4,7 @@ require (
 	github.com/Arceliar/ironwood v0.0.0-20210619124114-6ad55cae5031
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
+	github.com/MFAshby/stdemuxerhook v1.0.0 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/Shopify/sarama v1.29.1
 	github.com/codeclysm/extract v2.2.0+incompatible
@@ -33,7 +34,7 @@ require (
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2
 	github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0
-	github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd
+	github.com/matrix-org/pinecone v0.0.0-20211213132835-aa2808d77947
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.8
 	github.com/morikuni/aec v1.0.0 // indirect

go.sum

@@ -59,6 +59,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bE
 github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
+github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc=
+github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
@@ -997,8 +999,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 h1:RF
 github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc=
 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU=
 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
-github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd h1:/iX6jehN2sO8n4pn63U+7iDoNx18fjC6pQ2RpwyZtKk=
+github.com/matrix-org/pinecone v0.0.0-20211213132835-aa2808d77947 h1:TxO9TMFAuF+Vz3vZV53z5mjycWtF1+naY9ffs6QfZxc=
-github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
+github.com/matrix-org/pinecone v0.0.0-20211213132835-aa2808d77947/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

internal/httputil/http.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/matrix-org/dendrite/userapi/api"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 )
@@ -72,6 +73,9 @@ func PostJSON(
 		var errorBody struct {
 			Message string `json:"message"`
 		}
+		if _, ok := response.(*api.PerformKeyBackupResponse); ok { // TODO: remove this, once cross-boundary errors are a thing
+			return nil
+		}
 		if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil {
 			return fmt.Errorf("internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message)
 		}

internal/log_unix.go

@@ -12,13 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !windows
 // +build !windows
 
 package internal
 
 import (
+	"io/ioutil"
 	"log/syslog"
 
+	"github.com/MFAshby/stdemuxerhook"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/sirupsen/logrus"
 	lSyslog "github.com/sirupsen/logrus/hooks/syslog"
@@ -28,7 +31,7 @@ import (
 // If something fails here it means that the logging was improperly configured,
 // so we just exit with the error
 func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
-	logrus.SetReportCaller(true)
+	stdLogAdded := false
 	for _, hook := range hooks {
 		// Check we received a proper logging level
 		level, err := logrus.ParseLevel(hook.Level)
@@ -49,10 +52,18 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
 		case "syslog":
 			checkSyslogHookParams(hook.Params)
 			setupSyslogHook(hook, level, componentName)
+		case "std":
+			setupStdLogHook(level)
+			stdLogAdded = true
 		default:
 			logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type)
 		}
 	}
+	if !stdLogAdded {
+		setupStdLogHook(logrus.InfoLevel)
+	}
+	// Hooks are now configured for stdout/err, so throw away the default logger output
+	logrus.SetOutput(ioutil.Discard)
 }
 
 func checkSyslogHookParams(params map[string]interface{}) {
@@ -76,6 +87,10 @@ func checkSyslogHookParams(params map[string]interface{}) {
 
 }
 
+func setupStdLogHook(level logrus.Level) {
+	logrus.AddHook(&logLevelHook{level, stdemuxerhook.New(logrus.StandardLogger())})
+}
+
 func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) {
 	syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName)
 	if err == nil {

roomserver/api/api.go

@@ -12,9 +12,8 @@ import (
 type RoomserverInternalAPI interface {
 	// needed to avoid chicken and egg scenario when setting up the
 	// interdependencies between the roomserver and other input APIs
-	SetFederationAPI(fsAPI fsAPI.FederationInternalAPI)
+	SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing)
 	SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI)
-	SetKeyring(keyRing *gomatrixserverlib.KeyRing)
 
 	InputRoomEvents(
 		ctx context.Context,

roomserver/api/api_trace.go

@@ -17,12 +17,8 @@ type RoomserverInternalAPITrace struct {
 	Impl RoomserverInternalAPI
 }
 
-func (t *RoomserverInternalAPITrace) SetKeyring(keyRing *gomatrixserverlib.KeyRing) {
+func (t *RoomserverInternalAPITrace) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) {
-	t.Impl.SetKeyring(keyRing)
+	t.Impl.SetFederationAPI(fsAPI, keyRing)
-}
-
-func (t *RoomserverInternalAPITrace) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI) {
-	t.Impl.SetFederationAPI(fsAPI)
 }
 
 func (t *RoomserverInternalAPITrace) SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI) {

roomserver/internal/api.go

@@ -73,18 +73,12 @@ func NewRoomserverAPI(
 	return a
 }
 
-// SetKeyring sets the keyring to a given keyring. This is only useful for the P2P
-// demos and must be called after SetFederationSenderInputAPI.
-func (r *RoomserverInternalAPI) SetKeyring(keyRing *gomatrixserverlib.KeyRing) {
-	r.KeyRing = keyRing
-}
-
 // SetFederationInputAPI passes in a federation input API reference so that we can
 // avoid the chicken-and-egg problem of both the roomserver input API and the
 // federation input API being interdependent.
-func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI) {
+func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) {
 	r.fsAPI = fsAPI
-	r.SetKeyring(fsAPI.KeyRing())
+	r.KeyRing = keyRing
 
 	r.Inviter = &perform.Inviter{
 		DB:      r.DB,

roomserver/internal/input/input_events.go

@@ -122,7 +122,7 @@ func (r *Inputer) processRoomEvent(
 	}
 
 	// Store the event.
-	_, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected)
+	_, _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected)
 	if err != nil {
 		return "", fmt.Errorf("r.DB.StoreEvent: %w", err)
 	}

roomserver/internal/perform/perform_backfill.go

@@ -546,6 +546,7 @@ func joinEventsFromHistoryVisibility(
 
 func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixserverlib.HeaderedEvent) (types.RoomNID, map[string]types.Event) {
 	var roomNID types.RoomNID
+	var eventNID types.EventNID
 	backfilledEventMap := make(map[string]types.Event)
 	for j, ev := range events {
 		nidMap, err := db.EventNIDs(ctx, ev.AuthEventIDs())
@@ -559,10 +560,9 @@ func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixs
 			authNids[i] = nid
 			i++
 		}
-		var stateAtEvent types.StateAtEvent
 		var redactedEventID string
 		var redactionEvent *gomatrixserverlib.Event
-		roomNID, stateAtEvent, redactionEvent, redactedEventID, err = db.StoreEvent(ctx, ev.Unwrap(), authNids, false)
+		eventNID, roomNID, _, redactionEvent, redactedEventID, err = db.StoreEvent(ctx, ev.Unwrap(), authNids, false)
 		if err != nil {
 			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to persist event")
 			continue
@@ -581,7 +581,7 @@ func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixs
 			events[j] = ev
 		}
 		backfilledEventMap[ev.EventID()] = types.Event{
-			EventNID: stateAtEvent.StateEntry.EventNID,
+			EventNID: eventNID,
 			Event:    ev.Unwrap(),
 		}
 	}

roomserver/inthttp/client.go

@@ -83,12 +83,8 @@ func NewRoomserverClient(
 	}, nil
 }
 
-// SetKeyring no-ops in HTTP client mode as there is no chicken/egg scenario
-func (h *httpRoomserverInternalAPI) SetKeyring(keyRing *gomatrixserverlib.KeyRing) {
-}
-
 // SetFederationInputAPI no-ops in HTTP client mode as there is no chicken/egg scenario
-func (h *httpRoomserverInternalAPI) SetFederationAPI(fsAPI fsInputAPI.FederationInternalAPI) {
+func (h *httpRoomserverInternalAPI) SetFederationAPI(fsAPI fsInputAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) {
 }
 
 // SetAppserviceAPI no-ops in HTTP client mode as there is no chicken/egg scenario

roomserver/storage/interface.go

@@ -70,7 +70,7 @@ type Database interface {
 	StoreEvent(
 		ctx context.Context, event *gomatrixserverlib.Event, authEventNIDs []types.EventNID,
 		isRejected bool,
-	) (types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error)
+	) (types.EventNID, types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error)
 	// Look up the state entries for a list of string event IDs
 	// Returns an error if the there is an error talking to the database
 	// Returns a types.MissingEventError if the event IDs aren't in the database.

roomserver/storage/shared/storage.go

@@ -461,7 +461,7 @@ func (d *Database) GetLatestEventsForUpdate(
 func (d *Database) StoreEvent(
 	ctx context.Context, event *gomatrixserverlib.Event,
 	authEventNIDs []types.EventNID, isRejected bool,
-) (types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error) {
+) (types.EventNID, types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error) {
 	var (
 		roomNID          types.RoomNID
 		eventTypeNID     types.EventTypeNID
@@ -538,7 +538,7 @@ func (d *Database) StoreEvent(
 		return nil
 	})
 	if err != nil {
-		return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.Writer.Do: %w", err)
+		return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.Writer.Do: %w", err)
 	}
 
 	// We should attempt to update the previous events table with any
@@ -551,10 +551,10 @@ func (d *Database) StoreEvent(
 	if prevEvents := event.PrevEvents(); len(prevEvents) > 0 {
 		roomInfo, err = d.RoomInfo(ctx, event.RoomID())
 		if err != nil {
-			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.RoomInfo: %w", err)
+			return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.RoomInfo: %w", err)
 		}
 		if roomInfo == nil && len(prevEvents) > 0 {
-			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID())
+			return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID())
 		}
 		// Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of
 		// GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This
@@ -563,7 +563,7 @@ func (d *Database) StoreEvent(
 		// to do writes however then this will need to go inside `Writer.Do`.
 		updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo)
 		if err != nil {
-			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err)
+			return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err)
 		}
 		// Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents
 		// and EndTransaction in a writer then it's possible for a new write txn to be made between the two
@@ -580,11 +580,11 @@ func (d *Database) StoreEvent(
 			return err
 		})
 		if err != nil {
-			return 0, types.StateAtEvent{}, nil, "", err
+			return 0, 0, types.StateAtEvent{}, nil, "", err
 		}
 	}
 
-	return roomNID, types.StateAtEvent{
+	return eventNID, roomNID, types.StateAtEvent{
 		BeforeStateSnapshotNID: stateNID,
 		StateEntry: types.StateEntry{
 			StateKeyTuple: types.StateKeyTuple{

roomserver/storage/sqlite3/events_table.go

@@ -49,7 +49,8 @@ const eventsSchema = `
 const insertEventSQL = `
 	INSERT INTO roomserver_events (room_nid, event_type_nid, event_state_key_nid, event_id, reference_sha256, auth_event_nids, depth, is_rejected)
 	  VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
-	  ON CONFLICT DO NOTHING;
+	  ON CONFLICT DO NOTHING
+	  RETURNING event_nid, state_snapshot_nid;
 `
 
 const selectEventSQL = "" +
@@ -161,20 +162,13 @@ func (s *eventStatements) InsertEvent(
 ) (types.EventNID, types.StateSnapshotNID, error) {
 	// attempt to insert: the last_row_id is the event NID
 	var eventNID int64
+	var stateNID int64
 	insertStmt := sqlutil.TxStmt(txn, s.insertEventStmt)
-	result, err := insertStmt.ExecContext(
+	err := insertStmt.QueryRowContext(
 		ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID),
 		eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, isRejected,
-	)
+	).Scan(&eventNID, &stateNID)
-	if err != nil {
+	return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err
-		return 0, 0, err
-	}
-	modified, err := result.RowsAffected()
-	if modified == 0 && err == nil {
-		return 0, 0, sql.ErrNoRows
-	}
-	eventNID, err = result.LastInsertId()
-	return types.EventNID(eventNID), 0, err
 }
 
 func (s *eventStatements) SelectEvent(

setup/base/base.go

@@ -21,7 +21,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
@@ -79,7 +78,6 @@ type BaseDendrite struct {
 	SynapseAdminMux        *mux.Router
 	UseHTTPAPIs            bool
 	apiHttpClient          *http.Client
-	httpClient             *http.Client
 	Cfg                    *config.Dendrite
 	Caches                 *caching.Caches
 	DNSCache               *gomatrixserverlib.DNSCache
@@ -183,13 +181,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 			},
 		},
 	}
-	client := http.Client{Timeout: HTTPClientTimeout}
-	if cfg.FederationAPI.Proxy.Enabled {
-		client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{
-			Scheme: cfg.FederationAPI.Proxy.Protocol,
-			Host:   fmt.Sprintf("%s:%d", cfg.FederationAPI.Proxy.Host, cfg.FederationAPI.Proxy.Port),
-		})}
-	}
 
 	// Ideally we would only use SkipClean on routes which we know can allow '/' but due to
 	// https://github.com/gorilla/mux/issues/460 we have to attach this at the top router.
@@ -219,7 +210,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		InternalAPIMux:         mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(),
 		SynapseAdminMux:        mux.NewRouter().SkipClean(true).PathPrefix("/_synapse/").Subrouter().UseEncodedPath(),
 		apiHttpClient:          &apiClient,
-		httpClient:             &client,
 	}
 }
 

setup/mscs/msc2836/msc2836_test.go

@@ -7,7 +7,6 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"sort"
@@ -504,7 +503,7 @@ type testUserAPI struct {
 func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error {
 	dev, ok := u.accessTokens[req.AccessToken]
 	if !ok {
-		res.Err = fmt.Errorf("unknown token")
+		res.Err = "unknown token"
 		return nil
 	}
 	res.Device = &dev

setup/mscs/msc2946/msc2946_test.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"crypto/ed25519"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -347,7 +346,7 @@ type testUserAPI struct {
 func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error {
 	dev, ok := u.accessTokens[req.AccessToken]
 	if !ok {
-		res.Err = fmt.Errorf("unknown token")
+		res.Err = "unknown token"
 		return nil
 	}
 	res.Device = &dev

syncapi/consumers/eduserver_typing.go

@@ -74,6 +74,7 @@ func NewOutputTypingEventConsumer(
 func (s *OutputTypingEventConsumer) Start() error {
 	s.eduCache.SetTimeoutCallback(func(userID, roomID string, latestSyncPosition int64) {
 		pos := types.StreamPosition(latestSyncPosition)
+		s.stream.Advance(pos)
 		s.notifier.OnNewTyping(roomID, types.StreamingToken{TypingPosition: pos})
 	})
 	return s.typingConsumer.Start()

userapi/api/api.go

@@ -35,7 +35,7 @@ type UserInternalAPI interface {
 	PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error
 	PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error
 	PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error
-	PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse)
+	PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error
 	QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse)
 	QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error
 	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
@@ -186,7 +186,7 @@ type QueryAccessTokenRequest struct {
 // QueryAccessTokenResponse is the response for QueryAccessToken
 type QueryAccessTokenResponse struct {
 	Device *Device
-	Err    error // e.g ErrorForbidden
+	Err    string // e.g ErrorForbidden
 }
 
 // QueryAccountDataRequest is the request for QueryAccountData
@@ -295,6 +295,10 @@ type PerformDeviceCreationRequest struct {
 	IPAddr string
 	// Useragent for this device
 	UserAgent string
+	// NoDeviceListUpdate determines whether we should avoid sending a device list
+	// update for this account. Generally the only reason to do this is if the account
+	// is an appservice account.
+	NoDeviceListUpdate bool
 }
 
 // PerformDeviceCreationResponse is the response for PerformDeviceCreation

userapi/api/api_trace.go

@@ -74,11 +74,14 @@ func (t *UserInternalAPITrace) PerformOpenIDTokenCreation(ctx context.Context, r
 	util.GetLogger(ctx).Infof("PerformOpenIDTokenCreation req=%+v res=%+v", js(req), js(res))
 	return err
 }
-func (t *UserInternalAPITrace) PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) {
+func (t *UserInternalAPITrace) PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error {
-	t.Impl.PerformKeyBackup(ctx, req, res)
+	err := t.Impl.PerformKeyBackup(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformKeyBackup req=%+v res=%+v", js(req), js(res))
+	return err
 }
 func (t *UserInternalAPITrace) QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) {
 	t.Impl.QueryKeyBackup(ctx, req, res)
+	util.GetLogger(ctx).Infof("QueryKeyBackup req=%+v res=%+v", js(req), js(res))
 }
 func (t *UserInternalAPITrace) QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error {
 	err := t.Impl.QueryProfile(ctx, req, res)

userapi/internal/api.go

@@ -130,6 +130,9 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe
 	}
 	res.DeviceCreated = true
 	res.Device = dev
+	if req.NoDeviceListUpdate {
+		return nil
+	}
 	// create empty device keys and upload them to trigger device list changes
 	return a.deviceListUpdate(dev.UserID, []string{dev.ID})
 }
@@ -369,8 +372,11 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc
 func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error {
 	if req.AppServiceUserID != "" {
 		appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID)
+		if err != nil {
+			res.Err = err.Error()
+		}
 		res.Device = appServiceDevice
-		res.Err = err
+
 		return nil
 	}
 	device, err := a.DeviceDB.GetDeviceByAccessToken(ctx, req.AccessToken)
@@ -525,13 +531,16 @@ func (a *UserInternalAPI) QueryPresenceAfter(ctx context.Context, req *api.Query
 	return nil
 }
 
-func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) {
+func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) error {
 	// Delete metadata
 	if req.DeleteBackup {
 		if req.Version == "" {
 			res.BadInput = true
 			res.Error = "must specify a version to delete"
-			return
+			if res.Error != "" {
+				return fmt.Errorf(res.Error)
+			}
+			return nil
 		}
 		exists, err := a.AccountDB.DeleteKeyBackup(ctx, req.UserID, req.Version)
 		if err != nil {
@@ -539,7 +548,10 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		}
 		res.Exists = exists
 		res.Version = req.Version
-		return
+		if res.Error != "" {
+			return fmt.Errorf(res.Error)
+		}
+		return nil
 	}
 	// Create metadata
 	if req.Version == "" {
@@ -549,7 +561,10 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		}
 		res.Exists = err == nil
 		res.Version = version
-		return
+		if res.Error != "" {
+			return fmt.Errorf(res.Error)
+		}
+		return nil
 	}
 	// Update metadata
 	if len(req.Keys.Rooms) == 0 {
@@ -559,10 +574,17 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		}
 		res.Exists = err == nil
 		res.Version = req.Version
-		return
+		if res.Error != "" {
+			return fmt.Errorf(res.Error)
+		}
+		return nil
 	}
 	// Upload Keys for a specific version metadata
 	a.uploadBackupKeys(ctx, req, res)
+	if res.Error != "" {
+		return fmt.Errorf(res.Error)
+	}
+	return nil
 }
 
 func (a *UserInternalAPI) uploadBackupKeys(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) {

userapi/inthttp/client.go

@@ -248,7 +248,7 @@ func (h *httpUserInternalAPI) QueryPresenceForUser(ctx context.Context, req *api
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
 }
 
-func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) {
+func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) error {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformKeyBackup")
 	defer span.Finish()
 
@@ -257,6 +257,7 @@ func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Per
 	if err != nil {
 		res.Error = err.Error()
 	}
+	return nil
 }
 
 func (h *httpUserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyBackupRequest, res *api.QueryKeyBackupResponse) {

userapi/inthttp/server.go

@@ -16,6 +16,7 @@ package inthttp
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -234,6 +235,34 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(QueryKeyBackupPath,
+		httputil.MakeInternalAPI("queryKeyBackup", func(req *http.Request) util.JSONResponse {
+			request := api.QueryKeyBackupRequest{}
+			response := api.QueryKeyBackupResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			s.QueryKeyBackup(req.Context(), &request, &response)
+			if response.Error != "" {
+				return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", response.Error))
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(PerformKeyBackupPath,
+		httputil.MakeInternalAPI("performKeyBackup", func(req *http.Request) util.JSONResponse {
+			request := api.PerformKeyBackupRequest{}
+			response := api.PerformKeyBackupResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			err := s.PerformKeyBackup(req.Context(), &request, &response)
+			if err != nil {
+				return util.JSONResponse{Code: http.StatusBadRequest, JSON: &response}
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	internalAPIMux.Handle(InputPresenceDataPath,
 		httputil.MakeInternalAPI("inputPresenceDataPath", func(req *http.Request) util.JSONResponse {
 			request := api.InputPresenceRequest{}