Merge branch 'master' into feature/proxy

appservice/api/query.go

@@ -84,10 +84,10 @@ type AppServiceQueryAPI interface {
 }
 
 // AppServiceRoomAliasExistsPath is the HTTP path for the RoomAliasExists API
-const AppServiceRoomAliasExistsPath = "/api/appservice/RoomAliasExists"
+const AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists"
 
 // AppServiceUserIDExistsPath is the HTTP path for the UserIDExists API
-const AppServiceUserIDExistsPath = "/api/appservice/UserIDExists"
+const AppServiceUserIDExistsPath = "/appservice/UserIDExists"
 
 // httpAppServiceQueryAPI contains the URL to an appservice query API and a
 // reference to a httpClient used to reach it

appservice/appservice.go

@@ -16,6 +16,7 @@ package appservice
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"sync"
 	"time"
@@ -29,6 +30,7 @@ import (
 	"github.com/matrix-org/dendrite/appservice/workers"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/basecomponent"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/transactions"
@@ -82,9 +84,7 @@ func SetupAppServiceAPIComponent(
 		Cfg: base.Cfg,
 	}
 
-	if base.EnableHTTPAPIs {
+	appserviceQueryAPI.SetupHTTP(base.InternalAPIMux)
-		appserviceQueryAPI.SetupHTTP(http.DefaultServeMux)
-	}
 
 	consumer := consumers.NewOutputRoomEventConsumer(
 		base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB,
@@ -101,7 +101,7 @@ func SetupAppServiceAPIComponent(
 
 	// Set up HTTP Endpoints
 	routing.Setup(
-		base.APIMux, base.Cfg, rsAPI,
+		base.PublicAPIMux, base.Cfg, rsAPI,
 		accountsDB, federation, transactionsCache,
 	)
 
@@ -119,12 +119,12 @@ func generateAppServiceAccount(
 	ctx := context.Background()
 
 	// Create an account for the application service
-	acc, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID)
+	_, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID)
 	if err != nil {
+		if errors.Is(err, internal.ErrUserExists) { // This account already exists
+			return nil
+		}
 		return err
-	} else if acc == nil {
-		// This account already exists
-		return nil
 	}
 
 	// Create a dummy device with a dummy token for the application service

appservice/query/query.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
@@ -182,8 +183,8 @@ func makeHTTPClient() *http.Client {
 
 // SetupHTTP adds the AppServiceQueryPAI handlers to the http.ServeMux. This
 // handles and muxes incoming api requests the to internal AppServiceQueryAPI.
-func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) {
+func (a *AppServiceQueryAPI) SetupHTTP(internalAPIMux *mux.Router) {
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.AppServiceRoomAliasExistsPath,
 		internal.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
 			var request api.RoomAliasExistsRequest
@@ -197,7 +198,7 @@ func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.AppServiceUserIDExistsPath,
 		internal.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse {
 			var request api.UserIDExistsRequest

appservice/routing/routing.go

@@ -27,7 +27,7 @@ import (
 	"github.com/matrix-org/util"
 )
 
-const pathPrefixApp = "/_matrix/app/v1"
+const pathPrefixApp = "/app/v1"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.

appservice/storage/storage_wasm.go

@@ -34,7 +34,7 @@ func NewDatabase(
 	case "postgres":
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
+		return sqlite3.NewDatabase(uri.Path)
 	default:
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	}

build/docker/DendriteJS.Dockerfile

@@ -0,0 +1,107 @@
+# This dockerfile will build dendritejs and hook it up to riot-web, build that then dump the
+# resulting HTML/JS onto an nginx container for hosting. It requires no specific build context
+# as it pulls archives straight from github branches.
+FROM golang:1.13.7-alpine3.11 AS gobuild
+
+# Download and build dendrite
+WORKDIR /build
+ADD https://github.com/matrix-org/dendrite/archive/master.tar.gz /build/master.tar.gz
+RUN tar xvfz master.tar.gz
+WORKDIR /build/dendrite-master
+RUN GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs
+
+
+FROM node:14-stretch AS jsbuild
+# apparently some deps require python
+RUN apt-get update && apt-get -y install python
+
+# Download riot-web and libp2p repos
+WORKDIR /build
+ADD https://github.com/matrix-org/go-http-js-libp2p/archive/master.tar.gz /build/libp2p.tar.gz
+RUN tar xvfz libp2p.tar.gz
+ADD https://github.com/vector-im/riot-web/archive/matthew/p2p.tar.gz /build/p2p.tar.gz
+RUN tar xvfz p2p.tar.gz
+
+# Install deps for riot-web, symlink in libp2p repo and build that too
+WORKDIR /build/riot-web-matthew-p2p
+RUN yarn install
+RUN ln -s /build/go-http-js-libp2p-master /build/riot-web-matthew-p2p/node_modules/go-http-js-libp2p
+RUN (cd node_modules/go-http-js-libp2p && yarn install)
+COPY --from=gobuild /build/dendrite-master/main.wasm ./src/vector/dendrite.wasm
+# build it all
+RUN yarn build:p2p
+
+SHELL ["/bin/bash", "-c"]
+RUN echo $'\ 
+{ \n\
+    "default_server_config": { \n\
+        "m.homeserver": { \n\
+            "base_url": "https://p2p.riot.im", \n\
+            "server_name": "p2p.riot.im" \n\
+        }, \n\
+        "m.identity_server": { \n\
+            "base_url": "https://vector.im" \n\
+        } \n\
+    }, \n\
+    "disable_custom_urls": false, \n\
+    "disable_guests": true, \n\
+    "disable_login_language_selector": false, \n\
+    "disable_3pid_login": true, \n\
+    "brand": "Riot", \n\
+    "integrations_ui_url": "https://scalar.vector.im/", \n\
+    "integrations_rest_url": "https://scalar.vector.im/api", \n\
+    "integrations_widgets_urls": [ \n\
+        "https://scalar.vector.im/_matrix/integrations/v1", \n\
+        "https://scalar.vector.im/api", \n\
+        "https://scalar-staging.vector.im/_matrix/integrations/v1", \n\
+        "https://scalar-staging.vector.im/api", \n\
+        "https://scalar-staging.riot.im/scalar/api" \n\
+    ], \n\
+    "integrations_jitsi_widget_url": "https://scalar.vector.im/api/widgets/jitsi.html", \n\
+    "bug_report_endpoint_url": "https://riot.im/bugreports/submit", \n\
+    "defaultCountryCode": "GB", \n\
+    "showLabsSettings": false, \n\
+    "features": { \n\
+        "feature_pinning": "labs", \n\
+        "feature_custom_status": "labs", \n\
+        "feature_custom_tags": "labs", \n\
+        "feature_state_counters": "labs" \n\
+    }, \n\
+    "default_federate": true, \n\
+    "default_theme": "light", \n\
+    "roomDirectory": { \n\
+        "servers": [ \n\
+            "matrix.org" \n\
+        ] \n\
+    }, \n\
+    "welcomeUserId": "", \n\
+    "piwik": { \n\
+        "url": "https://piwik.riot.im/", \n\
+        "whitelistedHSUrls": ["https://matrix.org"], \n\
+        "whitelistedISUrls": ["https://vector.im", "https://matrix.org"], \n\
+        "siteId": 1 \n\
+    }, \n\
+    "enable_presence_by_hs_url": { \n\
+        "https://matrix.org": false, \n\
+        "https://matrix-client.matrix.org": false \n\
+    }, \n\
+    "settingDefaults": { \n\
+        "breadcrumbs": true \n\
+    } \n\
+}' > webapp/config.json
+
+FROM nginx
+# Add "Service-Worker-Allowed: /" header so the worker can sniff traffic on this domain rather
+# than just the path this gets hosted under. NB this newline echo syntax only works on bash.
+SHELL ["/bin/bash", "-c"]
+RUN echo $'\
+server { \n\
+    listen 80; \n\
+    add_header \'Service-Worker-Allowed\' \'/\'; \n\
+    location / { \n\
+        root   /usr/share/nginx/html; \n\
+        index  index.html index.htm; \n\
+    } \n\
+}' > /etc/nginx/conf.d/default.conf
+RUN sed -i 's/}/    application\/wasm  wasm;\n}/g' /etc/nginx/mime.types
+COPY --from=jsbuild /build/riot-web-matthew-p2p/webapp /usr/share/nginx/html

build/docker/hub/config/dendrite-config.yaml

@@ -110,11 +110,13 @@ listen:
     room_server: "room_server:7770"
     client_api: "client_api:7771"
     federation_api: "federation_api:7772"
+    server_key_api: "server_key_api:7778"
     sync_api: "sync_api:7773"
     media_api: "media_api:7774"
     public_rooms_api: "public_rooms_api:7775"
     federation_sender: "federation_sender:7776"
     edu_server: "edu_server:7777"
+    key_server: "key_server:7779"
 
 # The configuration for tracing the dendrite components.
 tracing:

build/docker/hub/docker-compose.polylith.yml

@@ -131,7 +131,7 @@ services:
       - internal
       
   key_server:
-    hostname: key_serverde
+    hostname: key_server
     image: matrixdotorg/dendrite:keyserver
     command: [
       "--config=dendrite.yaml"
@@ -141,6 +141,17 @@ services:
     networks:
         - internal
 
+  server_key_api:
+    hostname: server_key_api
+    image: matrixdotorg/dendrite:serverkeyapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
 networks:
   internal:
     attachable: true

build/docker/hub/images-build.sh

@@ -2,16 +2,17 @@
 
 cd $(git rev-parse --show-toplevel)
 
-docker build -f docker/hub/Dockerfile -t matrixdotorg/dendrite:latest .
+docker build -f build/docker/hub/Dockerfile -t matrixdotorg/dendrite:latest .
 
-docker build -t matrixdotorg/dendrite:clientapi         --build-arg component=dendrite-client-api-server        -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:clientapi         --build-arg component=dendrite-client-api-server        -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:clientproxy       --build-arg component=client-api-proxy                  -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:clientproxy       --build-arg component=client-api-proxy                  -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:eduserver         --build-arg component=dendrite-edu-server               -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:eduserver         --build-arg component=dendrite-edu-server               -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationapi     --build-arg component=dendrite-federation-api-server    -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:federationapi     --build-arg component=dendrite-federation-api-server    -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationsender  --build-arg component=dendrite-federation-sender-server -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:federationsender  --build-arg component=dendrite-federation-sender-server -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationproxy   --build-arg component=federation-api-proxy              -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:federationproxy   --build-arg component=federation-api-proxy              -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:keyserver         --build-arg component=dendrite-key-server               -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:keyserver         --build-arg component=dendrite-key-server               -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:mediaapi          --build-arg component=dendrite-media-api-server         -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:mediaapi          --build-arg component=dendrite-media-api-server         -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:publicroomsapi    --build-arg component=dendrite-public-rooms-api-server  -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:publicroomsapi    --build-arg component=dendrite-public-rooms-api-server  -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:roomserver        --build-arg component=dendrite-room-server              -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:roomserver        --build-arg component=dendrite-room-server              -f build/docker/hub/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:syncapi           --build-arg component=dendrite-sync-api-server          -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:syncapi           --build-arg component=dendrite-sync-api-server          -f build/docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:serverkeyapi      --build-arg component=dendrite-server-key-api-server    -f build/docker/hub/Dockerfile.component .

build/scripts/build-test-lint.sh

@@ -13,4 +13,4 @@ go build ./cmd/...
 ./scripts/find-lint.sh
 
 echo "Testing..."
-go test ./...
+go test -v ./...

clientapi/auth/storage/accounts/interface.go

@@ -29,6 +29,9 @@ type Database interface {
 	GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error)
 	SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error
 	SetDisplayName(ctx context.Context, localpart string, displayName string) error
+	// CreateAccount makes a new account with the given login name and password, and creates an empty profile
+	// for this account. If no password is supplied, the account will be a passwordless account. If the
+	// account already exists, it will return nil, ErrUserExists.
 	CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error)
 	CreateGuestAccount(ctx context.Context) (*authtypes.Account, error)
 	UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error

clientapi/auth/storage/accounts/postgres/storage.go

@@ -138,7 +138,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou
 
 // CreateAccount makes a new account with the given login name and password, and creates an empty profile
 // for this account. If no password is supplied, the account will be a passwordless account. If the
-// account already exists, it will return nil, nil.
+// account already exists, it will return nil, ErrUserExists.
 func (d *Database) CreateAccount(
 	ctx context.Context, localpart, plaintextPassword, appserviceID string,
 ) (acc *authtypes.Account, err error) {
@@ -164,7 +164,7 @@ func (d *Database) createAccount(
 	}
 	if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil {
 		if internal.IsUniqueConstraintViolationErr(err) {
-			return nil, nil
+			return nil, internal.ErrUserExists
 		}
 		return nil, err
 	}

clientapi/auth/storage/accounts/sqlite3/constraint.go

@@ -1,5 +1,4 @@
-// Copyright 2017-2018 New Vector Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,24 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// +build !wasm
+
 package sqlite3
 
 import (
-	"database/sql"
+	"errors"
+
+	"github.com/mattn/go-sqlite3"
 )
 
-// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
+func isConstraintError(err error) bool {
-type statementList []struct {
+	return errors.Is(err, sqlite3.ErrConstraint)
-	statement **sql.Stmt
-	sql       string
-}
-
-// prepare the SQL for each statement in the list and assign the result to the prepared statement.
-func (s statementList) prepare(db *sql.DB) (err error) {
-	for _, statement := range s {
-		if *statement.statement, err = db.Prepare(statement.sql); err != nil {
-			return
-		}
-	}
-	return
 }

clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go

@@ -0,0 +1,21 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package sqlite3
+
+func isConstraintError(err error) bool {
+	return false
+}

clientapi/auth/storage/accounts/sqlite3/storage.go

@@ -26,9 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
-
+	// Import the sqlite3 database driver.
-	// Import the postgres database driver.
-	_ "github.com/mattn/go-sqlite3"
 )
 
 // Database represents an account database
@@ -148,7 +146,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou
 
 // CreateAccount makes a new account with the given login name and password, and creates an empty profile
 // for this account. If no password is supplied, the account will be a passwordless account. If the
-// account already exists, it will return nil, nil.
+// account already exists, it will return nil, ErrUserExists.
 func (d *Database) CreateAccount(
 	ctx context.Context, localpart, plaintextPassword, appserviceID string,
 ) (acc *authtypes.Account, err error) {
@@ -172,8 +170,8 @@ func (d *Database) createAccount(
 		}
 	}
 	if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil {
-		if internal.IsUniqueConstraintViolationErr(err) {
+		if isConstraintError(err) {
-			return nil, nil
+			return nil, internal.ErrUserExists
 		}
 		return nil, err
 	}

clientapi/auth/storage/accounts/storage_wasm.go

@@ -36,7 +36,7 @@ func NewDatabase(
 	case "postgres":
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	case "file":
-		return sqlite3.NewDatabase(dataSourceName, serverName)
+		return sqlite3.NewDatabase(uri.Path, serverName)
 	default:
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	}

clientapi/auth/storage/devices/postgres/devices_table.go

@@ -206,9 +206,8 @@ func (s *devicesStatements) selectDeviceByID(
 	ctx context.Context, localpart, deviceID string,
 ) (*authtypes.Device, error) {
 	var dev authtypes.Device
-	var created sql.NullInt64
 	stmt := s.selectDeviceByIDStmt
-	err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&created)
+	err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&dev.DisplayName)
 	if err == nil {
 		dev.ID = deviceID
 		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
@@ -230,10 +229,17 @@ func (s *devicesStatements) selectDevicesByLocalpart(
 
 	for rows.Next() {
 		var dev authtypes.Device
-		err = rows.Scan(&dev.ID, &dev.DisplayName)
+		var id, displayname sql.NullString
+		err = rows.Scan(&id, &displayname)
 		if err != nil {
 			return devices, err
 		}
+		if id.Valid {
+			dev.ID = id.String
+		}
+		if displayname.Valid {
+			dev.DisplayName = displayname.String
+		}
 		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
 		devices = append(devices, dev)
 	}

clientapi/auth/storage/devices/sqlite3/devices_table.go

@@ -208,9 +208,8 @@ func (s *devicesStatements) selectDeviceByID(
 	ctx context.Context, localpart, deviceID string,
 ) (*authtypes.Device, error) {
 	var dev authtypes.Device
-	var created sql.NullInt64
 	stmt := s.selectDeviceByIDStmt
-	err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&created)
+	err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&dev.DisplayName)
 	if err == nil {
 		dev.ID = deviceID
 		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
@@ -231,10 +230,17 @@ func (s *devicesStatements) selectDevicesByLocalpart(
 
 	for rows.Next() {
 		var dev authtypes.Device
-		err = rows.Scan(&dev.ID, &dev.DisplayName)
+		var id, displayname sql.NullString
+		err = rows.Scan(&id, &displayname)
 		if err != nil {
 			return devices, err
 		}
+		if id.Valid {
+			dev.ID = id.String
+		}
+		if displayname.Valid {
+			dev.DisplayName = displayname.String
+		}
 		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
 		devices = append(devices, dev)
 	}

clientapi/auth/storage/devices/storage_wasm.go

@@ -36,7 +36,7 @@ func NewDatabase(
 	case "postgres":
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	case "file":
-		return sqlite3.NewDatabase(dataSourceName, serverName)
+		return sqlite3.NewDatabase(uri.Path, serverName)
 	default:
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	}

clientapi/clientapi.go

@@ -65,7 +65,7 @@ func SetupClientAPIComponent(
 	}
 
 	routing.Setup(
-		base.APIMux, base.Cfg, roomserverProducer, rsAPI, asAPI,
+		base.PublicAPIMux, base.Cfg, roomserverProducer, rsAPI, asAPI,
 		accountsDB, deviceDB, federation, *keyRing, userUpdateProducer,
 		syncProducer, eduProducer, transactionsCache, fsAPI,
 	)

clientapi/producers/eduserver.go

@@ -14,6 +14,7 @@ package producers
 
 import (
 	"context"
+	"encoding/json"
 	"time"
 
 	"github.com/matrix-org/dendrite/eduserver/api"
@@ -52,3 +53,28 @@ func (p *EDUServerProducer) SendTyping(
 
 	return err
 }
+
+// SendToDevice sends a typing event to EDU server
+func (p *EDUServerProducer) SendToDevice(
+	ctx context.Context, sender, userID, deviceID, eventType string,
+	message interface{},
+) error {
+	js, err := json.Marshal(message)
+	if err != nil {
+		return err
+	}
+	requestData := api.InputSendToDeviceEvent{
+		UserID:   userID,
+		DeviceID: deviceID,
+		SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{
+			Sender:  sender,
+			Type:    eventType,
+			Content: js,
+		},
+	}
+	request := api.InputSendToDeviceEventRequest{
+		InputSendToDeviceEvent: requestData,
+	}
+	response := api.InputSendToDeviceEventResponse{}
+	return p.InputAPI.InputSendToDeviceEvent(ctx, &request, &response)
+}

clientapi/routing/directory.go

@@ -77,7 +77,7 @@ func DirectoryRoom(
 			if fedErr != nil {
 				// TODO: Return 502 if the remote server errored.
 				// TODO: Return 504 if the remote server timed out.
-				util.GetLogger(req.Context()).WithError(err).Error("federation.LookupRoomAlias failed")
+				util.GetLogger(req.Context()).WithError(fedErr).Error("federation.LookupRoomAlias failed")
 				return jsonerror.InternalServerError()
 			}
 			res.RoomID = fedRes.RoomID

clientapi/routing/joinroom.go

@@ -75,6 +75,6 @@ func JoinRoomByIDOrAlias(
 		// TODO: Put the response struct somewhere internal.
 		JSON: struct {
 			RoomID string `json:"room_id"`
-		}{joinReq.RoomIDOrAlias},
+		}{joinRes.RoomID},
 	}
 }

clientapi/routing/register.go

@@ -830,15 +830,16 @@ func completeRegistration(
 
 	acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID)
 	if err != nil {
+		if errors.Is(err, internal.ErrUserExists) { // user already exists
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.UserInUse("Desired user ID is already taken."),
+			}
+		}
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.Unknown("failed to create account: " + err.Error()),
 		}
-	} else if acc == nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.UserInUse("Desired user ID is already taken."),
-		}
 	}
 
 	// Increment prometheus counter for created users

clientapi/routing/routing.go

@@ -36,9 +36,9 @@ import (
 	"github.com/matrix-org/util"
 )
 
-const pathPrefixV1 = "/_matrix/client/api/v1"
+const pathPrefixV1 = "/client/api/v1"
-const pathPrefixR0 = "/_matrix/client/r0"
+const pathPrefixR0 = "/client/r0"
-const pathPrefixUnstable = "/_matrix/client/unstable"
+const pathPrefixUnstable = "/client/unstable"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
@@ -47,7 +47,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable"
 // applied:
 // nolint: gocyclo
 func Setup(
-	apiMux *mux.Router, cfg *config.Dendrite,
+	publicAPIMux *mux.Router, cfg *config.Dendrite,
 	producer *producers.RoomserverProducer,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
@@ -62,7 +62,7 @@ func Setup(
 	federationSender federationSenderAPI.FederationSenderInternalAPI,
 ) {
 
-	apiMux.Handle("/_matrix/client/versions",
+	publicAPIMux.Handle("/client/versions",
 		internal.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusOK,
@@ -78,9 +78,9 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+	r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter()
-	v1mux := apiMux.PathPrefix(pathPrefixV1).Subrouter()
+	v1mux := publicAPIMux.PathPrefix(pathPrefixV1).Subrouter()
-	unstableMux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter()
+	unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter()
 
 	authData := auth.Data{
 		AccountDB:   accountDB,
@@ -274,6 +274,31 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
+	r0mux.Handle("/sendToDevice/{eventType}/{txnID}",
+		internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			txnID := vars["txnID"]
+			return SendToDevice(req, device, eduProducer, transactionsCache, vars["eventType"], &txnID)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	// This is only here because sytest refers to /unstable for this endpoint
+	// rather than r0. It's an exact duplicate of the above handler.
+	// TODO: Remove this if/when sytest is fixed!
+	unstableMux.Handle("/sendToDevice/{eventType}/{txnID}",
+		internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			txnID := vars["txnID"]
+			return SendToDevice(req, device, eduProducer, transactionsCache, vars["eventType"], &txnID)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
 	r0mux.Handle("/account/whoami",
 		internal.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			return Whoami(req, device)

clientapi/routing/sendtodevice.go

@@ -0,0 +1,70 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/internal/transactions"
+	"github.com/matrix-org/util"
+)
+
+// SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}
+// sends the device events to the EDU Server
+func SendToDevice(
+	req *http.Request, device *authtypes.Device,
+	eduProducer *producers.EDUServerProducer,
+	txnCache *transactions.Cache,
+	eventType string, txnID *string,
+) util.JSONResponse {
+	if txnID != nil {
+		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
+			return *res
+		}
+	}
+
+	var httpReq struct {
+		Messages map[string]map[string]json.RawMessage `json:"messages"`
+	}
+	resErr := httputil.UnmarshalJSONRequest(req, &httpReq)
+	if resErr != nil {
+		return *resErr
+	}
+
+	for userID, byUser := range httpReq.Messages {
+		for deviceID, message := range byUser {
+			if err := eduProducer.SendToDevice(
+				req.Context(), device.UserID, userID, deviceID, eventType, message,
+			); err != nil {
+				util.GetLogger(req.Context()).WithError(err).Error("eduProducer.SendToDevice failed")
+				return jsonerror.InternalServerError()
+			}
+		}
+	}
+
+	res := util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+
+	if txnID != nil {
+		txnCache.AddTransaction(device.AccessToken, *txnID, &res)
+	}
+
+	return res
+}

cmd/client-api-proxy/main.go

@@ -75,7 +75,6 @@ func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
 			// Pratically this means that any distinction between '%2F' and '/'
 			// in the URL will be lost by the time it reaches the target.
 			path := req.URL.Path
-			path = "api" + path
 			log.WithFields(log.Fields{
 				"path":   path,
 				"url":    targetURL,

cmd/create-account/main.go

@@ -69,13 +69,10 @@ func main() {
 		os.Exit(1)
 	}
 
-	account, err := accountDB.CreateAccount(context.Background(), *username, *password, "")
+	_, err = accountDB.CreateAccount(context.Background(), *username, *password, "")
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)
-	} else if account == nil {
-		fmt.Println("Username already exists")
-		os.Exit(1)
 	}
 
 	deviceDB, err := devices.NewDatabase(*database, nil, serverName)

cmd/dendrite-client-api-server/main.go

@@ -19,7 +19,6 @@ import (
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/internal/basecomponent"
-	"github.com/matrix-org/dendrite/internal/keydb"
 	"github.com/matrix-org/dendrite/internal/transactions"
 )
 
@@ -31,18 +30,19 @@ func main() {
 
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
-	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
+
+	serverKeyAPI := base.CreateHTTPServerKeyAPIs()
+	keyRing := serverKeyAPI.KeyRing()
 
 	asQuery := base.CreateHTTPAppServiceAPIs()
 	rsAPI := base.CreateHTTPRoomserverAPIs()
 	fsAPI := base.CreateHTTPFederationSenderAPIs()
 	rsAPI.SetFederationSenderAPI(fsAPI)
-	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
+	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB)
 
 	clientapi.SetupClientAPIComponent(
-		base, deviceDB, accountDB, federation, &keyRing,
+		base, deviceDB, accountDB, federation, keyRing,
 		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI,
 	)
 

cmd/dendrite-demo-libp2p/main.go

@@ -37,33 +37,23 @@ import (
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/dendrite/internal/keydb"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/mediaapi"
 	"github.com/matrix-org/dendrite/publicroomsapi"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/serverkeyapi"
 	"github.com/matrix-org/dendrite/syncapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/eduserver/cache"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 )
 
 func createKeyDB(
 	base *P2PDendrite,
-) keydb.Database {
+	db gomatrixserverlib.KeyDatabase,
-	db, err := keydb.NewDatabase(
+) {
-		string(base.Base.Cfg.Database.ServerKey),
-		base.Base.Cfg.DbProperties(),
-		base.Base.Cfg.Matrix.ServerName,
-		base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
-		base.Base.Cfg.Matrix.KeyID,
-	)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to keys db")
-	}
 	mdns := mDNSListener{
 		host:  base.LibP2P,
 		keydb: db,
@@ -78,7 +68,6 @@ func createKeyDB(
 		panic(err)
 	}
 	serv.RegisterNotifee(&mdns)
-	return db
 }
 
 func createFederationClient(
@@ -145,45 +134,52 @@ func main() {
 
 	accountDB := base.Base.CreateAccountsDB()
 	deviceDB := base.Base.CreateDeviceDB()
-	keyDB := createKeyDB(base)
 	federation := createFederationClient(base)
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
+
+	serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent(
+		&base.Base, federation,
+	)
+	keyRing := serverKeyAPI.KeyRing()
+	createKeyDB(
+		base, serverKeyAPI,
+	)
 
 	rsAPI := roomserver.SetupRoomServerComponent(
 		&base.Base, keyRing, federation,
 	)
 	eduInputAPI := eduserver.SetupEDUServerComponent(
-		&base.Base, cache.New(),
+		&base.Base, cache.New(), deviceDB,
 	)
 	asAPI := appservice.SetupAppServiceAPIComponent(
 		&base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
 	)
 	fsAPI := federationsender.SetupFederationSenderComponent(
-		&base.Base, federation, rsAPI, &keyRing,
+		&base.Base, federation, rsAPI, keyRing,
 	)
 	rsAPI.SetFederationSenderAPI(fsAPI)
 
 	clientapi.SetupClientAPIComponent(
 		&base.Base, deviceDB, accountDB,
-		federation, &keyRing, rsAPI,
+		federation, keyRing, rsAPI,
 		eduInputAPI, asAPI, transactions.New(), fsAPI,
 	)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-	federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
+	federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB)
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub)
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
 	publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later
 	syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg)
 
-	httpHandler := internal.WrapHandlerInCORS(base.Base.APIMux)
+	internal.SetupHTTPAPI(
-
+		http.DefaultServeMux,
-	// Set up the API endpoints we handle. /metrics is for prometheus, and is
+		base.Base.PublicAPIMux,
-	// not wrapped by CORS, while everything else is
+		base.Base.InternalAPIMux,
-	http.Handle("/metrics", promhttp.Handler())
+		&cfg,
-	http.Handle("/", httpHandler)
+		base.Base.EnableHTTPAPIs,
+	)
 
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {

cmd/dendrite-demo-libp2p/mdnslistener.go

@@ -21,12 +21,11 @@ import (
 
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/matrix-org/dendrite/internal/keydb"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type mDNSListener struct {
-	keydb keydb.Database
+	keydb gomatrixserverlib.KeyDatabase
 	host  host.Host
 }
 

cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go

@@ -44,8 +44,8 @@ type PublicRoomsServerDatabase struct {
 }
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil)
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName)
 	if err != nil {
 		return nil, err
 	}

cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go

@@ -47,8 +47,8 @@ type PublicRoomsServerDatabase struct {
 }
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil)
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName)
 	if err != nil {
 		return nil, err
 	}

cmd/dendrite-demo-libp2p/storage/storage.go

@@ -23,39 +23,40 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const schemePostgres = "postgres"
 const schemeFile = "file"
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT) (storage.Database, error) {
+func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (storage.Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht)
+		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
 	}
 	switch uri.Scheme {
 	case schemePostgres:
-		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht)
+		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
 	case schemeFile:
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
+		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
 	default:
-		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht)
+		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
 	}
 }
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub) (storage.Database, error) {
+func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (storage.Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub)
+		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
 	}
 	switch uri.Scheme {
 	case schemePostgres:
-		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub)
+		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
 	case schemeFile:
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
+		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
 	default:
-		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub)
+		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
 	}
 }

cmd/dendrite-edu-server/main.go

@@ -29,8 +29,9 @@ func main() {
 			logrus.WithError(err).Warn("BaseDendrite close failed")
 		}
 	}()
+	deviceDB := base.CreateDeviceDB()
 
-	eduserver.SetupEDUServerComponent(base, cache.New())
+	eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer))
 

cmd/dendrite-federation-api-server/main.go

@@ -20,7 +20,6 @@ import (
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/internal/basecomponent"
-	"github.com/matrix-org/dendrite/internal/keydb"
 )
 
 func main() {
@@ -30,19 +29,21 @@ func main() {
 
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
-	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
+
+	serverKeyAPI := base.CreateHTTPServerKeyAPIs()
+	keyRing := serverKeyAPI.KeyRing()
+
 	fsAPI := base.CreateHTTPFederationSenderAPIs()
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
 	rsAPI := base.CreateHTTPRoomserverAPIs()
 	asAPI := base.CreateHTTPAppServiceAPIs()
 	rsAPI.SetFederationSenderAPI(fsAPI)
-	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
+	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 
 	federationapi.SetupFederationAPIComponent(
-		base, accountDB, deviceDB, federation, &keyRing,
+		base, accountDB, deviceDB, federation, keyRing,
 		rsAPI, asAPI, fsAPI, eduProducer,
 	)
 

cmd/dendrite-federation-sender-server/main.go

@@ -17,7 +17,6 @@ package main
 import (
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/internal/basecomponent"
-	"github.com/matrix-org/dendrite/internal/keydb"
 )
 
 func main() {
@@ -26,11 +25,13 @@ func main() {
 	defer base.Close() // nolint: errcheck
 
 	federation := base.CreateFederationClient()
-	keyDB := base.CreateKeyDB()
+
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
+	serverKeyAPI := base.CreateHTTPServerKeyAPIs()
+	keyRing := serverKeyAPI.KeyRing()
+
 	rsAPI := base.CreateHTTPRoomserverAPIs()
 	fsAPI := federationsender.SetupFederationSenderComponent(
-		base, federation, rsAPI, &keyRing,
+		base, federation, rsAPI, keyRing,
 	)
 	rsAPI.SetFederationSenderAPI(fsAPI)
 

cmd/dendrite-monolith-server/main.go

@@ -27,15 +27,15 @@ import (
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/basecomponent"
-	"github.com/matrix-org/dendrite/internal/keydb"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/mediaapi"
 	"github.com/matrix-org/dendrite/publicroomsapi"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/serverkeyapi"
 	"github.com/matrix-org/dendrite/syncapi"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"github.com/sirupsen/logrus"
 )
@@ -45,60 +45,95 @@ var (
 	httpsBindAddr  = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
 	certFile       = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
 	keyFile        = flag.String("tls-key", "", "The PEM private key to use for TLS")
-	enableHTTPAPIs = flag.Bool("api", false, "Expose internal HTTP APIs in monolith mode")
+	enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)")
 )
 
 func main() {
 	cfg := basecomponent.ParseMonolithFlags()
+	if *enableHTTPAPIs {
+		// If the HTTP APIs are enabled then we need to update the Listen
+		// statements in the configuration so that we know where to find
+		// the API endpoints. They'll listen on the same port as the monolith
+		// itself.
+		addr := config.Address(*httpBindAddr)
+		cfg.Listen.RoomServer = addr
+		cfg.Listen.EDUServer = addr
+		cfg.Listen.AppServiceAPI = addr
+		cfg.Listen.FederationSender = addr
+		cfg.Listen.ServerKeyAPI = addr
+	}
+
 	base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs)
 	defer base.Close() // nolint: errcheck
 
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
-	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
-	rsAPI := roomserver.SetupRoomServerComponent(
+	serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent(
+		base, federation,
+	)
+	if base.EnableHTTPAPIs {
+		serverKeyAPI = base.CreateHTTPServerKeyAPIs()
+	}
+	keyRing := serverKeyAPI.KeyRing()
+
+	rsComponent := roomserver.SetupRoomServerComponent(
 		base, keyRing, federation,
 	)
+	rsAPI := rsComponent
+	if base.EnableHTTPAPIs {
+		rsAPI = base.CreateHTTPRoomserverAPIs()
+	}
+
 	eduInputAPI := eduserver.SetupEDUServerComponent(
-		base, cache.New(),
+		base, cache.New(), deviceDB,
 	)
+	if base.EnableHTTPAPIs {
+		eduInputAPI = base.CreateHTTPEDUServerAPIs()
+	}
+
 	asAPI := appservice.SetupAppServiceAPIComponent(
 		base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
 	)
+	if base.EnableHTTPAPIs {
+		asAPI = base.CreateHTTPAppServiceAPIs()
+	}
+
 	fsAPI := federationsender.SetupFederationSenderComponent(
-		base, federation, rsAPI, &keyRing,
+		base, federation, rsAPI, keyRing,
 	)
-	rsAPI.SetFederationSenderAPI(fsAPI)
+	if base.EnableHTTPAPIs {
+		fsAPI = base.CreateHTTPFederationSenderAPIs()
+	}
+	rsComponent.SetFederationSenderAPI(fsAPI)
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB,
-		federation, &keyRing, rsAPI,
+		federation, keyRing, rsAPI,
 		eduInputAPI, asAPI, transactions.New(), fsAPI,
 	)
+
 	keyserver.SetupKeyServerComponent(
 		base, deviceDB, accountDB,
 	)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
-	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
+	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
 	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil)
 	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)
 
-	httpHandler := internal.WrapHandlerInCORS(base.APIMux)
+	internal.SetupHTTPAPI(
-
+		http.DefaultServeMux,
-	// Set up the API endpoints we handle. /metrics is for prometheus, and is
+		base.PublicAPIMux,
-	// not wrapped by CORS, while everything else is
+		base.InternalAPIMux,
-	if cfg.Metrics.Enabled {
+		cfg,
-		http.Handle("/metrics", internal.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth))
+		base.EnableHTTPAPIs,
-	}
+	)
-	http.Handle("/", httpHandler)
 
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {

cmd/dendrite-public-rooms-api-server/main.go

@@ -32,7 +32,7 @@ func main() {
 	rsAPI := base.CreateHTTPRoomserverAPIs()
 	rsAPI.SetFederationSenderAPI(fsAPI)
 
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}

cmd/dendrite-room-server/main.go

@@ -16,7 +16,6 @@ package main
 
 import (
 	"github.com/matrix-org/dendrite/internal/basecomponent"
-	"github.com/matrix-org/dendrite/internal/keydb"
 	"github.com/matrix-org/dendrite/roomserver"
 )
 
@@ -24,9 +23,10 @@ func main() {
 	cfg := basecomponent.ParseFlags()
 	base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI", true)
 	defer base.Close() // nolint: errcheck
-	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
+
+	serverKeyAPI := base.CreateHTTPServerKeyAPIs()
+	keyRing := serverKeyAPI.KeyRing()
 
 	fsAPI := base.CreateHTTPFederationSenderAPIs()
 	rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation)

cmd/dendrite-server-key-api-server/main.go

@@ -0,0 +1,32 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/matrix-org/dendrite/internal/basecomponent"
+	"github.com/matrix-org/dendrite/serverkeyapi"
+)
+
+func main() {
+	cfg := basecomponent.ParseFlags()
+	base := basecomponent.NewBaseDendrite(cfg, "ServerKeyAPI", true)
+	defer base.Close() // nolint: errcheck
+
+	federation := base.CreateFederationClient()
+
+	serverkeyapi.SetupServerKeyAPIComponent(base, federation)
+
+	base.SetupAndServeHTTP(string(base.Cfg.Bind.ServerKeyAPI), string(base.Cfg.Listen.ServerKeyAPI))
+}

cmd/dendritejs/jsServer.go

@@ -49,15 +49,11 @@ func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
 		// we need to put this in an immediately invoked goroutine.
 		go func() {
 			resolve := pargs[0]
-			fmt.Println("Received request:")
-			fmt.Printf("%s\n", httpStr)
 			resStr, err := h.handle(httpStr)
 			errStr := ""
 			if err != nil {
 				errStr = err.Error()
 			}
-			fmt.Println("Sending response:")
-			fmt.Printf("%s\n", resStr)
 			resolve.Invoke(map[string]interface{}{
 				"result": resStr,
 				"error":  errStr,

cmd/dendritejs/keyfetcher.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
 const libp2pMatrixKeyID = "ed25519:libp2p-dendrite"
@@ -63,8 +62,6 @@ func (f *libp2pKeyFetcher) FetchKeys(
 		if err != nil {
 			return nil, fmt.Errorf("Failed to extract raw bytes from public key: %w", err)
 		}
-		util.GetLogger(ctx).Info("libp2pKeyFetcher.FetchKeys: Using public key %v for server name %s", pubKeyBytes, req.ServerName)
-
 		b64Key := gomatrixserverlib.Base64String(pubKeyBytes)
 		res[req] = gomatrixserverlib.PublicKeyLookupResult{
 			VerifyKey: gomatrixserverlib.VerifyKey{
@@ -82,3 +79,8 @@ func (f *libp2pKeyFetcher) FetchKeys(
 func (f *libp2pKeyFetcher) FetcherName() string {
 	return "libp2pKeyFetcher"
 }
+
+// no-op function for storing keys - we don't do any work to fetch them so don't bother storing.
+func (f *libp2pKeyFetcher) StoreKeys(ctx context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error {
+	return nil
+}

cmd/dendritejs/main.go

@@ -39,6 +39,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/syncapi"
 	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
+
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/sirupsen/logrus"
@@ -163,18 +164,19 @@ func main() {
 	cfg := &config.Dendrite{}
 	cfg.SetDefaults()
 	cfg.Kafka.UseNaffka = true
-	cfg.Database.Account = "file:dendritejs_account.db"
+	cfg.Database.Account = "file:/idb/dendritejs_account.db"
-	cfg.Database.AppService = "file:dendritejs_appservice.db"
+	cfg.Database.AppService = "file:/idb/dendritejs_appservice.db"
-	cfg.Database.Device = "file:dendritejs_device.db"
+	cfg.Database.Device = "file:/idb/dendritejs_device.db"
-	cfg.Database.FederationSender = "file:dendritejs_fedsender.db"
+	cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db"
-	cfg.Database.MediaAPI = "file:dendritejs_mediaapi.db"
+	cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db"
-	cfg.Database.Naffka = "file:dendritejs_naffka.db"
+	cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db"
-	cfg.Database.PublicRoomsAPI = "file:dendritejs_publicrooms.db"
+	cfg.Database.PublicRoomsAPI = "file:/idb/dendritejs_publicrooms.db"
-	cfg.Database.RoomServer = "file:dendritejs_roomserver.db"
+	cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db"
-	cfg.Database.ServerKey = "file:dendritejs_serverkey.db"
+	cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db"
-	cfg.Database.SyncAPI = "file:dendritejs_syncapi.db"
+	cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db"
 	cfg.Kafka.Topics.UserUpdates = "user_updates"
 	cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event"
+	cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event"
 	cfg.Kafka.Topics.OutputClientData = "output_client_data"
 	cfg.Kafka.Topics.OutputRoomEvent = "output_room_event"
 	cfg.Matrix.TrustedIDServers = []string{
@@ -194,23 +196,24 @@ func main() {
 
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
-	keyDB := base.CreateKeyDB()
 	federation := createFederationClient(cfg, node)
+
+	fetcher := &libp2pKeyFetcher{}
 	keyRing := gomatrixserverlib.KeyRing{
 		KeyFetchers: []gomatrixserverlib.KeyFetcher{
-			&libp2pKeyFetcher{},
+			fetcher,
 		},
-		KeyDatabase: keyDB,
+		KeyDatabase: fetcher,
 	}
-	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node)
 
 	rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation)
-	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
+	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB)
 	asQuery := appservice.SetupAppServiceAPIComponent(
 		base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
 	)
 	fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing)
 	rsAPI.SetFederationSenderAPI(fedSenderAPI)
+	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI)
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB,
@@ -220,16 +223,20 @@ func main() {
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}
 	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider)
 	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg)
 
-	httpHandler := internal.WrapHandlerInCORS(base.APIMux)
+	internal.SetupHTTPAPI(
-
+		http.DefaultServeMux,
-	http.Handle("/", httpHandler)
+		base.PublicAPIMux,
+		base.InternalAPIMux,
+		cfg,
+		base.EnableHTTPAPIs,
+	)
 
 	// Expose the matrix APIs via libp2p-js - for federation traffic
 	if node != nil {

cmd/dendritejs/publicrooms.go

@@ -17,23 +17,48 @@
 package main
 
 import (
+	"context"
+
+	"github.com/matrix-org/dendrite/federationsender/api"
 	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type libp2pPublicRoomsProvider struct {
 	node      *go_http_js_libp2p.P2pLocalNode
 	providers []go_http_js_libp2p.PeerInfo
+	fedSender api.FederationSenderInternalAPI
 }
 
-func NewLibP2PPublicRoomsProvider(node *go_http_js_libp2p.P2pLocalNode) *libp2pPublicRoomsProvider {
+func NewLibP2PPublicRoomsProvider(node *go_http_js_libp2p.P2pLocalNode, fedSender api.FederationSenderInternalAPI) *libp2pPublicRoomsProvider {
 	p := &libp2pPublicRoomsProvider{
-		node: node,
+		node:      node,
+		fedSender: fedSender,
 	}
 	node.RegisterFoundProviders(p.foundProviders)
 	return p
 }
 
 func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p.PeerInfo) {
+	// work out the diff then poke for new ones
+	seen := make(map[string]bool, len(p.providers))
+	for _, pr := range p.providers {
+		seen[pr.Id] = true
+	}
+	var newPeers []gomatrixserverlib.ServerName
+	for _, pi := range peerInfos {
+		if !seen[pi.Id] {
+			newPeers = append(newPeers, gomatrixserverlib.ServerName(pi.Id))
+		}
+	}
+	if len(newPeers) > 0 {
+		var res api.PerformServersAliveResponse
+		// ignore errors, we don't care.
+		p.fedSender.PerformServersAlive(context.Background(), &api.PerformServersAliveRequest{
+			Servers: newPeers,
+		}, &res)
+	}
+
 	p.providers = peerInfos
 }
 

cmd/federation-api-proxy/main.go

@@ -73,7 +73,6 @@ func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
 			// Pratically this means that any distinction between '%2F' and '/'
 			// in the URL will be lost by the time it reaches the target.
 			path := req.URL.Path
-			path = "api" + path
 			log.WithFields(log.Fields{
 				"path":   path,
 				"url":    targetURL,

dendrite-config.yaml

@@ -104,7 +104,8 @@ kafka:
     topics:
         output_room_event: roomserverOutput
         output_client_data: clientapiOutput
-        output_typing_event: eduServerOutput
+        output_typing_event: eduServerTypingOutput
+        output_send_to_device_event: eduServerSendToDeviceOutput
         user_updates: userUpdates
 
 # The postgres connection configs for connecting to the databases e.g a postgres:// URI
@@ -138,6 +139,7 @@ listen:
     appservice_api: "localhost:7777"
     edu_server: "localhost:7778"
     key_server: "localhost:7779"
+    server_key_api: "localhost:7780"
 
 # The configuration for tracing the dendrite components.
 tracing:

docs/p2p.md

@@ -1,6 +1,6 @@
 ## Peer-to-peer Matrix
 
-These are the instructions for setting up P2P Dendrite, current as of March 2020. There's both Go stuff and JS stuff to do to set this up.
+These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up.
 
 
 ### Dendrite
@@ -28,14 +28,13 @@ Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`.
 
 ### Riot-web
 
-You need to check out these repos:
+You need to check out this repo:
 
 ```
 $ git clone git@github.com:matrix-org/go-http-js-libp2p.git
-$ git clone git@github.com:matrix-org/go-sqlite3-js.git
 ```
 
-Make sure to `yarn install` in both of these repos. Then:
+Make sure to `yarn install` in the repo. Then:
 
 - `$ cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" ./src/vector/`
 - Comment out the lines in `wasm_exec.js` which contains:
@@ -49,7 +48,6 @@ if (!global.fs && global.require) {
 - Add the following symlinks: they HAVE to be symlinks as the diff in `webpack.config.js` references specific paths.
 ```
 $ cd node_modules
-$ ln -s ../../go-sqlite-js # NB: NOT go-sqlite3-js
 $ ln -s ../../go-http-js-libp2p
 ```
 
@@ -65,14 +63,7 @@ You need a Chrome and a Firefox running to test locally as service workers don't
 
 Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and register with `http://localhost:8080` as your HS URL.
 
-You can join rooms by room alias e.g `/join #foo:bar`.
+You can:
-
+ - join rooms by room alias e.g `/join #foo:bar`.
-### Known issues
+ - invite specific users to a room.
-
+ - explore the published room list. All members of the room can re-publish aliases (unlike Synapse).
-- When registering you may be unable to find the server, it'll seem flakey. This happens because the SW, particularly in Firefox,
-  gets killed after 30s of inactivity. When you are not registered, you aren't doing `/sync` calls to keep the SW alive, so if you
-  don't register for a while and idle on the page, the HS will disappear. To fix, unregister the SW, and then refresh the page.
-
-- The libp2p layer has rate limits, so frequent Federation traffic may cause the connection to drop and messages to not be transferred.
-  I guess in other words, don't send too much traffic?
-

eduserver/api/input.go

@@ -1,3 +1,7 @@
+// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -37,6 +41,12 @@ type InputTypingEvent struct {
 	OriginServerTS gomatrixserverlib.Timestamp `json:"origin_server_ts"`
 }
 
+type InputSendToDeviceEvent struct {
+	UserID   string `json:"user_id"`
+	DeviceID string `json:"device_id"`
+	gomatrixserverlib.SendToDeviceEvent
+}
+
 // InputTypingEventRequest is a request to EDUServerInputAPI
 type InputTypingEventRequest struct {
 	InputTypingEvent InputTypingEvent `json:"input_typing_event"`
@@ -45,6 +55,14 @@ type InputTypingEventRequest struct {
 // InputTypingEventResponse is a response to InputTypingEvents
 type InputTypingEventResponse struct{}
 
+// InputSendToDeviceEventRequest is a request to EDUServerInputAPI
+type InputSendToDeviceEventRequest struct {
+	InputSendToDeviceEvent InputSendToDeviceEvent `json:"input_send_to_device_event"`
+}
+
+// InputSendToDeviceEventResponse is a response to InputSendToDeviceEventRequest
+type InputSendToDeviceEventResponse struct{}
+
 // EDUServerInputAPI is used to write events to the typing server.
 type EDUServerInputAPI interface {
 	InputTypingEvent(
@@ -52,10 +70,19 @@ type EDUServerInputAPI interface {
 		request *InputTypingEventRequest,
 		response *InputTypingEventResponse,
 	) error
+
+	InputSendToDeviceEvent(
+		ctx context.Context,
+		request *InputSendToDeviceEventRequest,
+		response *InputSendToDeviceEventResponse,
+	) error
 }
 
 // EDUServerInputTypingEventPath is the HTTP path for the InputTypingEvent API.
-const EDUServerInputTypingEventPath = "/api/eduserver/input"
+const EDUServerInputTypingEventPath = "/eduserver/input"
+
+// EDUServerInputSendToDeviceEventPath is the HTTP path for the InputSendToDeviceEvent API.
+const EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice"
 
 // NewEDUServerInputAPIHTTP creates a EDUServerInputAPI implemented by talking to a HTTP POST API.
 func NewEDUServerInputAPIHTTP(eduServerURL string, httpClient *http.Client) (EDUServerInputAPI, error) {
@@ -70,7 +97,7 @@ type httpEDUServerInputAPI struct {
 	httpClient   *http.Client
 }
 
-// InputRoomEvents implements EDUServerInputAPI
+// InputTypingEvent implements EDUServerInputAPI
 func (h *httpEDUServerInputAPI) InputTypingEvent(
 	ctx context.Context,
 	request *InputTypingEventRequest,
@@ -82,3 +109,16 @@ func (h *httpEDUServerInputAPI) InputTypingEvent(
 	apiURL := h.eduServerURL + EDUServerInputTypingEventPath
 	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
+
+// InputSendToDeviceEvent implements EDUServerInputAPI
+func (h *httpEDUServerInputAPI) InputSendToDeviceEvent(
+	ctx context.Context,
+	request *InputSendToDeviceEventRequest,
+	response *InputSendToDeviceEventResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "InputSendToDeviceEvent")
+	defer span.Finish()
+
+	apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath
+	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

eduserver/api/output.go

@@ -1,3 +1,7 @@
+// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -12,7 +16,11 @@
 
 package api
 
-import "time"
+import (
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
 
 // OutputTypingEvent is an entry in typing server output kafka log.
 // This contains the event with extra fields used to create 'm.typing' event
@@ -32,3 +40,12 @@ type TypingEvent struct {
 	UserID string `json:"user_id"`
 	Typing bool   `json:"typing"`
 }
+
+// OutputSendToDeviceEvent is an entry in the send-to-device output kafka log.
+// This contains the full event content, along with the user ID and device ID
+// to which it is destined.
+type OutputSendToDeviceEvent struct {
+	UserID   string `json:"user_id"`
+	DeviceID string `json:"device_id"`
+	gomatrixserverlib.SendToDeviceEvent
+}

eduserver/cache/cache.go

@@ -1,3 +1,7 @@
+// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -109,6 +113,19 @@ func (t *EDUCache) AddTypingUser(
 	return t.GetLatestSyncPosition()
 }
 
+// AddSendToDeviceMessage increases the sync position for
+// send-to-device updates.
+// Returns the sync position before update, as the caller
+// will use this to record the current stream position
+// at the time that the send-to-device message was sent.
+func (t *EDUCache) AddSendToDeviceMessage() int64 {
+	t.Lock()
+	defer t.Unlock()
+	latestSyncPosition := t.latestSyncPosition
+	t.latestSyncPosition++
+	return latestSyncPosition
+}
+
 // addUser with mutex lock & replace the previous timer.
 // Returns the latest typing sync position after update.
 func (t *EDUCache) addUser(

eduserver/cache/cache_test.go

@@ -1,3 +1,7 @@
+// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at

eduserver/eduserver.go

@@ -1,3 +1,7 @@
+// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -13,8 +17,7 @@
 package eduserver
 
 import (
-	"net/http"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
-
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/eduserver/input"
@@ -28,16 +31,18 @@ import (
 func SetupEDUServerComponent(
 	base *basecomponent.BaseDendrite,
 	eduCache *cache.EDUCache,
+	deviceDB devices.Database,
 ) api.EDUServerInputAPI {
 	inputAPI := &input.EDUServerInputAPI{
-		Cache:                  eduCache,
+		Cache:                        eduCache,
-		Producer:               base.KafkaProducer,
+		DeviceDB:                     deviceDB,
-		OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent),
+		Producer:                     base.KafkaProducer,
+		OutputTypingEventTopic:       string(base.Cfg.Kafka.Topics.OutputTypingEvent),
+		OutputSendToDeviceEventTopic: string(base.Cfg.Kafka.Topics.OutputSendToDeviceEvent),
+		ServerName:                   base.Cfg.Matrix.ServerName,
 	}
 
-	if base.EnableHTTPAPIs {
+	inputAPI.SetupHTTP(base.InternalAPIMux)
-		inputAPI.SetupHTTP(http.DefaultServeMux)
-	}
 
 	return inputAPI
 }

eduserver/input/input.go

@@ -1,3 +1,7 @@
+// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -19,11 +23,14 @@ import (
 	"time"
 
 	"github.com/Shopify/sarama"
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // EDUServerInputAPI implements api.EDUServerInputAPI
@@ -32,8 +39,14 @@ type EDUServerInputAPI struct {
 	Cache *cache.EDUCache
 	// The kafka topic to output new typing events to.
 	OutputTypingEventTopic string
+	// The kafka topic to output new send to device events to.
+	OutputSendToDeviceEventTopic string
 	// kafka producer
 	Producer sarama.SyncProducer
+	// device database
+	DeviceDB devices.Database
+	// our server name
+	ServerName gomatrixserverlib.ServerName
 }
 
 // InputTypingEvent implements api.EDUServerInputAPI
@@ -53,10 +66,20 @@ func (t *EDUServerInputAPI) InputTypingEvent(
 		t.Cache.RemoveUser(ite.UserID, ite.RoomID)
 	}
 
-	return t.sendEvent(ite)
+	return t.sendTypingEvent(ite)
 }
 
-func (t *EDUServerInputAPI) sendEvent(ite *api.InputTypingEvent) error {
+// InputTypingEvent implements api.EDUServerInputAPI
+func (t *EDUServerInputAPI) InputSendToDeviceEvent(
+	ctx context.Context,
+	request *api.InputSendToDeviceEventRequest,
+	response *api.InputSendToDeviceEventResponse,
+) error {
+	ise := &request.InputSendToDeviceEvent
+	return t.sendToDeviceEvent(ise)
+}
+
+func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error {
 	ev := &api.TypingEvent{
 		Type:   gomatrixserverlib.MTyping,
 		RoomID: ite.RoomID,
@@ -89,9 +112,68 @@ func (t *EDUServerInputAPI) sendEvent(ite *api.InputTypingEvent) error {
 	return err
 }
 
+func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) error {
+	devices := []string{}
+	localpart, domain, err := gomatrixserverlib.SplitID('@', ise.UserID)
+	if err != nil {
+		return err
+	}
+
+	// If the event is targeted locally then we want to expand the wildcard
+	// out into individual device IDs so that we can send them to each respective
+	// device. If the event isn't targeted locally then we can't expand the
+	// wildcard as we don't know about the remote devices, so instead we leave it
+	// as-is, so that the federation sender can send it on with the wildcard intact.
+	if domain == t.ServerName && ise.DeviceID == "*" {
+		devs, err := t.DeviceDB.GetDevicesByLocalpart(context.TODO(), localpart)
+		if err != nil {
+			return err
+		}
+		for _, dev := range devs {
+			devices = append(devices, dev.ID)
+		}
+	} else {
+		devices = append(devices, ise.DeviceID)
+	}
+
+	for _, device := range devices {
+		ote := &api.OutputSendToDeviceEvent{
+			UserID:            ise.UserID,
+			DeviceID:          device,
+			SendToDeviceEvent: ise.SendToDeviceEvent,
+		}
+
+		logrus.WithFields(logrus.Fields{
+			"user_id":    ise.UserID,
+			"device_id":  ise.DeviceID,
+			"event_type": ise.Type,
+		}).Info("handling send-to-device message")
+
+		eventJSON, err := json.Marshal(ote)
+		if err != nil {
+			logrus.WithError(err).Error("sendToDevice failed json.Marshal")
+			return err
+		}
+
+		m := &sarama.ProducerMessage{
+			Topic: string(t.OutputSendToDeviceEventTopic),
+			Key:   sarama.StringEncoder(ote.UserID),
+			Value: sarama.ByteEncoder(eventJSON),
+		}
+
+		_, _, err = t.Producer.SendMessage(m)
+		if err != nil {
+			logrus.WithError(err).Error("sendToDevice failed t.Producer.SendMessage")
+			return err
+		}
+	}
+
+	return nil
+}
+
 // SetupHTTP adds the EDUServerInputAPI handlers to the http.ServeMux.
-func (t *EDUServerInputAPI) SetupHTTP(servMux *http.ServeMux) {
+func (t *EDUServerInputAPI) SetupHTTP(internalAPIMux *mux.Router) {
-	servMux.Handle(api.EDUServerInputTypingEventPath,
+	internalAPIMux.Handle(api.EDUServerInputTypingEventPath,
 		internal.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse {
 			var request api.InputTypingEventRequest
 			var response api.InputTypingEventResponse
@@ -104,4 +186,17 @@ func (t *EDUServerInputAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(api.EDUServerInputSendToDeviceEventPath,
+		internal.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse {
+			var request api.InputSendToDeviceEventRequest
+			var response api.InputSendToDeviceEventResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := t.InputSendToDeviceEvent(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }

federationapi/federationapi.go

@@ -44,7 +44,7 @@ func SetupFederationAPIComponent(
 	roomserverProducer := producers.NewRoomserverProducer(rsAPI)
 
 	routing.Setup(
-		base.APIMux, base.Cfg, rsAPI, asAPI, roomserverProducer,
+		base.PublicAPIMux, base.Cfg, rsAPI, asAPI, roomserverProducer,
 		eduProducer, federationSenderAPI, *keyRing,
 		federation, accountsDB, deviceDB,
 	)

federationapi/routing/devices.go

@@ -15,19 +15,13 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
-type userDevicesResponse struct {
-	UserID   string             `json:"user_id"`
-	StreamID int                `json:"stream_id"`
-	Devices  []authtypes.Device `json:"devices"`
-}
-
 // GetUserDevices for the given user id
 func GetUserDevices(
 	req *http.Request,
@@ -42,20 +36,30 @@ func GetUserDevices(
 		}
 	}
 
+	response := gomatrixserverlib.RespUserDevices{
+		UserID: userID,
+		// TODO: we should return an incrementing stream ID each time the device
+		// list changes for delta changes to be recognised
+		StreamID: 0,
+	}
+
 	devs, err := deviceDB.GetDevicesByLocalpart(req.Context(), localpart)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDevicesByLocalPart failed")
 		return jsonerror.InternalServerError()
 	}
 
+	for _, dev := range devs {
+		device := gomatrixserverlib.RespUserDevice{
+			DeviceID:    dev.ID,
+			DisplayName: dev.DisplayName,
+			Keys:        []gomatrixserverlib.RespUserDeviceKeys{},
+		}
+		response.Devices = append(response.Devices, device)
+	}
+
 	return util.JSONResponse{
 		Code: 200,
-		// TODO: we should return an incrementing stream ID each time the device
+		JSON: response,
-		// list changes for delta changes to be recognised
-		JSON: userDevicesResponse{
-			UserID:   userID,
-			StreamID: 0,
-			Devices:  devs,
-		},
 	}
 }

federationapi/routing/missingevents.go

@@ -60,9 +60,14 @@ func GetMissingEvents(
 	}
 
 	eventsResponse.Events = filterEvents(eventsResponse.Events, gme.MinDepth, roomID)
+
+	resp := gomatrixserverlib.RespMissingEvents{
+		Events: gomatrixserverlib.UnwrapEventHeaders(eventsResponse.Events),
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: eventsResponse,
+		JSON: resp,
 	}
 }
 

federationapi/routing/routing.go

@@ -31,9 +31,9 @@ import (
 )
 
 const (
-	pathPrefixV2Keys       = "/_matrix/key/v2"
+	pathPrefixV2Keys       = "/key/v2"
-	pathPrefixV1Federation = "/_matrix/federation/v1"
+	pathPrefixV1Federation = "/federation/v1"
-	pathPrefixV2Federation = "/_matrix/federation/v2"
+	pathPrefixV2Federation = "/federation/v2"
 )
 
 // Setup registers HTTP handlers with the given ServeMux.
@@ -42,21 +42,25 @@ const (
 // applied:
 // nolint: gocyclo
 func Setup(
-	apiMux *mux.Router,
+	publicAPIMux *mux.Router,
 	cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	producer *producers.RoomserverProducer,
 	eduProducer *producers.EDUServerProducer,
-	federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
+	fsAPI federationSenderAPI.FederationSenderInternalAPI,
 	keys gomatrixserverlib.KeyRing,
 	federation *gomatrixserverlib.FederationClient,
 	accountDB accounts.Database,
 	deviceDB devices.Database,
 ) {
-	v2keysmux := apiMux.PathPrefix(pathPrefixV2Keys).Subrouter()
+	v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter()
-	v1fedmux := apiMux.PathPrefix(pathPrefixV1Federation).Subrouter()
+	v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter()
-	v2fedmux := apiMux.PathPrefix(pathPrefixV2Federation).Subrouter()
+	v2fedmux := publicAPIMux.PathPrefix(pathPrefixV2Federation).Subrouter()
+
+	wakeup := &internal.FederationWakeups{
+		FsAPI: fsAPI,
+	}
 
 	localKeys := internal.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse {
 		return LocalKeys(cfg)
@@ -71,7 +75,7 @@ func Setup(
 	v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/send/{txnID}", internal.MakeFedAPI(
-		"federation_send", cfg.Matrix.ServerName, keys,
+		"federation_send", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -85,7 +89,7 @@ func Setup(
 	)).Methods(http.MethodPut, http.MethodOptions)
 
 	v2fedmux.Handle("/invite/{roomID}/{eventID}", internal.MakeFedAPI(
-		"federation_invite", cfg.Matrix.ServerName, keys,
+		"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -105,7 +109,7 @@ func Setup(
 	)).Methods(http.MethodPost, http.MethodOptions)
 
 	v1fedmux.Handle("/exchange_third_party_invite/{roomID}", internal.MakeFedAPI(
-		"exchange_third_party_invite", cfg.Matrix.ServerName, keys,
+		"exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -118,7 +122,7 @@ func Setup(
 	)).Methods(http.MethodPut, http.MethodOptions)
 
 	v1fedmux.Handle("/event/{eventID}", internal.MakeFedAPI(
-		"federation_get_event", cfg.Matrix.ServerName, keys,
+		"federation_get_event", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -131,7 +135,7 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/state/{roomID}", internal.MakeFedAPI(
-		"federation_get_state", cfg.Matrix.ServerName, keys,
+		"federation_get_state", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -144,7 +148,7 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/state_ids/{roomID}", internal.MakeFedAPI(
-		"federation_get_state_ids", cfg.Matrix.ServerName, keys,
+		"federation_get_state_ids", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -157,7 +161,7 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/event_auth/{roomID}/{eventID}", internal.MakeFedAPI(
-		"federation_get_event_auth", cfg.Matrix.ServerName, keys,
+		"federation_get_event_auth", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars := mux.Vars(httpReq)
 			return GetEventAuth(
@@ -167,16 +171,16 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/query/directory", internal.MakeFedAPI(
-		"federation_query_room_alias", cfg.Matrix.ServerName, keys,
+		"federation_query_room_alias", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			return RoomAliasToID(
-				httpReq, federation, cfg, rsAPI, federationSenderAPI,
+				httpReq, federation, cfg, rsAPI, fsAPI,
 			)
 		},
 	)).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/query/profile", internal.MakeFedAPI(
-		"federation_query_profile", cfg.Matrix.ServerName, keys,
+		"federation_query_profile", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			return GetProfile(
 				httpReq, accountDB, cfg, asAPI,
@@ -185,7 +189,7 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/user/devices/{userID}", internal.MakeFedAPI(
-		"federation_user_devices", cfg.Matrix.ServerName, keys,
+		"federation_user_devices", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -198,7 +202,7 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/make_join/{roomID}/{eventID}", internal.MakeFedAPI(
-		"federation_make_join", cfg.Matrix.ServerName, keys,
+		"federation_make_join", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -227,7 +231,7 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI(
-		"federation_send_join", cfg.Matrix.ServerName, keys,
+		"federation_send_join", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -249,7 +253,7 @@ func Setup(
 	)).Methods(http.MethodPut)
 
 	v2fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI(
-		"federation_send_join", cfg.Matrix.ServerName, keys,
+		"federation_send_join", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -264,7 +268,7 @@ func Setup(
 	)).Methods(http.MethodPut)
 
 	v1fedmux.Handle("/make_leave/{roomID}/{eventID}", internal.MakeFedAPI(
-		"federation_make_leave", cfg.Matrix.ServerName, keys,
+		"federation_make_leave", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -279,7 +283,7 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	v2fedmux.Handle("/send_leave/{roomID}/{eventID}", internal.MakeFedAPI(
-		"federation_send_leave", cfg.Matrix.ServerName, keys,
+		"federation_send_leave", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -301,7 +305,7 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/get_missing_events/{roomID}", internal.MakeFedAPI(
-		"federation_get_missing_events", cfg.Matrix.ServerName, keys,
+		"federation_get_missing_events", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {
@@ -312,7 +316,7 @@ func Setup(
 	)).Methods(http.MethodPost)
 
 	v1fedmux.Handle("/backfill/{roomID}", internal.MakeFedAPI(
-		"federation_backfill", cfg.Matrix.ServerName, keys,
+		"federation_backfill", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
 			vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq))
 			if err != nil {

federationapi/routing/send.go

@@ -76,6 +76,7 @@ func Send(
 	resp, err := t.processTransaction()
 	if err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed")
+		return util.ErrorResponse(err)
 	}
 
 	// https://matrix.org/docs/spec/server_server/r0.1.3#put-matrix-federation-v1-send-txnid
@@ -117,7 +118,7 @@ type txnFederationClient interface {
 func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
 	results := make(map[string]gomatrixserverlib.PDUResult)
 
-	var pdus []gomatrixserverlib.HeaderedEvent
+	pdus := []gomatrixserverlib.HeaderedEvent{}
 	for _, pdu := range t.PDUs {
 		var header struct {
 			RoomID string `json:"room_id"`
@@ -264,6 +265,25 @@ func (t *txnReq) processEDUs(edus []gomatrixserverlib.EDU) {
 			if err := t.eduProducer.SendTyping(t.context, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil {
 				util.GetLogger(t.context).WithError(err).Error("Failed to send typing event to edu server")
 			}
+		case gomatrixserverlib.MDirectToDevice:
+			// https://matrix.org/docs/spec/server_server/r0.1.3#m-direct-to-device-schema
+			var directPayload gomatrixserverlib.ToDeviceMessage
+			if err := json.Unmarshal(e.Content, &directPayload); err != nil {
+				util.GetLogger(t.context).WithError(err).Error("Failed to unmarshal send-to-device events")
+				continue
+			}
+			for userID, byUser := range directPayload.Messages {
+				for deviceID, message := range byUser {
+					// TODO: check that the user and the device actually exist here
+					if err := t.eduProducer.SendToDevice(t.context, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil {
+						util.GetLogger(t.context).WithError(err).WithFields(logrus.Fields{
+							"sender":    directPayload.Sender,
+							"user_id":   userID,
+							"device_id": deviceID,
+						}).Error("Failed to send send-to-device event to edu server")
+					}
+				}
+			}
 		default:
 			util.GetLogger(t.context).WithField("type", e.Type).Warn("unhandled edu")
 		}

federationapi/routing/send_test.go

@@ -77,6 +77,14 @@ func (p *testEDUProducer) InputTypingEvent(
 	return nil
 }
 
+func (p *testEDUProducer) InputSendToDeviceEvent(
+	ctx context.Context,
+	request *eduAPI.InputSendToDeviceEventRequest,
+	response *eduAPI.InputSendToDeviceEventResponse,
+) error {
+	return nil
+}
+
 type testRoomserverAPI struct {
 	inputRoomEvents           []api.InputRoomEvent
 	queryStateAfterEvents     func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse

federationsender/api/api.go

@@ -42,6 +42,12 @@ type FederationSenderInternalAPI interface {
 		request *PerformLeaveRequest,
 		response *PerformLeaveResponse,
 	) error
+	// Notifies the federation sender that these servers may be online and to retry sending messages.
+	PerformServersAlive(
+		ctx context.Context,
+		request *PerformServersAliveRequest,
+		response *PerformServersAliveResponse,
+	) error
 }
 
 // NewFederationSenderInternalAPIHTTP creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API.

federationsender/api/perform.go

@@ -11,13 +11,16 @@ import (
 
 const (
 	// FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API.
-	FederationSenderPerformDirectoryLookupRequestPath = "/api/federationsender/performDirectoryLookup"
+	FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup"
 
 	// FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API.
-	FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest"
+	FederationSenderPerformJoinRequestPath = "/federationsender/performJoinRequest"
 
 	// FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API.
-	FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest"
+	FederationSenderPerformLeaveRequestPath = "/federationsender/performLeaveRequest"
+
+	// FederationSenderPerformServersAlivePath is the HTTP path for the PerformServersAlive API.
+	FederationSenderPerformServersAlivePath = "/federationsender/performServersAlive"
 )
 
 type PerformDirectoryLookupRequest struct {
@@ -44,8 +47,9 @@ func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup(
 }
 
 type PerformJoinRequest struct {
-	RoomID      string                 `json:"room_id"`
+	RoomID string `json:"room_id"`
-	UserID      string                 `json:"user_id"`
+	UserID string `json:"user_id"`
+	// The sorted list of servers to try. Servers will be tried sequentially, after de-duplication.
 	ServerNames types.ServerNames      `json:"server_names"`
 	Content     map[string]interface{} `json:"content"`
 }
@@ -87,3 +91,22 @@ func (h *httpFederationSenderInternalAPI) PerformLeave(
 	apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath
 	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
+
+type PerformServersAliveRequest struct {
+	Servers []gomatrixserverlib.ServerName
+}
+
+type PerformServersAliveResponse struct {
+}
+
+func (h *httpFederationSenderInternalAPI) PerformServersAlive(
+	ctx context.Context,
+	request *PerformServersAliveRequest,
+	response *PerformServersAliveResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformServersAlive")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderPerformServersAlivePath
+	return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

federationsender/api/query.go

@@ -11,10 +11,10 @@ import (
 )
 
 // FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API.
-const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom"
+const FederationSenderQueryJoinedHostsInRoomPath = "/federationsender/queryJoinedHostsInRoom"
 
 // FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API.
-const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom"
+const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom"
 
 // QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom
 type QueryJoinedHostsInRoomRequest struct {

federationsender/federationsender.go

@@ -15,8 +15,6 @@
 package federationsender
 
 import (
-	"net/http"
-
 	"github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/federationsender/consumers"
 	"github.com/matrix-org/dendrite/federationsender/internal"
@@ -69,12 +67,10 @@ func SetupFederationSenderComponent(
 
 	queryAPI := internal.NewFederationSenderInternalAPI(
 		federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing,
-		statistics,
+		statistics, queues,
 	)
 
-	if base.EnableHTTPAPIs {
+	queryAPI.SetupHTTP(base.InternalAPIMux)
-		queryAPI.SetupHTTP(http.DefaultServeMux)
-	}
 
 	return queryAPI
 }

federationsender/internal/api.go

@@ -4,8 +4,10 @@ import (
 	"encoding/json"
 	"net/http"
 
+	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/federationsender/producers"
+	"github.com/matrix-org/dendrite/federationsender/queue"
 	"github.com/matrix-org/dendrite/federationsender/storage"
 	"github.com/matrix-org/dendrite/federationsender/types"
 	"github.com/matrix-org/dendrite/internal"
@@ -23,6 +25,7 @@ type FederationSenderInternalAPI struct {
 	producer   *producers.RoomserverProducer
 	federation *gomatrixserverlib.FederationClient
 	keyRing    *gomatrixserverlib.KeyRing
+	queues     *queue.OutgoingQueues
 }
 
 func NewFederationSenderInternalAPI(
@@ -31,6 +34,7 @@ func NewFederationSenderInternalAPI(
 	federation *gomatrixserverlib.FederationClient,
 	keyRing *gomatrixserverlib.KeyRing,
 	statistics *types.Statistics,
+	queues *queue.OutgoingQueues,
 ) *FederationSenderInternalAPI {
 	return &FederationSenderInternalAPI{
 		db:         db,
@@ -39,12 +43,13 @@ func NewFederationSenderInternalAPI(
 		federation: federation,
 		keyRing:    keyRing,
 		statistics: statistics,
+		queues:     queues,
 	}
 }
 
 // SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux.
-func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) {
+func (f *FederationSenderInternalAPI) SetupHTTP(internalAPIMux *mux.Router) {
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.FederationSenderQueryJoinedHostsInRoomPath,
 		internal.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse {
 			var request api.QueryJoinedHostsInRoomRequest
@@ -58,7 +63,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.FederationSenderQueryJoinedHostServerNamesInRoomPath,
 		internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
 			var request api.QueryJoinedHostServerNamesInRoomRequest
@@ -72,7 +77,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(api.FederationSenderPerformJoinRequestPath,
+	internalAPIMux.Handle(api.FederationSenderPerformJoinRequestPath,
 		internal.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse {
 			var request api.PerformJoinRequest
 			var response api.PerformJoinResponse
@@ -85,7 +90,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(api.FederationSenderPerformLeaveRequestPath,
+	internalAPIMux.Handle(api.FederationSenderPerformLeaveRequestPath,
 		internal.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse {
 			var request api.PerformLeaveRequest
 			var response api.PerformLeaveResponse
@@ -98,4 +103,30 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(api.FederationSenderPerformDirectoryLookupRequestPath,
+		internal.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse {
+			var request api.PerformDirectoryLookupRequest
+			var response api.PerformDirectoryLookupResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := f.PerformDirectoryLookup(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(api.FederationSenderPerformServersAlivePath,
+		internal.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse {
+			var request api.PerformServersAliveRequest
+			var response api.PerformServersAliveResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := f.PerformServersAlive(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }

federationsender/internal/perform.go

@@ -46,8 +46,19 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 		supportedVersions = append(supportedVersions, version)
 	}
 
-	// Deduplicate the server names we were provided.
+	// Deduplicate the server names we were provided but keep the ordering
-	util.SortAndUnique(request.ServerNames)
+	// as this encodes useful information about which servers are most likely
+	// to respond.
+	seenSet := make(map[gomatrixserverlib.ServerName]bool)
+	var uniqueList []gomatrixserverlib.ServerName
+	for _, srv := range request.ServerNames {
+		if seenSet[srv] {
+			continue
+		}
+		seenSet[srv] = true
+		uniqueList = append(uniqueList, srv)
+	}
+	request.ServerNames = uniqueList
 
 	// Try each server that we were provided until we land on one that
 	// successfully completes the make-join send-join dance.
@@ -264,3 +275,16 @@ func (r *FederationSenderInternalAPI) PerformLeave(
 		request.RoomID, len(request.ServerNames),
 	)
 }
+
+// PerformServersAlive implements api.FederationSenderInternalAPI
+func (r *FederationSenderInternalAPI) PerformServersAlive(
+	ctx context.Context,
+	request *api.PerformServersAliveRequest,
+	response *api.PerformServersAliveResponse,
+) (err error) {
+	for _, srv := range request.Servers {
+		r.queues.RetryServer(srv)
+	}
+
+	return nil
+}

federationsender/queue/destinationqueue.go

@@ -39,6 +39,7 @@ type destinationQueue struct {
 	origin             gomatrixserverlib.ServerName            // origin of requests
 	destination        gomatrixserverlib.ServerName            // destination of requests
 	running            atomic.Bool                             // is the queue worker running?
+	backingOff         atomic.Bool                             // true if we're backing off
 	statistics         *types.ServerStatistics                 // statistics about this remote server
 	incomingPDUs       chan *gomatrixserverlib.HeaderedEvent   // PDUs to send
 	incomingEDUs       chan *gomatrixserverlib.EDU             // EDUs to send
@@ -47,6 +48,28 @@ type destinationQueue struct {
 	pendingPDUs        []*gomatrixserverlib.HeaderedEvent      // owned by backgroundSend
 	pendingEDUs        []*gomatrixserverlib.EDU                // owned by backgroundSend
 	pendingInvites     []*gomatrixserverlib.InviteV2Request    // owned by backgroundSend
+	retryServerCh      chan bool                               // interrupts backoff
+}
+
+// retry will clear the blacklist state and attempt to send built up events to the server,
+// resetting and interrupting any backoff timers.
+func (oq *destinationQueue) retry() {
+	// TODO: We don't send all events in the case where the server has been blacklisted as we
+	// drop events instead then. This means we will send the oldest N events (chan size, currently 128)
+	// and then skip ahead a lot which feels non-ideal but equally we can't persist thousands of events
+	// in-memory to maybe-send it one day. Ideally we would just shove these pending events in a database
+	// so we can send a lot of events.
+	oq.statistics.Success()
+	// if we were backing off, swap to not backing off and interrupt the select.
+	// We need to use an atomic bool here to prevent multiple calls to retry() blocking on the channel
+	// as it is unbuffered.
+	if oq.backingOff.CAS(true, false) {
+		oq.retryServerCh <- true
+	}
+	if !oq.running.Load() {
+		log.Infof("Restarting queue for %s", oq.destination)
+		go oq.backgroundSend()
+	}
 }
 
 // Send event adds the event to the pending queue for the destination.
@@ -110,12 +133,26 @@ func (oq *destinationQueue) backgroundSend() {
 			// of the queue and they will all be added to transactions
 			// in order.
 			oq.pendingPDUs = append(oq.pendingPDUs, pdu)
+			// If there are any more things waiting in the channel queue
+			// then read them. This is safe because we guarantee only
+			// having one goroutine per destination queue, so the channel
+			// isn't being consumed anywhere else.
+			for len(oq.incomingPDUs) > 0 {
+				oq.pendingPDUs = append(oq.pendingPDUs, <-oq.incomingPDUs)
+			}
 		case edu := <-oq.incomingEDUs:
 			// Likewise for EDUs, although we should probably not try
 			// too hard with some EDUs (like typing notifications) after
 			// a certain amount of time has passed.
 			// TODO: think about EDU expiry some more
 			oq.pendingEDUs = append(oq.pendingEDUs, edu)
+			// If there are any more things waiting in the channel queue
+			// then read them. This is safe because we guarantee only
+			// having one goroutine per destination queue, so the channel
+			// isn't being consumed anywhere else.
+			for len(oq.incomingEDUs) > 0 {
+				oq.pendingEDUs = append(oq.pendingEDUs, <-oq.incomingEDUs)
+			}
 		case invite := <-oq.incomingInvites:
 			// There's no strict ordering requirement for invites like
 			// there is for transactions, so we put the invite onto the
@@ -126,6 +163,13 @@ func (oq *destinationQueue) backgroundSend() {
 				[]*gomatrixserverlib.InviteV2Request{invite},
 				oq.pendingInvites...,
 			)
+			// If there are any more things waiting in the channel queue
+			// then read them. This is safe because we guarantee only
+			// having one goroutine per destination queue, so the channel
+			// isn't being consumed anywhere else.
+			for len(oq.incomingInvites) > 0 {
+				oq.pendingInvites = append(oq.pendingInvites, <-oq.incomingInvites)
+			}
 		case <-time.After(time.Second * 30):
 			// The worker is idle so stop the goroutine. It'll
 			// get restarted automatically the next time we
@@ -134,9 +178,15 @@ func (oq *destinationQueue) backgroundSend() {
 		}
 
 		// If we are backing off this server then wait for the
-		// backoff duration to complete first.
+		// backoff duration to complete first, or until explicitly
+		// told to retry.
 		if backoff, duration := oq.statistics.BackoffDuration(); backoff {
-			<-time.After(duration)
+			oq.backingOff.Store(true)
+			select {
+			case <-time.After(duration):
+			case <-oq.retryServerCh:
+			}
+			oq.backingOff.Store(false)
 		}
 
 		// How many things do we have waiting?

federationsender/queue/queue.go

@@ -52,6 +52,12 @@ func NewOutgoingQueues(
 	}
 }
 
+func (oqs *OutgoingQueues) getQueueIfExists(destination gomatrixserverlib.ServerName) *destinationQueue {
+	oqs.queuesMutex.Lock()
+	defer oqs.queuesMutex.Unlock()
+	return oqs.queues[destination]
+}
+
 func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue {
 	oqs.queuesMutex.Lock()
 	defer oqs.queuesMutex.Unlock()
@@ -66,6 +72,7 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d
 			incomingPDUs:    make(chan *gomatrixserverlib.HeaderedEvent, 128),
 			incomingEDUs:    make(chan *gomatrixserverlib.EDU, 128),
 			incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128),
+			retryServerCh:   make(chan bool),
 		}
 		oqs.queues[destination] = oq
 	}
@@ -160,6 +167,15 @@ func (oqs *OutgoingQueues) SendEDU(
 	return nil
 }
 
+// RetryServer attempts to resend events to the given server if we had given up.
+func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) {
+	q := oqs.getQueueIfExists(srv)
+	if q == nil {
+		return
+	}
+	q.retry()
+}
+
 // filterAndDedupeDests removes our own server from the list of destinations
 // and deduplicates any servers in the list that may appear more than once.
 func filterAndDedupeDests(origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName) (

federationsender/storage/storage_wasm.go

@@ -33,7 +33,7 @@ func NewDatabase(
 	}
 	switch uri.Scheme {
 	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
+		return sqlite3.NewDatabase(uri.Path)
 	case "postgres":
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	default:

go.mod

@@ -16,9 +16,9 @@ require (
 	github.com/libp2p/go-libp2p-record v0.1.2
 	github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
-	github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd
+	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200602125825-24ff01093eca
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible

go.sum

@@ -352,12 +352,12 @@ github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+
 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4=
-github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd h1:C1FV4dRKF1uuGK8UH01+IoW6zZpfsTV1MvQimZvt418=
+github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf/iHhWlLWd+kCgG+Fsg4Dc+xBl7hptfK7lD0zY=
-github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
+github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b h1:nAmSc1KvQOumoRTz/LD68KyrB6Q5/6q7CmQ5Bswc2nM=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200602125825-24ff01093eca h1:s/dJePRDKjD1fGeoTnEYFqPmp1v7fC6GTd6iFwCKxw8=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200602125825-24ff01093eca/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=

internal/basecomponent/base.go

@@ -22,11 +22,8 @@ import (
 	"net/url"
 	"time"
 
-	"golang.org/x/crypto/ed25519"
-
 	"github.com/matrix-org/dendrite/internal/caching"
-	"github.com/matrix-org/dendrite/internal/keydb"
+	"github.com/matrix-org/dendrite/internal/httpapis"
-	"github.com/matrix-org/dendrite/internal/keydb/cache"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/naffka"
@@ -43,6 +40,7 @@ import (
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api"
 	"github.com/sirupsen/logrus"
 
 	_ "net/http/pprof"
@@ -57,8 +55,9 @@ type BaseDendrite struct {
 	componentName string
 	tracerCloser  io.Closer
 
-	// APIMux should be used to register new public matrix api endpoints
+	// PublicAPIMux should be used to register new public matrix api endpoints
-	APIMux         *mux.Router
+	PublicAPIMux   *mux.Router
+	InternalAPIMux *mux.Router
 	EnableHTTPAPIs bool
 	httpClient     *http.Client
 	Cfg            *config.Dendrite
@@ -103,14 +102,17 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs
 			Host: fmt.Sprintf("%s:%d", cfg.Proxy.Host, cfg.Proxy.Port),
 		})}
 	}
-
+  
+	httpmux := mux.NewRouter()
+  
 	return &BaseDendrite{
 		componentName:  componentName,
 		EnableHTTPAPIs: enableHTTPAPIs,
 		tracerCloser:   closer,
 		Cfg:            cfg,
 		ImmutableCache: cache,
-		APIMux:         mux.NewRouter().UseEncodedPath(),
+		PublicAPIMux:   httpmux.PathPrefix(httpapis.PublicPathPrefix).Subrouter().UseEncodedPath(),
+		InternalAPIMux: httpmux.PathPrefix(httpapis.InternalPathPrefix).Subrouter().UseEncodedPath(),
 		httpClient:     &client,
 		KafkaConsumer:  kafkaConsumer,
 		KafkaProducer:  kafkaProducer,
@@ -162,6 +164,20 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede
 	return f
 }
 
+// CreateHTTPServerKeyAPIs returns ServerKeyInternalAPI for hitting the server key
+// API over HTTP
+func (b *BaseDendrite) CreateHTTPServerKeyAPIs() serverKeyAPI.ServerKeyInternalAPI {
+	f, err := serverKeyAPI.NewServerKeyInternalAPIHTTP(
+		b.Cfg.ServerKeyAPIURL(),
+		b.httpClient,
+		b.ImmutableCache,
+	)
+	if err != nil {
+		logrus.WithError(err).Panic("NewServerKeyInternalAPIHTTP failed", b.httpClient)
+	}
+	return f
+}
+
 // CreateDeviceDB creates a new instance of the device database. Should only be
 // called once per component.
 func (b *BaseDendrite) CreateDeviceDB() devices.Database {
@@ -184,27 +200,6 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
 	return db
 }
 
-// CreateKeyDB creates a new instance of the key database. Should only be called
-// once per component.
-func (b *BaseDendrite) CreateKeyDB() keydb.Database {
-	db, err := keydb.NewDatabase(
-		string(b.Cfg.Database.ServerKey),
-		b.Cfg.DbProperties(),
-		b.Cfg.Matrix.ServerName,
-		b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
-		b.Cfg.Matrix.KeyID,
-	)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to keys db")
-	}
-
-	cachedDB, err := cache.NewKeyDatabase(db, b.ImmutableCache)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to create key cache wrapper")
-	}
-	return cachedDB
-}
-
 // CreateFederationClient creates a new federation client. Should only be called
 // once per component.
 func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient {
@@ -230,7 +225,13 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) {
 		WriteTimeout: HTTPServerTimeout,
 	}
 
-	internal.SetupHTTPAPI(http.DefaultServeMux, internal.WrapHandlerInCORS(b.APIMux), b.Cfg)
+	internal.SetupHTTPAPI(
+		http.DefaultServeMux,
+		b.PublicAPIMux,
+		b.InternalAPIMux,
+		b.Cfg,
+		b.EnableHTTPAPIs,
+	)
 	logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr)
 
 	err := serv.ListenAndServe()
@@ -264,7 +265,15 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
 
 	uri, err := url.Parse(string(cfg.Database.Naffka))
 	if err != nil || uri.Scheme == "file" {
-		db, err = sqlutil.Open(internal.SQLiteDriverName(), string(cfg.Database.Naffka), nil)
+		var cs string
+		if uri.Opaque != "" { // file:filename.db
+			cs = uri.Opaque
+		} else if uri.Path != "" { // file:///path/to/filename.db
+			cs = uri.Path
+		} else {
+			logrus.Panic("file uri has no filename")
+		}
+		db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil)
 		if err != nil {
 			logrus.WithError(err).Panic("Failed to open naffka database")
 		}

internal/config/config.go

@@ -152,6 +152,8 @@ type Dendrite struct {
 			OutputClientData Topic `yaml:"output_client_data"`
 			// Topic for eduserver/api.OutputTypingEvent events.
 			OutputTypingEvent Topic `yaml:"output_typing_event"`
+			// Topic for eduserver/api.OutputSendToDeviceEvent events.
+			OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"`
 			// Topic for user updates (profile, presence)
 			UserUpdates Topic `yaml:"user_updates"`
 		}
@@ -223,6 +225,7 @@ type Dendrite struct {
 		MediaAPI         Address `yaml:"media_api"`
 		ClientAPI        Address `yaml:"client_api"`
 		FederationAPI    Address `yaml:"federation_api"`
+		ServerKeyAPI     Address `yaml:"server_key_api"`
 		AppServiceAPI    Address `yaml:"appservice_api"`
 		SyncAPI          Address `yaml:"sync_api"`
 		RoomServer       Address `yaml:"room_server"`
@@ -237,6 +240,7 @@ type Dendrite struct {
 		MediaAPI         Address `yaml:"media_api"`
 		ClientAPI        Address `yaml:"client_api"`
 		FederationAPI    Address `yaml:"federation_api"`
+		ServerKeyAPI     Address `yaml:"server_key_api"`
 		AppServiceAPI    Address `yaml:"appservice_api"`
 		SyncAPI          Address `yaml:"sync_api"`
 		RoomServer       Address `yaml:"room_server"`
@@ -626,6 +630,7 @@ func (config *Dendrite) checkListen(configErrs *configErrors) {
 	checkNotEmpty(configErrs, "listen.sync_api", string(config.Listen.SyncAPI))
 	checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer))
 	checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer))
+	checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer))
 }
 
 // checkLogging verifies the parameters logging.* are valid.
@@ -757,6 +762,15 @@ func (config *Dendrite) FederationSenderURL() string {
 	return "http://" + string(config.Listen.FederationSender)
 }
 
+// FederationSenderURL returns an HTTP URL for where the federation sender is listening.
+func (config *Dendrite) ServerKeyAPIURL() string {
+	// Hard code the server key API server to talk HTTP for now.
+	// If we support HTTPS we need to think of a practical way to do certificate validation.
+	// People setting up servers shouldn't need to get a certificate valid for the public
+	// internet for an internal API.
+	return "http://" + string(config.Listen.ServerKeyAPI)
+}
+
 // SetupTracing configures the opentracing using the supplied configuration.
 func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) {
 	if !config.Tracing.Enabled {

internal/http/http.go

@@ -6,7 +6,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
+	"strings"
 
+	"github.com/matrix-org/dendrite/internal/httpapis"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 )
@@ -21,6 +24,14 @@ func PostJSON(
 		return err
 	}
 
+	parsedAPIURL, err := url.Parse(apiURL)
+	if err != nil {
+		return err
+	}
+
+	parsedAPIURL.Path = httpapis.InternalPathPrefix + strings.TrimLeft(parsedAPIURL.Path, "/")
+	apiURL = parsedAPIURL.String()
+
 	req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes))
 	if err != nil {
 		return err
@@ -48,10 +59,10 @@ func PostJSON(
 		var errorBody struct {
 			Message string `json:"message"`
 		}
-		if err = json.NewDecoder(res.Body).Decode(&errorBody); err != nil {
+		if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil {
-			return err
+			return fmt.Errorf("Internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message)
 		}
-		return fmt.Errorf("api: %d: %s", res.StatusCode, errorBody.Message)
+		return fmt.Errorf("Internal API: %d from %s", res.StatusCode, apiURL)
 	}
 	return json.NewDecoder(res.Body).Decode(response)
 }

internal/httpapi.go

@@ -1,17 +1,22 @@
 package internal
 
 import (
+	"context"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/http/httputil"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
+	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	federationsenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httpapis"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -168,6 +173,7 @@ func MakeFedAPI(
 	metricsName string,
 	serverName gomatrixserverlib.ServerName,
 	keyRing gomatrixserverlib.KeyRing,
+	wakeup *FederationWakeups,
 	f func(*http.Request, *gomatrixserverlib.FederationRequest) util.JSONResponse,
 ) http.Handler {
 	h := func(req *http.Request) util.JSONResponse {
@@ -177,18 +183,48 @@ func MakeFedAPI(
 		if fedReq == nil {
 			return errResp
 		}
+		go wakeup.Wakeup(req.Context(), fedReq.Origin())
 		return f(req, fedReq)
 	}
 	return MakeExternalAPI(metricsName, h)
 }
 
+type FederationWakeups struct {
+	FsAPI   federationsenderAPI.FederationSenderInternalAPI
+	origins sync.Map
+}
+
+func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib.ServerName) {
+	key, keyok := f.origins.Load(origin)
+	if keyok {
+		lastTime, ok := key.(time.Time)
+		if ok && time.Since(lastTime) < time.Minute {
+			return
+		}
+	}
+	aliveReq := federationsenderAPI.PerformServersAliveRequest{
+		Servers: []gomatrixserverlib.ServerName{origin},
+	}
+	aliveRes := federationsenderAPI.PerformServersAliveResponse{}
+	if err := f.FsAPI.PerformServersAlive(ctx, &aliveReq, &aliveRes); err != nil {
+		util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
+			"origin": origin,
+		}).Warn("incoming federation request failed to notify server alive")
+	} else {
+		f.origins.Store(origin, time.Now())
+	}
+}
+
 // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics
 // listener.
-func SetupHTTPAPI(servMux *http.ServeMux, apiMux http.Handler, cfg *config.Dendrite) {
+func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) {
 	if cfg.Metrics.Enabled {
 		servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth))
 	}
-	servMux.Handle("/api/", http.StripPrefix("/api", apiMux))
+	if enableHTTPAPIs {
+		servMux.Handle(httpapis.InternalPathPrefix, internalApiMux)
+	}
+	servMux.Handle(httpapis.PublicPathPrefix, WrapHandlerInCORS(publicApiMux))
 }
 
 // WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics

internal/httpapis/paths.go

@@ -0,0 +1,6 @@
+package httpapis
+
+const (
+	PublicPathPrefix   = "/_matrix/"
+	InternalPathPrefix = "/api/"
+)

internal/keydb/keyring.go

@@ -1,74 +0,0 @@
-// Copyright 2017 New Vector Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package keydb
-
-import (
-	"encoding/base64"
-
-	"github.com/matrix-org/dendrite/internal/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
-	"golang.org/x/crypto/ed25519"
-)
-
-// CreateKeyRing creates and configures a KeyRing object.
-//
-// It creates the necessary key fetchers and collects them into a KeyRing
-// backed by the given KeyDatabase.
-func CreateKeyRing(client gomatrixserverlib.Client,
-	keyDB gomatrixserverlib.KeyDatabase,
-	cfg config.KeyPerspectives) gomatrixserverlib.KeyRing {
-
-	fetchers := gomatrixserverlib.KeyRing{
-		KeyFetchers: []gomatrixserverlib.KeyFetcher{
-			&gomatrixserverlib.DirectKeyFetcher{
-				Client: client,
-			},
-		},
-		KeyDatabase: keyDB,
-	}
-
-	logrus.Info("Enabled direct key fetcher")
-
-	var b64e = base64.StdEncoding.WithPadding(base64.NoPadding)
-	for _, ps := range cfg {
-		perspective := &gomatrixserverlib.PerspectiveKeyFetcher{
-			PerspectiveServerName: ps.ServerName,
-			PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{},
-			Client:                client,
-		}
-
-		for _, key := range ps.Keys {
-			rawkey, err := b64e.DecodeString(key.PublicKey)
-			if err != nil {
-				logrus.WithError(err).WithFields(logrus.Fields{
-					"server_name": ps.ServerName,
-					"public_key":  key.PublicKey,
-				}).Warn("Couldn't parse perspective key")
-				continue
-			}
-			perspective.PerspectiveServerKeys[key.KeyID] = rawkey
-		}
-
-		fetchers.KeyFetchers = append(fetchers.KeyFetchers, perspective)
-
-		logrus.WithFields(logrus.Fields{
-			"server_name":     ps.ServerName,
-			"num_public_keys": len(ps.Keys),
-		}).Info("Enabled perspective key fetcher")
-	}
-
-	return fetchers
-}

internal/sql.go

@@ -1,4 +1,6 @@
 // Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,11 +18,17 @@ package internal
 
 import (
 	"database/sql"
+	"errors"
 	"fmt"
 	"runtime"
 	"time"
+
+	"go.uber.org/atomic"
 )
 
+// ErrUserExists is returned if a username already exists in the database.
+var ErrUserExists = errors.New("Username already exists")
+
 // A Transaction is something that can be committed or rolledback.
 type Transaction interface {
 	// Commit the transaction
@@ -107,3 +115,60 @@ type DbProperties interface {
 	MaxOpenConns() int
 	ConnMaxLifetime() time.Duration
 }
+
+// TransactionWriter allows queuing database writes so that you don't
+// contend on database locks in, e.g. SQLite. Only one task will run
+// at a time on a given TransactionWriter.
+type TransactionWriter struct {
+	running atomic.Bool
+	todo    chan transactionWriterTask
+}
+
+func NewTransactionWriter() *TransactionWriter {
+	return &TransactionWriter{
+		todo: make(chan transactionWriterTask),
+	}
+}
+
+// transactionWriterTask represents a specific task.
+type transactionWriterTask struct {
+	db   *sql.DB
+	f    func(txn *sql.Tx) error
+	wait chan error
+}
+
+// Do queues a task to be run by a TransactionWriter. The function
+// provided will be ran within a transaction as supplied by the
+// database parameter. This will block until the task is finished.
+func (w *TransactionWriter) Do(db *sql.DB, f func(txn *sql.Tx) error) error {
+	if w.todo == nil {
+		return errors.New("not initialised")
+	}
+	if !w.running.Load() {
+		go w.run()
+	}
+	task := transactionWriterTask{
+		db:   db,
+		f:    f,
+		wait: make(chan error, 1),
+	}
+	w.todo <- task
+	return <-task.wait
+}
+
+// run processes the tasks for a given transaction writer. Only one
+// of these goroutines will run at a time. A transaction will be
+// opened using the database object from the task and then this will
+// be passed as a parameter to the task function.
+func (w *TransactionWriter) run() {
+	if !w.running.CAS(false, true) {
+		return
+	}
+	defer w.running.Store(false)
+	for task := range w.todo {
+		task.wait <- WithTransaction(task.db, func(txn *sql.Tx) error {
+			return task.f(txn)
+		})
+		close(task.wait)
+	}
+}

keyserver/keyserver.go

@@ -28,5 +28,5 @@ func SetupKeyServerComponent(
 	deviceDB devices.Database,
 	accountsDB accounts.Database,
 ) {
-	routing.Setup(base.APIMux, base.Cfg, accountsDB, deviceDB)
+	routing.Setup(base.PublicAPIMux, base.Cfg, accountsDB, deviceDB)
 }

keyserver/routing/routing.go

@@ -27,7 +27,7 @@ import (
 	"github.com/matrix-org/util"
 )
 
-const pathPrefixR0 = "/_matrix/client/r0"
+const pathPrefixR0 = "/client/r0"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
@@ -36,11 +36,11 @@ const pathPrefixR0 = "/_matrix/client/r0"
 // applied:
 // nolint: gocyclo
 func Setup(
-	apiMux *mux.Router, cfg *config.Dendrite,
+	publicAPIMux *mux.Router, cfg *config.Dendrite,
 	accountDB accounts.Database,
 	deviceDB devices.Database,
 ) {
-	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+	r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter()
 
 	authData := auth.Data{
 		AccountDB:   accountDB,

mediaapi/mediaapi.go

@@ -35,6 +35,6 @@ func SetupMediaAPIComponent(
 	}
 
 	routing.Setup(
-		base.APIMux, base.Cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(),
+		base.PublicAPIMux, base.Cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(),
 	)
 }

mediaapi/routing/routing.go

@@ -33,7 +33,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
-const pathPrefixR0 = "/_matrix/media/r0"
+const pathPrefixR0 = "/media/r0"
 
 // Setup registers the media API HTTP handlers
 //
@@ -41,13 +41,13 @@ const pathPrefixR0 = "/_matrix/media/r0"
 // applied:
 // nolint: gocyclo
 func Setup(
-	apiMux *mux.Router,
+	publicAPIMux *mux.Router,
 	cfg *config.Dendrite,
 	db storage.Database,
 	deviceDB devices.Database,
 	client *gomatrixserverlib.Client,
 ) {
-	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+	r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter()
 
 	activeThumbnailGeneration := &types.ActiveThumbnailGeneration{
 		PathToResult: map[string]*types.ThumbnailGenerationResult{},

mediaapi/storage/storage_wasm.go

@@ -35,7 +35,7 @@ func Open(
 	case "postgres":
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	case "file":
-		return sqlite3.Open(dataSourceName)
+		return sqlite3.Open(uri.Path)
 	default:
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	}

publicroomsapi/directory/public_rooms.go

@@ -16,7 +16,9 @@ package directory
 
 import (
 	"context"
+	"math/rand"
 	"net/http"
+	"sort"
 	"strconv"
 	"sync"
 	"time"
@@ -96,6 +98,28 @@ func GetPostPublicRoomsWithExternal(
 	// downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16
 	fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit))
 	response.Chunk = append(response.Chunk, fedRooms...)
+
+	// de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers
+	// are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit)
+	var publicRooms []gomatrixserverlib.PublicRoom
+	haveRoomIDs := make(map[string]bool)
+	rand.Shuffle(len(response.Chunk), func(i, j int) {
+		response.Chunk[i], response.Chunk[j] = response.Chunk[j], response.Chunk[i]
+	})
+	for _, r := range response.Chunk {
+		if haveRoomIDs[r.RoomID] {
+			continue
+		}
+		haveRoomIDs[r.RoomID] = true
+		publicRooms = append(publicRooms, r)
+	}
+	// sort by member count
+	sort.SliceStable(publicRooms, func(i, j int) bool {
+		return publicRooms[i].JoinedMembersCount > publicRooms[j].JoinedMembersCount
+	})
+
+	response.Chunk = publicRooms
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: response,

publicroomsapi/publicroomsapi.go

@@ -43,5 +43,5 @@ func SetupPublicRoomsAPIComponent(
 		logrus.WithError(err).Panic("failed to start public rooms server consumer")
 	}
 
-	routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider)
+	routing.Setup(base.PublicAPIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider)
 }

publicroomsapi/routing/routing.go

@@ -31,7 +31,7 @@ import (
 	"github.com/matrix-org/util"
 )
 
-const pathPrefixR0 = "/_matrix/client/r0"
+const pathPrefixR0 = "/client/r0"
 
 // Setup configures the given mux with publicroomsapi server listeners
 //
@@ -39,10 +39,10 @@ const pathPrefixR0 = "/_matrix/client/r0"
 // applied:
 // nolint: gocyclo
 func Setup(
-	apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI,
+	publicAPIMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI,
 	fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider,
 ) {
-	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+	r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter()
 
 	authData := auth.Data{
 		AccountDB:   nil,
@@ -79,7 +79,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	// Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is.
-	apiMux.Handle("/_matrix/federation/v1/publicRooms",
+	publicAPIMux.Handle("/federation/v1/publicRooms",
 		internal.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse {
 			return directory.GetPostPublicRooms(req, publicRoomsDB)
 		}),

publicroomsapi/storage/postgres/storage.go

@@ -30,20 +30,22 @@ import (
 type PublicRoomsServerDatabase struct {
 	db *sql.DB
 	internal.PartitionOffsetStatements
-	statements publicRoomsStatements
+	statements      publicRoomsStatements
+	localServerName gomatrixserverlib.ServerName
 }
 
 type attributeValue interface{}
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties) (*PublicRoomsServerDatabase, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
 	var db *sql.DB
 	var err error
 	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	storage := PublicRoomsServerDatabase{
-		db: db,
+		db:              db,
+		localServerName: localServerName,
 	}
 	if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil {
 		return nil, err
@@ -243,6 +245,9 @@ func (d *PublicRoomsServerDatabase) updateBooleanAttribute(
 func (d *PublicRoomsServerDatabase) updateRoomAliases(
 	ctx context.Context, aliasesEvent gomatrixserverlib.Event,
 ) error {
+	if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) {
+		return nil // only store our own aliases
+	}
 	var content internal.AliasesContent
 	if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil {
 		return err

publicroomsapi/storage/sqlite3/storage.go

@@ -32,20 +32,22 @@ import (
 type PublicRoomsServerDatabase struct {
 	db *sql.DB
 	internal.PartitionOffsetStatements
-	statements publicRoomsStatements
+	statements      publicRoomsStatements
+	localServerName gomatrixserverlib.ServerName
 }
 
 type attributeValue interface{}
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
 	var db *sql.DB
 	var err error
 	if db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	storage := PublicRoomsServerDatabase{
-		db: db,
+		db:              db,
+		localServerName: localServerName,
 	}
 	if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil {
 		return nil, err
@@ -245,6 +247,9 @@ func (d *PublicRoomsServerDatabase) updateBooleanAttribute(
 func (d *PublicRoomsServerDatabase) updateRoomAliases(
 	ctx context.Context, aliasesEvent gomatrixserverlib.Event,
 ) error {
+	if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) {
+		return nil // only store our own aliases
+	}
 	var content internal.AliasesContent
 	if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil {
 		return err

publicroomsapi/storage/storage.go

@@ -22,23 +22,24 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const schemePostgres = "postgres"
 const schemeFile = "file"
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties)
+		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName)
 	}
 	switch uri.Scheme {
 	case schemePostgres:
-		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties)
+		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName)
 	case schemeFile:
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
+		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
 	default:
-		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties)
+		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName)
 	}
 }

publicroomsapi/storage/storage_wasm.go

@@ -19,10 +19,11 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, err
@@ -31,7 +32,7 @@ func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) {
 	case "postgres":
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	case "file":
-		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
+		return sqlite3.NewPublicRoomsServerDatabase(uri.Path, localServerName)
 	default:
 		return nil, fmt.Errorf("Cannot use postgres implementation")
 	}

roomserver/api/alias.go

@@ -85,19 +85,19 @@ type RemoveRoomAliasRequest struct {
 type RemoveRoomAliasResponse struct{}
 
 // RoomserverSetRoomAliasPath is the HTTP path for the SetRoomAlias API.
-const RoomserverSetRoomAliasPath = "/api/roomserver/setRoomAlias"
+const RoomserverSetRoomAliasPath = "/roomserver/setRoomAlias"
 
 // RoomserverGetRoomIDForAliasPath is the HTTP path for the GetRoomIDForAlias API.
-const RoomserverGetRoomIDForAliasPath = "/api/roomserver/GetRoomIDForAlias"
+const RoomserverGetRoomIDForAliasPath = "/roomserver/GetRoomIDForAlias"
 
 // RoomserverGetAliasesForRoomIDPath is the HTTP path for the GetAliasesForRoomID API.
-const RoomserverGetAliasesForRoomIDPath = "/api/roomserver/GetAliasesForRoomID"
+const RoomserverGetAliasesForRoomIDPath = "/roomserver/GetAliasesForRoomID"
 
 // RoomserverGetCreatorIDForAliasPath is the HTTP path for the GetCreatorIDForAlias API.
-const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias"
+const RoomserverGetCreatorIDForAliasPath = "/roomserver/GetCreatorIDForAlias"
 
 // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API.
-const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias"
+const RoomserverRemoveRoomAliasPath = "/roomserver/removeRoomAlias"
 
 // SetRoomAlias implements RoomserverAliasAPI
 func (h *httpRoomserverInternalAPI) SetRoomAlias(

roomserver/api/input.go

@@ -103,7 +103,7 @@ type InputRoomEventsResponse struct {
 }
 
 // RoomserverInputRoomEventsPath is the HTTP path for the InputRoomEvents API.
-const RoomserverInputRoomEventsPath = "/api/roomserver/inputRoomEvents"
+const RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents"
 
 // InputRoomEvents implements RoomserverInputAPI
 func (h *httpRoomserverInternalAPI) InputRoomEvents(

roomserver/api/perform.go

@@ -10,10 +10,10 @@ import (
 
 const (
 	// RoomserverPerformJoinPath is the HTTP path for the PerformJoin API.
-	RoomserverPerformJoinPath = "/api/roomserver/performJoin"
+	RoomserverPerformJoinPath = "/roomserver/performJoin"
 
 	// RoomserverPerformLeavePath is the HTTP path for the PerformLeave API.
-	RoomserverPerformLeavePath = "/api/roomserver/performLeave"
+	RoomserverPerformLeavePath = "/roomserver/performLeave"
 )
 
 type PerformJoinRequest struct {
@@ -24,6 +24,7 @@ type PerformJoinRequest struct {
 }
 
 type PerformJoinResponse struct {
+	RoomID string `json:"room_id"`
 }
 
 func (h *httpRoomserverInternalAPI) PerformJoin(

roomserver/api/query.go

@@ -273,40 +273,40 @@ type QueryRoomVersionForRoomResponse struct {
 }
 
 // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API.
-const RoomserverQueryLatestEventsAndStatePath = "/api/roomserver/queryLatestEventsAndState"
+const RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState"
 
 // RoomserverQueryStateAfterEventsPath is the HTTP path for the QueryStateAfterEvents API.
-const RoomserverQueryStateAfterEventsPath = "/api/roomserver/queryStateAfterEvents"
+const RoomserverQueryStateAfterEventsPath = "/roomserver/queryStateAfterEvents"
 
 // RoomserverQueryEventsByIDPath is the HTTP path for the QueryEventsByID API.
-const RoomserverQueryEventsByIDPath = "/api/roomserver/queryEventsByID"
+const RoomserverQueryEventsByIDPath = "/roomserver/queryEventsByID"
 
 // RoomserverQueryMembershipForUserPath is the HTTP path for the QueryMembershipForUser API.
-const RoomserverQueryMembershipForUserPath = "/api/roomserver/queryMembershipForUser"
+const RoomserverQueryMembershipForUserPath = "/roomserver/queryMembershipForUser"
 
 // RoomserverQueryMembershipsForRoomPath is the HTTP path for the QueryMembershipsForRoom API
-const RoomserverQueryMembershipsForRoomPath = "/api/roomserver/queryMembershipsForRoom"
+const RoomserverQueryMembershipsForRoomPath = "/roomserver/queryMembershipsForRoom"
 
 // RoomserverQueryInvitesForUserPath is the HTTP path for the QueryInvitesForUser API
-const RoomserverQueryInvitesForUserPath = "/api/roomserver/queryInvitesForUser"
+const RoomserverQueryInvitesForUserPath = "/roomserver/queryInvitesForUser"
 
 // RoomserverQueryServerAllowedToSeeEventPath is the HTTP path for the QueryServerAllowedToSeeEvent API
-const RoomserverQueryServerAllowedToSeeEventPath = "/api/roomserver/queryServerAllowedToSeeEvent"
+const RoomserverQueryServerAllowedToSeeEventPath = "/roomserver/queryServerAllowedToSeeEvent"
 
 // RoomserverQueryMissingEventsPath is the HTTP path for the QueryMissingEvents API
-const RoomserverQueryMissingEventsPath = "/api/roomserver/queryMissingEvents"
+const RoomserverQueryMissingEventsPath = "/roomserver/queryMissingEvents"
 
 // RoomserverQueryStateAndAuthChainPath is the HTTP path for the QueryStateAndAuthChain API
-const RoomserverQueryStateAndAuthChainPath = "/api/roomserver/queryStateAndAuthChain"
+const RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain"
 
 // RoomserverQueryBackfillPath is the HTTP path for the QueryBackfillPath API
-const RoomserverQueryBackfillPath = "/api/roomserver/queryBackfill"
+const RoomserverQueryBackfillPath = "/roomserver/queryBackfill"
 
 // RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API
-const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVersionCapabilities"
+const RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities"
 
 // RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API
-const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom"
+const RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom"
 
 // QueryLatestEventsAndState implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState(

roomserver/internal/api.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"github.com/Shopify/sarama"
+	"github.com/gorilla/mux"
 	fsAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/caching"
@@ -32,8 +33,8 @@ type RoomserverInternalAPI struct {
 
 // SetupHTTP adds the RoomserverInternalAPI handlers to the http.ServeMux.
 // nolint: gocyclo
-func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
+func (r *RoomserverInternalAPI) SetupHTTP(internalAPIMux *mux.Router) {
-	servMux.Handle(api.RoomserverInputRoomEventsPath,
+	internalAPIMux.Handle(api.RoomserverInputRoomEventsPath,
 		internal.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse {
 			var request api.InputRoomEventsRequest
 			var response api.InputRoomEventsResponse
@@ -46,7 +47,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(api.RoomserverPerformJoinPath,
+	internalAPIMux.Handle(api.RoomserverPerformJoinPath,
 		internal.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse {
 			var request api.PerformJoinRequest
 			var response api.PerformJoinResponse
@@ -59,7 +60,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(api.RoomserverPerformLeavePath,
+	internalAPIMux.Handle(api.RoomserverPerformLeavePath,
 		internal.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse {
 			var request api.PerformLeaveRequest
 			var response api.PerformLeaveResponse
@@ -72,7 +73,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryLatestEventsAndStatePath,
 		internal.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse {
 			var request api.QueryLatestEventsAndStateRequest
@@ -86,7 +87,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryStateAfterEventsPath,
 		internal.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse {
 			var request api.QueryStateAfterEventsRequest
@@ -100,7 +101,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryEventsByIDPath,
 		internal.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse {
 			var request api.QueryEventsByIDRequest
@@ -114,7 +115,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryMembershipForUserPath,
 		internal.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse {
 			var request api.QueryMembershipForUserRequest
@@ -128,7 +129,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryMembershipsForRoomPath,
 		internal.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse {
 			var request api.QueryMembershipsForRoomRequest
@@ -142,7 +143,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryInvitesForUserPath,
 		internal.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse {
 			var request api.QueryInvitesForUserRequest
@@ -156,7 +157,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryServerAllowedToSeeEventPath,
 		internal.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse {
 			var request api.QueryServerAllowedToSeeEventRequest
@@ -170,7 +171,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryMissingEventsPath,
 		internal.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse {
 			var request api.QueryMissingEventsRequest
@@ -184,7 +185,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryStateAndAuthChainPath,
 		internal.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse {
 			var request api.QueryStateAndAuthChainRequest
@@ -198,7 +199,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryBackfillPath,
 		internal.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse {
 			var request api.QueryBackfillRequest
@@ -212,7 +213,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryRoomVersionCapabilitiesPath,
 		internal.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse {
 			var request api.QueryRoomVersionCapabilitiesRequest
@@ -226,7 +227,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverQueryRoomVersionForRoomPath,
 		internal.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse {
 			var request api.QueryRoomVersionForRoomRequest
@@ -240,7 +241,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverSetRoomAliasPath,
 		internal.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse {
 			var request api.SetRoomAliasRequest
@@ -254,7 +255,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverGetRoomIDForAliasPath,
 		internal.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse {
 			var request api.GetRoomIDForAliasRequest
@@ -268,7 +269,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverGetCreatorIDForAliasPath,
 		internal.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse {
 			var request api.GetCreatorIDForAliasRequest
@@ -282,7 +283,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverGetAliasesForRoomIDPath,
 		internal.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse {
 			var request api.GetAliasesForRoomIDRequest
@@ -296,7 +297,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	servMux.Handle(
+	internalAPIMux.Handle(
 		api.RoomserverRemoveRoomAliasPath,
 		internal.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse {
 			var request api.RemoveRoomAliasRequest

roomserver/internal/perform_join.go

@@ -90,6 +90,12 @@ func (r *RoomserverInternalAPI) performJoinRoomByID(
 	req *api.PerformJoinRequest,
 	res *api.PerformJoinResponse, // nolint:unparam
 ) error {
+	// By this point, if req.RoomIDOrAlias contained an alias, then
+	// it will have been overwritten with a room ID by performJoinRoomByAlias.
+	// We should now include this in the response so that the CS API can
+	// return the right room ID.
+	res.RoomID = req.RoomIDOrAlias
+
 	// Get the domain part of the room ID.
 	_, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias)
 	if err != nil {
@@ -121,20 +127,30 @@ func (r *RoomserverInternalAPI) performJoinRoomByID(
 		return fmt.Errorf("eb.SetContent: %w", err)
 	}
 
-	// First work out if this is in response to an existing invite.
+	// First work out if this is in response to an existing invite
-	// If it is then we avoid the situation where we might think we
+	// from a federated server. If it is then we avoid the situation
-	// know about a room in the following section but don't know the
+	// where we might think we know about a room in the following
-	// latest state as all of our users have left.
+	// section but don't know the latest state as all of our users
+	// have left.
 	isInvitePending, inviteSender, err := r.isInvitePending(ctx, req.RoomIDOrAlias, req.UserID)
 	if err == nil && isInvitePending {
-		// Add the server of the person who invited us to the server list,
+		// Check if there's an invite pending.
-		// as they should be a fairly good bet.
+		_, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender)
-		if _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender); ierr == nil {
+		if ierr != nil {
-			req.ServerNames = append(req.ServerNames, inviterDomain)
+			return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
 		}
 
-		// Perform a federated room join.
+		// Check that the domain isn't ours. If it's local then we don't
-		return r.performFederatedJoinRoomByID(ctx, req, res)
+		// need to do anything as our own copy of the room state will be
+		// up-to-date.
+		if inviterDomain != r.Cfg.Matrix.ServerName {
+			// Add the server of the person who invited us to the server list,
+			// as they should be a fairly good bet.
+			req.ServerNames = append(req.ServerNames, inviterDomain)
+
+			// Perform a federated room join.
+			return r.performFederatedJoinRoomByID(ctx, req, res)
+		}
 	}
 
 	// Try to construct an actual join event from the template.

roomserver/roomserver.go

@@ -15,8 +15,6 @@
 package roomserver
 
 import (
-	"net/http"
-
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -51,9 +49,7 @@ func SetupRoomServerComponent(
 		KeyRing:              keyRing,
 	}
 
-	if base.EnableHTTPAPIs {
+	internalAPI.SetupHTTP(base.InternalAPIMux)
-		internalAPI.SetupHTTP(http.DefaultServeMux)
-	}
 
 	return &internalAPI
 }

roomserver/storage/interface.go

@@ -63,29 +63,80 @@ type Database interface {
 	SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error)
 	// Look up a room version from the room NID.
 	GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error)
+	// Stores a matrix room event in the database
 	StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error)
+	// Look up the state entries for a list of string event IDs
+	// Returns an error if the there is an error talking to the database
+	// Returns a types.MissingEventError if the event IDs aren't in the database.
 	StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error)
+	// Look up the string event state keys for a list of numeric event state keys
+	// Returns an error if there was a problem talking to the database.
 	EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error)
+	// Look up the numeric IDs for a list of events.
+	// Returns an error if there was a problem talking to the database.
 	EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error)
+	// Set the state at an event. FIXME TODO: "at"
 	SetState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error
+	// Lookup the event IDs for a batch of event numeric IDs.
+	// Returns an error if the retrieval went wrong.
 	EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error)
+	// Look up the latest events in a room in preparation for an update.
+	// The RoomRecentEventsUpdater must have Commit or Rollback called on it if this doesn't return an error.
+	// Returns the latest events in the room and the last eventID sent to the log along with an updater.
+	// If this returns an error then no further action is required.
 	GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error)
+	// Look up event ID by transaction's info.
+	// This is used to determine if the room event is processed/processing already.
+	// Returns an empty string if no such event exists.
 	GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error)
+	// Look up the numeric ID for the room.
+	// Returns 0 if the room doesn't exists.
+	// Returns an error if there was a problem talking to the database.
 	RoomNID(ctx context.Context, roomID string) (types.RoomNID, error)
 	// RoomNIDExcludingStubs is a special variation of RoomNID that will return 0 as if the room
 	// does not exist if the room has no latest events. This can happen when we've received an
 	// invite over federation for a room that we don't know anything else about yet.
 	RoomNIDExcludingStubs(ctx context.Context, roomID string) (types.RoomNID, error)
+	// Look up event references for the latest events in the room and the current state snapshot.
+	// Returns the latest events, the current state and the maximum depth of the latest events plus 1.
+	// Returns an error if there was a problem talking to the database.
 	LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error)
+	// Look up the active invites targeting a user in a room and return the
+	// numeric state key IDs for the user IDs who sent them.
+	// Returns an error if there was a problem talking to the database.
 	GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error)
+	// Save a given room alias with the room ID it refers to.
+	// Returns an error if there was a problem talking to the database.
 	SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error
+	// Look up the room ID a given alias refers to.
+	// Returns an error if there was a problem talking to the database.
 	GetRoomIDForAlias(ctx context.Context, alias string) (string, error)
+	// Look up all aliases referring to a given room ID.
+	// Returns an error if there was a problem talking to the database.
 	GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error)
+	// Get the user ID of the creator of an alias.
+	// Returns an error if there was a problem talking to the database.
 	GetCreatorIDForAlias(ctx context.Context, alias string) (string, error)
+	// Remove a given room alias.
+	// Returns an error if there was a problem talking to the database.
 	RemoveRoomAlias(ctx context.Context, alias string) error
+	// Build a membership updater for the target user in a room.
 	MembershipUpdater(ctx context.Context, roomID, targetUserID string, targetLocal bool, roomVersion gomatrixserverlib.RoomVersion) (types.MembershipUpdater, error)
+	// Lookup the membership of a given user in a given room.
+	// Returns the numeric ID of the latest membership event sent from this user
+	// in this room, along a boolean set to true if the user is still in this room,
+	// false if not.
+	// Returns an error if there was a problem talking to the database.
 	GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error)
+	// Lookup the membership event numeric IDs for all user that are or have
+	// been members of a given room. Only lookup events of "join" membership if
+	// joinOnly is set to true.
+	// Returns an error if there was a problem talking to the database.
 	GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool, localOnly bool) ([]types.EventNID, error)
+	// EventsFromIDs looks up the Events for a list of event IDs. Does not error if event was
+	// not found.
+	// Returns an error if the retrieval went wrong.
 	EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error)
+	// Look up the room version for a given room.
 	GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error)
 }

roomserver/storage/postgres/event_json_table.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 
+	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
 
@@ -58,32 +60,28 @@ type eventJSONStatements struct {
 	bulkSelectEventJSONStmt *sql.Stmt
 }
 
-func (s *eventJSONStatements) prepare(db *sql.DB) (err error) {
+func NewPostgresEventJSONTable(db *sql.DB) (tables.EventJSON, error) {
-	_, err = db.Exec(eventJSONSchema)
+	s := &eventJSONStatements{}
+	_, err := db.Exec(eventJSONSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
-	return statementList{
+	return s, shared.StatementList{
 		{&s.insertEventJSONStmt, insertEventJSONSQL},
 		{&s.bulkSelectEventJSONStmt, bulkSelectEventJSONSQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *eventJSONStatements) insertEventJSON(
+func (s *eventJSONStatements) InsertEventJSON(
-	ctx context.Context, eventNID types.EventNID, eventJSON []byte,
+	ctx context.Context, txn *sql.Tx, eventNID types.EventNID, eventJSON []byte,
 ) error {
 	_, err := s.insertEventJSONStmt.ExecContext(ctx, int64(eventNID), eventJSON)
 	return err
 }
 
-type eventJSONPair struct {
+func (s *eventJSONStatements) BulkSelectEventJSON(
-	EventNID  types.EventNID
-	EventJSON []byte
-}
-
-func (s *eventJSONStatements) bulkSelectEventJSON(
 	ctx context.Context, eventNIDs []types.EventNID,
-) ([]eventJSONPair, error) {
+) ([]tables.EventJSONPair, error) {
 	rows, err := s.bulkSelectEventJSONStmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs))
 	if err != nil {
 		return nil, err
@@ -94,7 +92,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON(
 	// because of the unique constraint on event NIDs.
 	// So we can allocate an array of the correct size now.
 	// We might get fewer results than NIDs so we adjust the length of the slice before returning it.
-	results := make([]eventJSONPair, len(eventNIDs))
+	results := make([]tables.EventJSONPair, len(eventNIDs))
 	i := 0
 	for ; rows.Next(); i++ {
 		result := &results[i]

roomserver/storage/postgres/event_state_keys_table.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
 
@@ -74,20 +76,21 @@ type eventStateKeyStatements struct {
 	bulkSelectEventStateKeyStmt    *sql.Stmt
 }
 
-func (s *eventStateKeyStatements) prepare(db *sql.DB) (err error) {
+func NewPostgresEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) {
-	_, err = db.Exec(eventStateKeysSchema)
+	s := &eventStateKeyStatements{}
+	_, err := db.Exec(eventStateKeysSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
-	return statementList{
+	return s, shared.StatementList{
 		{&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL},
 		{&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL},
 		{&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL},
 		{&s.bulkSelectEventStateKeyStmt, bulkSelectEventStateKeySQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *eventStateKeyStatements) insertEventStateKeyNID(
+func (s *eventStateKeyStatements) InsertEventStateKeyNID(
 	ctx context.Context, txn *sql.Tx, eventStateKey string,
 ) (types.EventStateKeyNID, error) {
 	var eventStateKeyNID int64
@@ -96,7 +99,7 @@ func (s *eventStateKeyStatements) insertEventStateKeyNID(
 	return types.EventStateKeyNID(eventStateKeyNID), err
 }
 
-func (s *eventStateKeyStatements) selectEventStateKeyNID(
+func (s *eventStateKeyStatements) SelectEventStateKeyNID(
 	ctx context.Context, txn *sql.Tx, eventStateKey string,
 ) (types.EventStateKeyNID, error) {
 	var eventStateKeyNID int64
@@ -105,7 +108,7 @@ func (s *eventStateKeyStatements) selectEventStateKeyNID(
 	return types.EventStateKeyNID(eventStateKeyNID), err
 }
 
-func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID(
+func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID(
 	ctx context.Context, eventStateKeys []string,
 ) (map[string]types.EventStateKeyNID, error) {
 	rows, err := s.bulkSelectEventStateKeyNIDStmt.QueryContext(
@@ -128,7 +131,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID(
 	return result, rows.Err()
 }
 
-func (s *eventStateKeyStatements) bulkSelectEventStateKey(
+func (s *eventStateKeyStatements) BulkSelectEventStateKey(
 	ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID,
 ) (map[types.EventStateKeyNID]string, error) {
 	nIDs := make(pq.Int64Array, len(eventStateKeyNIDs))