Merge branch 'master' into event-redaction

.circleci/config.yml

@@ -1,32 +0,0 @@
-version: 2
-jobs:
-  dendrite:
-    docker:
-      - image: matrixdotorg/sytest-dendrite
-    working_directory: /src
-    steps:
-      - checkout
-
-      # Set up dendrite
-      - run: 
-          name: Build Dendrite
-          command: ./build.sh
-      - run: 
-          name: Copy dummy keys to root
-          command: |
-            mv .circleci/matrix_key.pem .
-            mv .circleci/server.key .
-      - run:
-          name: Run sytest with whitelisted tests
-          command: /dendrite_sytest.sh
-
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-workflows:
-  version: 2
-  build:
-    jobs:
-      - dendrite

.circleci/matrix_key.pem

@@ -1,5 +0,0 @@
------BEGIN MATRIX PRIVATE KEY-----
-Key-ID: ed25519:zXtB
-
-jDyHsx0EXbAfvM32yBEKQfIy1FHrmwtB1uMAbm5INBg=
------END MATRIX PRIVATE KEY-----

.circleci/server.key

@@ -1,52 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCanRCqP11MLIQh
-nC26+A1oyBsFfH7auZ3pqE/WFDrCCIoc7ek7cF3fZU7q8OYI+Q9L5V8fobuLb6FB
-iXD5zZ6pBAI0VNjAS8yi8VluXIv6pJKsVY3k2hGiU7xRoEhkzckZBaEiruspQbcX
-ziNoWoueVBB1a4Eproqzy225cTcoprHsJIPXj0HpW/jKcmahmlM/OrqRAxTwxpb/
-moI6MWIeN4n7h55N6dU1ScVvBS7gZpZQ28d8akuvG3m8kE8q1OPFYGvrNeowD4sp
-qDPFijhbygwpzDQlAWriPcqV9KhuGRnYRGTGvuluOttmpgNhNFVxVAlwZJuMVAMU
-Jhek66ntKsxWkF5LsO8ls20hmHyyAsL7+rb2ZjuRtEwE8SwOstU2AIIXoSTtqXjX
-zC8Ew0VB9MCInJoJC/+iKTLoDqXRZeDKGFx1A2F3Y+Er+Z41HcwgqKRsPqZ066yR
-6iKAb5rzJutnEARtbSrNipy9nHE5hIgKJzgOnggcegypcAj3nqbfFFCZA2CFNXoG
-XFkmBHEpz38pPLI5z6HpeZRRySoIyahk9IfSwM3aB1aUi//8CcpAodGvYGNQkQ3W
-HkrZmM4MtC25I5RyMpYJQWKFpx1cOVPf2ASqaJ+IX1JJTv9dSdYHY/rxsxaiXiry
-+uI7UITRvUKgAOrExfSAXco73bgUFwIDAQABAoICAQCP9QX7PhxEPH6aPKxnlWYG
-1aozJYOHa6QYVlpfXV6IIyNVZD7w1OLSiaU9IydL23nelKZI8XGJllpyhuHl9Qlx
-HQZga0+VW/4hCM7X7tt2d50JUG9ZUaFxnr2M0swU73X6Ej/B51OVilZLl+dn1kaB
-GIxqh7ovcRA774EuVLei5fJriGQpZH1eJgAznujoNqSkDq5/Lntk48LcIqR2Qly0
-/ck/pTpEGSAnCZUGlbDbxyjWCIxozx/A3rguVb8ghi+9KtXQntZ6AT71fmMV3mgz
-LqC8miFDA1rdY+MoVDAusrhZoPSkCEWYGL0HijNDYlLbvf874rDhq6diL0V8jOAd
-PGOx5BY6VUWbSQAUtKpMuNSL6tidkOACGPwbuH7OIaG+yGZ0/Oiy3fureiAEg5VU
-piyp6F7p1g0vgQEnj4CHiCQlX48bjC/mm8758DeaH8H5T++A8MOgRhgFVb9f01R+
-NMzszMziuVNDYe01cwdY1TXUx5b0o+opsbPm6sNp/7afL9Hou1epP9zQC0I8ulfP
-fgrKTddMwlNjoBuDMQ8GqoK275YU4wtyhUMfjr3xQ0JwP46cZbhhc4nh6qcRSNTf
-yVuKv/pT/bJcSmg5JOCS8qdK0BQhAvUin9HvgSAV9QmZVpxzT/xhqwuRlLDKW+VR
-XyPt996f3L4CTXI9h88AQQKCAQEAycBChu3/ZKl8a90anOlv9PwmaaXfLBKH9Rkw
-aeZrMilxTJAb+LEsmtj35rF5KPeBP6ARpX5gmvKJVzCDHT9YgNs+6C3E+l2f1/3a
-TcjZKPTukT2gJdCgejhEgTzAwEse322GSptuyidtNpY7NgbAxP4VdDMOmPYbzufb
-5BqxmfiGsfXgdvQkj8/MzHuGhhft4SU6ED/Ax+EPUWVV7kBr2995kGDF5z5CuJkb
-SJjmVxAJZP/kC2Z/iPnP51G0hiCxHp7+gPY4mvvkHvhJGnGH/vutjRjoe28BENlP
-MgB68S1/U3NGSUzWv86pT1OdHd+qynWj/NzF7Gp/T/ju8VZBXwKCAQEAxDAMSOfF
-dizsU7cJbf6vxi6XJHjhwWUWD2vMznKz1D4mkByeY8aSOc8kQZsE5nd4ZgwkYTaZ
-gItjGjM5y5dpKurfKdqQ+dA6PS03h3p+tp1lZp9/dI9X/DfkTO/LUdrfkVVcbQhE
-zqc6C35qO98rhJdsRwhOF28mOc/4bbs0XjC5dEoBGyFt7Fbn2mYoCo4FSHl7WIq6
-TZR9pLAvxjqEZ6Dwrzpp9wtdLIQYPga+KVKcDT/DStThXDTCNt5PyDE9c8eImFww
-u0T87Er5hSEQgodURxDOZh+9ktIfXzMtxiAJ3iDCEPc3NNnLCWfKMhwGsVTCCXj6
-WuHTOe79tOaQSQKCAQEAqBN52PsRl4TzWNEcyLhZQxmFzuIXKJpPlctkX/VMPL/1
-2bj89JR1+pLjA9e6fnyjuqPZz6uXQ77m2DJcKNOLId6Fa9wljAbPkZu0cLTw5YQX
-8/wJHTfPWcLin2BDnG94yt5t0F3pUJTEEYPa1EmP8w1SRjn64Ue3JwpWUJREfWdk
-n4GdfLwscXrGvVvzWGc7ECR5WOwj6OEAZ+kqS5BzyvtERRm6BcoCv9Mdvb9Tthhw
-Gypri2vat/yWTbnt0QgPRtliYYG+6q8K/xoNnPAUQkLd9PxZQevaUXUY2yk3QxGK
-T7VrSsmu5qB+wM2ByU9686xJ7/DlGu4mHjPerEQVtQKCAQBcM3iSitpyP4qRjWQR
-HbDeIudFbMosaaWEedU28REynkLhV5HYsmnmYUNY0dHrvhoHW419YnuhveBFX+25
-kN8MHHXk5aNcxE+akLWYJimHCVGueScdUIC5OEtDHS8guQx48PUPCOPNeyn8XNzw
-ZmG9Xqy0dWK+AK6mXOcUKvbhjWSbEmySo5NVj0JHkdsfmr9A4Fbntcr4yuCBlYve
-TYIMccark3hZci3HzgzWmbSlFv3f/Cd787A19VWRE8nK+9k1oIDBmhIM8M8s/c9m
-kbOApLkm7O8Tb7dYWQgFZbgNdOEuU5bhAk4fuHuDYBPWmPVMQdkvOnvuWlM61ubF
-LdaBAoIBACDpbb5AQIYsWWOnoXuuGh+YY4kmnaBFpsbgEYkZSy92AaLr4Ibf49WN
-oqNDX73YaJlURaGPYMC9J2Huq7TZcewH3SwkVA3N5UmDoijkM4juRfADAfVIMxB5
-+9paWeEfnYC/o377FTJIJ9hHJWIaWSoiJZLYDBmoYdxmk8DSHAJCeWsjYDzPybsH
-7RyMPIa1u7lVdgOPEOBi1OIg7ASLxGKiHQtrYHq99GcaVvU/UxoNRMcSnPfY3G8R
-pGah+EndSCb2F20ouDyvlKfOylAltH2BeNc3B4PeP7ZhlVr7bfyOAfC2Z7FNDm3J
-+yaBExKfroZjsksctNAcAbgpuvhLLG8=
------END PRIVATE KEY-----

CONTRIBUTING.md

@@ -25,7 +25,9 @@ run](scripts/build-test-lint.sh).
 When a Pull Request is submitted, continuous integration jobs are run
 automatically to ensure the code builds and is relatively well-written. Checks
 are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/) and
-[CircleCI](https://circleci.com/gh/matrix-org/dendrite/).
+[CircleCI](https://circleci.com/gh/matrix-org/dendrite/). The Buildkite
+pipeline can be found in Matrix.org's [pipelines
+repository](https://github.com/matrix-org/pipelines).
 
 If a job fails, click the "details" button and you should be taken to the job's
 logs.

clientapi/auth/authtypes/device.go

@@ -21,5 +21,9 @@ type Device struct {
 	// The access_token granted to this device.
 	// This uniquely identifies the device from all other devices and clients.
 	AccessToken string
+	// The unique ID of the session identified by the access token.
+	// Can be used as a secure substitution in places where data needs to be
+	// associated with access tokens.
+	SessionID int64
 	// TODO: display name, last used timestamp, keys, etc
 }

clientapi/auth/storage/accounts/account_data_table.go

@@ -120,28 +120,17 @@ func (s *accountDataStatements) selectAccountData(
 
 func (s *accountDataStatements) selectAccountDataByType(
 	ctx context.Context, localpart, roomID, dataType string,
-) (data []gomatrixserverlib.ClientEvent, err error) {
+) (data *gomatrixserverlib.ClientEvent, err error) {
-	data = []gomatrixserverlib.ClientEvent{}
-
 	stmt := s.selectAccountDataByTypeStmt
-	rows, err := stmt.QueryContext(ctx, localpart, roomID, dataType)
+	var content []byte
-	if err != nil {
+
+	if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil {
 		return
 	}
 
-	for rows.Next() {
+	data = &gomatrixserverlib.ClientEvent{
-		var content []byte
+		Type:    dataType,
-
+		Content: content,
-		if err = rows.Scan(&content); err != nil {
-			return
-		}
-
-		ac := gomatrixserverlib.ClientEvent{
-			Type:    dataType,
-			Content: content,
-		}
-
-		data = append(data, ac)
 	}
 
 	return

clientapi/auth/storage/accounts/storage.go

@@ -263,11 +263,11 @@ func (d *Database) GetAccountData(ctx context.Context, localpart string) (
 
 // GetAccountDataByType returns account data matching a given
 // localpart, room ID and type.
-// If no account data could be found, returns an empty array
+// If no account data could be found, returns nil
 // Returns an error if there was an issue with the retrieval
 func (d *Database) GetAccountDataByType(
 	ctx context.Context, localpart, roomID, dataType string,
-) (data []gomatrixserverlib.ClientEvent, err error) {
+) (data *gomatrixserverlib.ClientEvent, err error) {
 	return d.accountDatas.selectAccountDataByType(
 		ctx, localpart, roomID, dataType,
 	)

clientapi/auth/storage/devices/devices_table.go

@@ -27,11 +27,19 @@ import (
 )
 
 const devicesSchema = `
+-- This sequence is used for automatic allocation of session_id.
+CREATE SEQUENCE IF NOT EXISTS device_session_id_seq START 1;
+
 -- Stores data about devices.
 CREATE TABLE IF NOT EXISTS device_devices (
     -- The access token granted to this device. This has to be the primary key
     -- so we can distinguish which device is making a given request.
     access_token TEXT NOT NULL PRIMARY KEY,
+    -- The auto-allocated unique ID of the session identified by the access token.
+    -- This can be used as a secure substitution of the access token in situations
+    -- where data is associated with access tokens (e.g. transaction storage),
+    -- so we don't have to store users' access tokens everywhere.
+    session_id BIGINT NOT NULL DEFAULT nextval('device_session_id_seq'),
     -- The device identifier. This only needs to uniquely identify a device for a given user, not globally.
     -- access_tokens will be clobbered based on the device ID for a user.
     device_id TEXT NOT NULL,
@@ -51,10 +59,11 @@ CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(loca
 `
 
 const insertDeviceSQL = "" +
-	"INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)"
+	"INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" +
+	" RETURNING session_id"
 
 const selectDeviceByTokenSQL = "" +
-	"SELECT device_id, localpart FROM device_devices WHERE access_token = $1"
+	"SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1"
 
 const selectDeviceByIDSQL = "" +
 	"SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2"
@@ -120,14 +129,16 @@ func (s *devicesStatements) insertDevice(
 	displayName *string,
 ) (*authtypes.Device, error) {
 	createdTimeMS := time.Now().UnixNano() / 1000000
+	var sessionID int64
 	stmt := common.TxStmt(txn, s.insertDeviceStmt)
-	if _, err := stmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName); err != nil {
+	if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName).Scan(&sessionID); err != nil {
 		return nil, err
 	}
 	return &authtypes.Device{
 		ID:          id,
 		UserID:      userutil.MakeUserID(localpart, s.serverName),
 		AccessToken: accessToken,
+		SessionID:   sessionID,
 	}, nil
 }
 
@@ -161,7 +172,7 @@ func (s *devicesStatements) selectDeviceByToken(
 	var dev authtypes.Device
 	var localpart string
 	stmt := s.selectDeviceByTokenStmt
-	err := stmt.QueryRowContext(ctx, accessToken).Scan(&dev.ID, &localpart)
+	err := stmt.QueryRowContext(ctx, accessToken).Scan(&dev.SessionID, &dev.ID, &localpart)
 	if err == nil {
 		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
 		dev.AccessToken = accessToken

clientapi/routing/auth_fallback.go

@@ -0,0 +1,210 @@
+// Copyright 2019 Parminder Singh <parmsingh129@gmail.com>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"html/template"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/util"
+)
+
+// recaptchaTemplate is an HTML webpage template for recaptcha auth
+const recaptchaTemplate = `
+<html>
+<head>
+<title>Authentication</title>
+<meta name='viewport' content='width=device-width, initial-scale=1,
+    user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
+<script src="https://www.google.com/recaptcha/api.js"
+    async defer></script>
+<script src="//code.jquery.com/jquery-1.11.2.min.js"></script>
+<script>
+function captchaDone() {
+    $('#registrationForm').submit();
+}
+</script>
+</head>
+<body>
+<form id="registrationForm" method="post" action="{{.myUrl}}">
+    <div>
+        <p>
+        Hello! We need to prevent computer programs and other automated
+        things from creating accounts on this server.
+        </p>
+        <p>
+        Please verify that you're not a robot.
+        </p>
+		<input type="hidden" name="session" value="{{.session}}" />
+        <div class="g-recaptcha"
+            data-sitekey="{{.siteKey}}"
+            data-callback="captchaDone">
+        </div>
+        <noscript>
+        <input type="submit" value="All Done" />
+        </noscript>
+        </div>
+    </div>
+</form>
+</body>
+</html>
+`
+
+// successTemplate is an HTML template presented to the user after successful
+// recaptcha completion
+const successTemplate = `
+<html>
+<head>
+<title>Success!</title>
+<meta name='viewport' content='width=device-width, initial-scale=1,
+    user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
+<script>
+if (window.onAuthDone) {
+    window.onAuthDone();
+} else if (window.opener && window.opener.postMessage) {
+    window.opener.postMessage("authDone", "*");
+}
+</script>
+</head>
+<body>
+    <div>
+        <p>Thank you!</p>
+        <p>You may now close this window and return to the application.</p>
+    </div>
+</body>
+</html>
+`
+
+// serveTemplate fills template data and serves it using http.ResponseWriter
+func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]string) {
+	t := template.Must(template.New("response").Parse(templateHTML))
+	if err := t.Execute(w, data); err != nil {
+		panic(err)
+	}
+}
+
+// AuthFallback implements GET and POST /auth/{authType}/fallback/web?session={sessionID}
+func AuthFallback(
+	w http.ResponseWriter, req *http.Request, authType string,
+	cfg config.Dendrite,
+) *util.JSONResponse {
+	sessionID := req.URL.Query().Get("session")
+
+	if sessionID == "" {
+		return writeHTTPMessage(w, req,
+			"Session ID not provided",
+			http.StatusBadRequest,
+		)
+	}
+
+	serveRecaptcha := func() {
+		data := map[string]string{
+			"myUrl":   req.URL.String(),
+			"session": sessionID,
+			"siteKey": cfg.Matrix.RecaptchaPublicKey,
+		}
+		serveTemplate(w, recaptchaTemplate, data)
+	}
+
+	serveSuccess := func() {
+		data := map[string]string{}
+		serveTemplate(w, successTemplate, data)
+	}
+
+	if req.Method == http.MethodGet {
+		// Handle Recaptcha
+		if authType == authtypes.LoginTypeRecaptcha {
+			if err := checkRecaptchaEnabled(&cfg, w, req); err != nil {
+				return err
+			}
+
+			serveRecaptcha()
+			return nil
+		}
+		return &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Unknown auth stage type"),
+		}
+	} else if req.Method == http.MethodPost {
+		// Handle Recaptcha
+		if authType == authtypes.LoginTypeRecaptcha {
+			if err := checkRecaptchaEnabled(&cfg, w, req); err != nil {
+				return err
+			}
+
+			clientIP := req.RemoteAddr
+			err := req.ParseForm()
+			if err != nil {
+				res := httputil.LogThenError(req, err)
+				return &res
+			}
+
+			response := req.Form.Get("g-recaptcha-response")
+			if err := validateRecaptcha(&cfg, response, clientIP); err != nil {
+				util.GetLogger(req.Context()).Error(err)
+				return err
+			}
+
+			// Success. Add recaptcha as a completed login flow
+			AddCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
+
+			serveSuccess()
+			return nil
+		}
+
+		return &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Unknown auth stage type"),
+		}
+	}
+	return &util.JSONResponse{
+		Code: http.StatusMethodNotAllowed,
+		JSON: jsonerror.NotFound("Bad method"),
+	}
+}
+
+// checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver.
+func checkRecaptchaEnabled(
+	cfg *config.Dendrite,
+	w http.ResponseWriter,
+	req *http.Request,
+) *util.JSONResponse {
+	if !cfg.Matrix.RecaptchaEnabled {
+		return writeHTTPMessage(w, req,
+			"Recaptcha login is disabled on this Homeserver",
+			http.StatusBadRequest,
+		)
+	}
+	return nil
+}
+
+// writeHTTPMessage writes the given header and message to the HTTP response writer.
+// Returns an error JSONResponse obtained through httputil.LogThenError if the writing failed, otherwise nil.
+func writeHTTPMessage(
+	w http.ResponseWriter, req *http.Request,
+	message string, header int,
+) *util.JSONResponse {
+	w.WriteHeader(header)
+	_, err := w.Write([]byte(message))
+	if err != nil {
+		res := httputil.LogThenError(req, err)
+		return &res
+	}
+	return nil
+}

clientapi/routing/createroom.go

@@ -106,7 +106,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 		}
 	}
 
-	var CreationContent common.CreateContent
+	var CreationContent gomatrixserverlib.CreateContent
 	err = json.Unmarshal(creationContentBytes, &CreationContent)
 	if err != nil {
 		return &util.JSONResponse{
@@ -196,7 +196,7 @@ func createRoom(
 		return httputil.LogThenError(req, err)
 	}
 
-	membershipContent := common.MemberContent{
+	membershipContent := gomatrixserverlib.MemberContent{
 		Membership:  gomatrixserverlib.Join,
 		DisplayName: profile.DisplayName,
 		AvatarURL:   profile.AvatarURL,
@@ -246,7 +246,7 @@ func createRoom(
 		{"m.room.member", userID, membershipContent},
 		{"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)},
 		// TODO: m.room.canonical_alias
-		{"m.room.join_rules", "", common.JoinRulesContent{JoinRule: joinRules}},
+		{"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}},
 		{"m.room.history_visibility", "", common.HistoryVisibilityContent{HistoryVisibility: historyVisibility}},
 	}
 	if r.GuestCanJoin {

clientapi/routing/membership.go

@@ -144,7 +144,7 @@ func buildMembershipEvent(
 		membership = gomatrixserverlib.Leave
 	}
 
-	content := common.MemberContent{
+	content := gomatrixserverlib.MemberContent{
 		Membership:  membership,
 		DisplayName: profile.DisplayName,
 		AvatarURL:   profile.AvatarURL,

clientapi/routing/profile.go

@@ -332,7 +332,7 @@ func buildMembershipEvents(
 			StateKey: &userID,
 		}
 
-		content := common.MemberContent{
+		content := gomatrixserverlib.MemberContent{
 			Membership: gomatrixserverlib.Join,
 		}
 

clientapi/routing/register.go

@@ -29,6 +29,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/matrix-org/dendrite/common/config"
@@ -70,12 +71,17 @@ func init() {
 }
 
 // sessionsDict keeps track of completed auth stages for each session.
+// It shouldn't be passed by value because it contains a mutex.
 type sessionsDict struct {
+	sync.Mutex
 	sessions map[string][]authtypes.LoginType
 }
 
 // GetCompletedStages returns the completed stages for a session.
-func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType {
+func (d *sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType {
+	d.Lock()
+	defer d.Unlock()
+
 	if completedStages, ok := d.sessions[sessionID]; ok {
 		return completedStages
 	}
@@ -83,23 +89,25 @@ func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType
 	return make([]authtypes.LoginType, 0)
 }
 
-// AddCompletedStage records that a session has completed an auth stage.
-func (d *sessionsDict) AddCompletedStage(sessionID string, stage authtypes.LoginType) {
-	// Return if the stage is already present
-	for _, completedStage := range d.GetCompletedStages(sessionID) {
-		if completedStage == stage {
-			return
-		}
-	}
-	d.sessions[sessionID] = append(d.GetCompletedStages(sessionID), stage)
-}
-
 func newSessionsDict() *sessionsDict {
 	return &sessionsDict{
 		sessions: make(map[string][]authtypes.LoginType),
 	}
 }
 
+// AddCompletedSessionStage records that a session has completed an auth stage.
+func AddCompletedSessionStage(sessionID string, stage authtypes.LoginType) {
+	sessions.Lock()
+	defer sessions.Unlock()
+
+	for _, completedStage := range sessions.sessions[sessionID] {
+		if completedStage == stage {
+			return
+		}
+	}
+	sessions.sessions[sessionID] = append(sessions.sessions[sessionID], stage)
+}
+
 var (
 	// TODO: Remove old sessions. Need to do so on a session-specific timeout.
 	// sessions stores the completed flow stages for all sessions. Referenced using their sessionID.
@@ -530,7 +538,7 @@ func handleRegistrationFlow(
 		}
 
 		// Add Recaptcha to the list of completed registration stages
-		sessions.AddCompletedStage(sessionID, authtypes.LoginTypeRecaptcha)
+		AddCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
 
 	case authtypes.LoginTypeSharedSecret:
 		// Check shared secret against config
@@ -543,7 +551,7 @@ func handleRegistrationFlow(
 		}
 
 		// Add SharedSecret to the list of completed registration stages
-		sessions.AddCompletedStage(sessionID, authtypes.LoginTypeSharedSecret)
+		AddCompletedSessionStage(sessionID, authtypes.LoginTypeSharedSecret)
 
 	case "":
 		// Extract the access token from the request, if there's one to extract
@@ -573,7 +581,7 @@ func handleRegistrationFlow(
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
 		// Add Dummy to the list of completed registration stages
-		sessions.AddCompletedStage(sessionID, authtypes.LoginTypeDummy)
+		AddCompletedSessionStage(sessionID, authtypes.LoginTypeDummy)
 
 	default:
 		return util.JSONResponse{

clientapi/routing/room_tagging.go

@@ -59,7 +59,7 @@ func GetTags(
 		return httputil.LogThenError(req, err)
 	}
 
-	if len(data) == 0 {
+	if data == nil {
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: struct{}{},
@@ -68,7 +68,7 @@ func GetTags(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: data[0].Content,
+		JSON: data.Content,
 	}
 }
 
@@ -103,8 +103,8 @@ func PutTag(
 	}
 
 	var tagContent gomatrix.TagContent
-	if len(data) > 0 {
+	if data != nil {
-		if err = json.Unmarshal(data[0].Content, &tagContent); err != nil {
+		if err = json.Unmarshal(data.Content, &tagContent); err != nil {
 			return httputil.LogThenError(req, err)
 		}
 	} else {
@@ -155,7 +155,7 @@ func DeleteTag(
 	}
 
 	// If there are no tags in the database, exit
-	if len(data) == 0 {
+	if data == nil {
 		// Spec only defines 200 responses for this endpoint so we don't return anything else.
 		return util.JSONResponse{
 			Code: http.StatusOK,
@@ -164,7 +164,7 @@ func DeleteTag(
 	}
 
 	var tagContent gomatrix.TagContent
-	err = json.Unmarshal(data[0].Content, &tagContent)
+	err = json.Unmarshal(data.Content, &tagContent)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
@@ -204,7 +204,7 @@ func obtainSavedTags(
 	userID string,
 	roomID string,
 	accountDB *accounts.Database,
-) (string, []gomatrixserverlib.ClientEvent, error) {
+) (string, *gomatrixserverlib.ClientEvent, error) {
 	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		return "", nil, err

clientapi/routing/routing.go

@@ -267,6 +267,13 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
+	r0mux.Handle("/auth/{authType}/fallback/web",
+		common.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse {
+			vars := mux.Vars(req)
+			return AuthFallback(w, req, vars["authType"], cfg)
+		}),
+	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
+
 	r0mux.Handle("/pushrules/",
 		common.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse {
 			// TODO: Implement push rules API

clientapi/routing/sendevent.go

@@ -60,18 +60,18 @@ func SendEvent(
 		return *resErr
 	}
 
-	var txnAndDeviceID *api.TransactionID
+	var txnAndSessionID *api.TransactionID
 	if txnID != nil {
-		txnAndDeviceID = &api.TransactionID{
+		txnAndSessionID = &api.TransactionID{
 			TransactionID: *txnID,
-			DeviceID:      device.ID,
+			SessionID:     device.SessionID,
 		}
 	}
 
 	// pass the new event to the roomserver and receive the correct event ID
 	// event ID in case of duplicate transaction is discarded
 	eventID, err := producer.SendEvents(
-		req.Context(), []gomatrixserverlib.Event{*e}, cfg.Matrix.ServerName, txnAndDeviceID,
+		req.Context(), []gomatrixserverlib.Event{*e}, cfg.Matrix.ServerName, txnAndSessionID,
 	)
 	if err != nil {
 		return httputil.LogThenError(req, err)

clientapi/threepid/invites.go

@@ -56,10 +56,10 @@ type idServerLookupResponse struct {
 
 // idServerLookupResponse represents the response described at https://matrix.org/docs/spec/client_server/r0.2.0.html#invitation-storage
 type idServerStoreInviteResponse struct {
-	PublicKey   string             `json:"public_key"`
+	PublicKey   string                        `json:"public_key"`
-	Token       string             `json:"token"`
+	Token       string                        `json:"token"`
-	DisplayName string             `json:"display_name"`
+	DisplayName string                        `json:"display_name"`
-	PublicKeys  []common.PublicKey `json:"public_keys"`
+	PublicKeys  []gomatrixserverlib.PublicKey `json:"public_keys"`
 }
 
 var (
@@ -342,7 +342,7 @@ func emit3PIDInviteEvent(
 	}
 
 	validityURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/pubkey/isvalid", body.IDServer)
-	content := common.ThirdPartyInviteContent{
+	content := gomatrixserverlib.ThirdPartyInviteContent{
 		DisplayName:    res.DisplayName,
 		KeyValidityURL: validityURL,
 		PublicKey:      res.PublicKey,

common/config/config.go

@@ -498,6 +498,11 @@ func (config *Dendrite) checkMatrix(configErrs *configErrors) {
 	checkNotEmpty(configErrs, "matrix.server_name", string(config.Matrix.ServerName))
 	checkNotEmpty(configErrs, "matrix.private_key", string(config.Matrix.PrivateKeyPath))
 	checkNotZero(configErrs, "matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths)))
+	if config.Matrix.RecaptchaEnabled {
+		checkNotEmpty(configErrs, "matrix.recaptcha_public_key", string(config.Matrix.RecaptchaPublicKey))
+		checkNotEmpty(configErrs, "matrix.recaptcha_private_key", string(config.Matrix.RecaptchaPrivateKey))
+		checkNotEmpty(configErrs, "matrix.recaptcha_siteverify_api", string(config.Matrix.RecaptchaSiteVerifyAPI))
+	}
 }
 
 // checkMedia verifies the parameters media.* are valid.

common/eventcontent.go

@@ -14,55 +14,7 @@
 
 package common
 
-// CreateContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-create
+import "github.com/matrix-org/gomatrixserverlib"
-type CreateContent struct {
-	Creator     string       `json:"creator"`
-	Federate    *bool        `json:"m.federate,omitempty"`
-	RoomVersion string       `json:"room_version,omitempty"`
-	Predecessor PreviousRoom `json:"predecessor,omitempty"`
-}
-
-// PreviousRoom is the "Previous Room" structure defined at https://matrix.org/docs/spec/client_server/r0.5.0#m-room-create
-type PreviousRoom struct {
-	RoomID  string `json:"room_id"`
-	EventID string `json:"event_id"`
-}
-
-// MemberContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member
-type MemberContent struct {
-	Membership       string    `json:"membership"`
-	DisplayName      string    `json:"displayname,omitempty"`
-	AvatarURL        string    `json:"avatar_url,omitempty"`
-	Reason           string    `json:"reason,omitempty"`
-	ThirdPartyInvite *TPInvite `json:"third_party_invite,omitempty"`
-}
-
-// TPInvite is the "Invite" structure defined at http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member
-type TPInvite struct {
-	DisplayName string         `json:"display_name"`
-	Signed      TPInviteSigned `json:"signed"`
-}
-
-// TPInviteSigned is the "signed" structure defined at http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member
-type TPInviteSigned struct {
-	MXID       string                       `json:"mxid"`
-	Signatures map[string]map[string]string `json:"signatures"`
-	Token      string                       `json:"token"`
-}
-
-// ThirdPartyInviteContent is the content event for https://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-third-party-invite
-type ThirdPartyInviteContent struct {
-	DisplayName    string      `json:"display_name"`
-	KeyValidityURL string      `json:"key_validity_url"`
-	PublicKey      string      `json:"public_key"`
-	PublicKeys     []PublicKey `json:"public_keys"`
-}
-
-// PublicKey is the PublicKeys structure in https://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-third-party-invite
-type PublicKey struct {
-	KeyValidityURL string `json:"key_validity_url"`
-	PublicKey      string `json:"public_key"`
-}
 
 // NameContent is the event content for https://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-name
 type NameContent struct {
@@ -79,51 +31,26 @@ type GuestAccessContent struct {
 	GuestAccess string `json:"guest_access"`
 }
 
-// JoinRulesContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-join-rules
-type JoinRulesContent struct {
-	JoinRule string `json:"join_rule"`
-}
-
 // HistoryVisibilityContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-history-visibility
 type HistoryVisibilityContent struct {
 	HistoryVisibility string `json:"history_visibility"`
 }
 
-// PowerLevelContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-power-levels
-type PowerLevelContent struct {
-	EventsDefault int            `json:"events_default"`
-	Invite        int            `json:"invite"`
-	StateDefault  int            `json:"state_default"`
-	Redact        int            `json:"redact"`
-	Ban           int            `json:"ban"`
-	UsersDefault  int            `json:"users_default"`
-	Events        map[string]int `json:"events"`
-	Kick          int            `json:"kick"`
-	Users         map[string]int `json:"users"`
-}
-
 // InitialPowerLevelsContent returns the initial values for m.room.power_levels on room creation
 // if they have not been specified.
 // http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-power-levels
 // https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/handlers/room.py#L294
-func InitialPowerLevelsContent(roomCreator string) PowerLevelContent {
+func InitialPowerLevelsContent(roomCreator string) (c gomatrixserverlib.PowerLevelContent) {
-	return PowerLevelContent{
+	c.Defaults()
-		EventsDefault: 0,
+	c.Events = map[string]int64{
-		Invite:        0,
+		"m.room.name":               50,
-		StateDefault:  50,
+		"m.room.power_levels":       100,
-		Redact:        50,
+		"m.room.history_visibility": 100,
-		Ban:           50,
+		"m.room.canonical_alias":    50,
-		UsersDefault:  0,
+		"m.room.avatar":             50,
-		Events: map[string]int{
-			"m.room.name":               50,
-			"m.room.power_levels":       100,
-			"m.room.history_visibility": 100,
-			"m.room.canonical_alias":    50,
-			"m.room.avatar":             50,
-		},
-		Kick:  50,
-		Users: map[string]int{roomCreator: 100},
 	}
+	c.Users = map[string]int64{roomCreator: 100}
+	return c
 }
 
 // AliasesContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-aliases

common/httpapi.go

@@ -10,6 +10,7 @@ import (
 	"github.com/matrix-org/util"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -43,6 +44,24 @@ func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse
 	return http.HandlerFunc(withSpan)
 }
 
+// MakeHTMLAPI adds Span metrics to the HTML Handler function
+// This is used to serve HTML alongside JSON error messages
+func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request) *util.JSONResponse) http.Handler {
+	withSpan := func(w http.ResponseWriter, req *http.Request) {
+		span := opentracing.StartSpan(metricsName)
+		defer span.Finish()
+		req = req.WithContext(opentracing.ContextWithSpan(req.Context(), span))
+		if err := f(w, req); err != nil {
+			h := util.MakeJSONAPI(util.NewJSONRequestHandler(func(req *http.Request) util.JSONResponse {
+				return *err
+			}))
+			h.ServeHTTP(w, req)
+		}
+	}
+
+	return prometheus.InstrumentHandler(metricsName, http.HandlerFunc(withSpan))
+}
+
 // MakeInternalAPI turns a util.JSONRequestHandler function into an http.Handler.
 // This is used for APIs that are internal to dendrite.
 // If we are passed a tracing context in the request headers then we use that

federationapi/routing/routing.go

@@ -64,6 +64,7 @@ func Setup(
 	// {keyID} argument and always return a response containing all of the keys.
 	v2keysmux.Handle("/server/{keyID}", localKeys).Methods(http.MethodGet)
 	v2keysmux.Handle("/server/", localKeys).Methods(http.MethodGet)
+	v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet)
 
 	v1fedmux.Handle("/send/{txnID}", common.MakeFedAPI(
 		"federation_send", cfg.Matrix.ServerName, keys,

federationapi/routing/threepid.go

@@ -27,7 +27,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
-	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/config"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 
@@ -38,11 +37,11 @@ import (
 )
 
 type invite struct {
-	MXID   string                `json:"mxid"`
+	MXID   string                                         `json:"mxid"`
-	RoomID string                `json:"room_id"`
+	RoomID string                                         `json:"room_id"`
-	Sender string                `json:"sender"`
+	Sender string                                         `json:"sender"`
-	Token  string                `json:"token"`
+	Token  string                                         `json:"token"`
-	Signed common.TPInviteSigned `json:"signed"`
+	Signed gomatrixserverlib.MemberThirdPartyInviteSigned `json:"signed"`
 }
 
 type invites struct {
@@ -199,11 +198,11 @@ func createInviteFrom3PIDInvite(
 		return nil, err
 	}
 
-	content := common.MemberContent{
+	content := gomatrixserverlib.MemberContent{
 		AvatarURL:   profile.AvatarURL,
 		DisplayName: profile.DisplayName,
 		Membership:  gomatrixserverlib.Invite,
-		ThirdPartyInvite: &common.TPInvite{
+		ThirdPartyInvite: &gomatrixserverlib.MemberThirdPartyInvite{
 			Signed: inv.Signed,
 		},
 	}
@@ -330,7 +329,7 @@ func sendToRemoteServer(
 func fillDisplayName(
 	builder *gomatrixserverlib.EventBuilder, authEvents gomatrixserverlib.AuthEvents,
 ) error {
-	var content common.MemberContent
+	var content gomatrixserverlib.MemberContent
 	if err := json.Unmarshal(builder.Content, &content); err != nil {
 		return err
 	}
@@ -343,7 +342,7 @@ func fillDisplayName(
 		return nil
 	}
 
-	var thirdPartyInviteContent common.ThirdPartyInviteContent
+	var thirdPartyInviteContent gomatrixserverlib.ThirdPartyInviteContent
 	if err := json.Unmarshal(thirdPartyInviteEvent.Content(), &thirdPartyInviteContent); err != nil {
 		return err
 	}

federationsender/api/query.go

@@ -0,0 +1,98 @@
+package api
+
+import (
+	"context"
+	"net/http"
+
+	commonHTTP "github.com/matrix-org/dendrite/common/http"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/federationsender/types"
+	"github.com/opentracing/opentracing-go"
+)
+
+// QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom
+type QueryJoinedHostsInRoomRequest struct {
+	RoomID string `json:"room_id"`
+}
+
+// QueryJoinedHostsInRoomResponse is a response to QueryJoinedHostsInRoom
+type QueryJoinedHostsInRoomResponse struct {
+	JoinedHosts []types.JoinedHost `json:"joined_hosts"`
+}
+
+// QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames
+type QueryJoinedHostServerNamesInRoomRequest struct {
+	RoomID string `json:"room_id"`
+}
+
+// QueryJoinedHostServerNamesResponse is a response to QueryJoinedHostServerNames
+type QueryJoinedHostServerNamesInRoomResponse struct {
+	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
+}
+
+// FederationSenderQueryAPI is used to query information from the federation sender.
+type FederationSenderQueryAPI interface {
+	// Query the joined hosts and the membership events accounting for their participation in a room.
+	// Note that if a server has multiple users in the room, it will have multiple entries in the returned slice.
+	// See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version.
+	QueryJoinedHostsInRoom(
+		ctx context.Context,
+		request *QueryJoinedHostsInRoomRequest,
+		response *QueryJoinedHostsInRoomResponse,
+	) error
+	// Query the server names of the joined hosts in a room.
+	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
+	// containing only the server names (without information for membership events).
+	QueryJoinedHostServerNamesInRoom(
+		ctx context.Context,
+		request *QueryJoinedHostServerNamesInRoomRequest,
+		response *QueryJoinedHostServerNamesInRoomResponse,
+	) error
+}
+
+// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API.
+const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom"
+
+// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API.
+const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom"
+
+// NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API.
+// If httpClient is nil then it uses the http.DefaultClient
+func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) FederationSenderQueryAPI {
+	if httpClient == nil {
+		httpClient = http.DefaultClient
+	}
+	return &httpFederationSenderQueryAPI{federationSenderURL, httpClient}
+}
+
+type httpFederationSenderQueryAPI struct {
+	federationSenderURL string
+	httpClient          *http.Client
+}
+
+// QueryJoinedHostsInRoom implements FederationSenderQueryAPI
+func (h *httpFederationSenderQueryAPI) QueryJoinedHostsInRoom(
+	ctx context.Context,
+	request *QueryJoinedHostsInRoomRequest,
+	response *QueryJoinedHostsInRoomResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
+// QueryJoinedHostServerNamesInRoom implements FederationSenderQueryAPI
+func (h *httpFederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom(
+	ctx context.Context,
+	request *QueryJoinedHostServerNamesInRoomRequest,
+	response *QueryJoinedHostServerNamesInRoomResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostServerNamesInRoom")
+	defer span.Finish()
+
+	apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

federationsender/federationsender.go

@@ -15,11 +15,15 @@
 package federationsender
 
 import (
+	"net/http"
+
 	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/federationsender/consumers"
+	"github.com/matrix-org/dendrite/federationsender/query"
 	"github.com/matrix-org/dendrite/federationsender/queue"
 	"github.com/matrix-org/dendrite/federationsender/storage"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
@@ -29,8 +33,8 @@ import (
 func SetupFederationSenderComponent(
 	base *basecomponent.BaseDendrite,
 	federation *gomatrixserverlib.FederationClient,
-	queryAPI api.RoomserverQueryAPI,
+	rsQueryAPI roomserverAPI.RoomserverQueryAPI,
-) {
+) api.FederationSenderQueryAPI {
 	federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender))
 	if err != nil {
 		logrus.WithError(err).Panic("failed to connect to federation sender db")
@@ -40,7 +44,7 @@ func SetupFederationSenderComponent(
 
 	rsConsumer := consumers.NewOutputRoomEventConsumer(
 		base.Cfg, base.KafkaConsumer, queues,
-		federationSenderDB, queryAPI,
+		federationSenderDB, rsQueryAPI,
 	)
 	if err = rsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start room server consumer")
@@ -52,4 +56,11 @@ func SetupFederationSenderComponent(
 	if err := tsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start typing server consumer")
 	}
+
+	queryAPI := query.FederationSenderQueryAPI{
+		DB: federationSenderDB,
+	}
+	queryAPI.SetupHTTP(http.DefaultServeMux)
+
+	return &queryAPI
 }

federationsender/query/query.go

@@ -0,0 +1,91 @@
+package query
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/federationsender/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// FederationSenderQueryDatabase has the APIs needed to implement the query API.
+type FederationSenderQueryDatabase interface {
+	GetJoinedHosts(
+		ctx context.Context, roomID string,
+	) ([]types.JoinedHost, error)
+}
+
+// FederationSenderQueryAPI is an implementation of api.FederationSenderQueryAPI
+type FederationSenderQueryAPI struct {
+	DB FederationSenderQueryDatabase
+}
+
+// QueryJoinedHostsInRoom implements api.FederationSenderQueryAPI
+func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom(
+	ctx context.Context,
+	request *api.QueryJoinedHostsInRoomRequest,
+	response *api.QueryJoinedHostsInRoomResponse,
+) (err error) {
+	response.JoinedHosts, err = f.DB.GetJoinedHosts(ctx, request.RoomID)
+	return
+}
+
+// QueryJoinedHostServerNamesInRoom implements api.FederationSenderQueryAPI
+func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom(
+	ctx context.Context,
+	request *api.QueryJoinedHostServerNamesInRoomRequest,
+	response *api.QueryJoinedHostServerNamesInRoomResponse,
+) (err error) {
+	joinedHosts, err := f.DB.GetJoinedHosts(ctx, request.RoomID)
+	if err != nil {
+		return
+	}
+
+	serverNamesSet := make(map[gomatrixserverlib.ServerName]bool, len(joinedHosts))
+	for _, host := range joinedHosts {
+		serverNamesSet[host.ServerName] = true
+	}
+
+	response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(serverNamesSet))
+	for name := range serverNamesSet {
+		response.ServerNames = append(response.ServerNames, name)
+	}
+
+	return
+}
+
+// SetupHTTP adds the FederationSenderQueryAPI handlers to the http.ServeMux.
+func (f *FederationSenderQueryAPI) SetupHTTP(servMux *http.ServeMux) {
+	servMux.Handle(
+		api.FederationSenderQueryJoinedHostsInRoomPath,
+		common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse {
+			var request api.QueryJoinedHostsInRoomRequest
+			var response api.QueryJoinedHostsInRoomResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.FederationSenderQueryJoinedHostServerNamesInRoomPath,
+		common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
+			var request api.QueryJoinedHostServerNamesInRoomRequest
+			var response api.QueryJoinedHostServerNamesInRoomResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+}

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d
 	github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20190814163046-d6285a18401f
 	github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0
 	github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5
 	github.com/matttproud/golang_protobuf_extensions v1.0.1

go.sum

@@ -58,6 +58,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 h1:B8
 github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 h1:xr69Hk6QM3RIN6JSvx3RpDowBGpHpDDqhqXCeySwYow=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20190814163046-d6285a18401f h1:20CZL7ApB7xgR7sZF9yD/qpsP51Sfx0TTgUJ3vKgnZQ=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20190814163046-d6285a18401f/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E=
 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A=
 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A=
 github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo=

roomserver/api/input.go

@@ -75,9 +75,9 @@ type InputRoomEvent struct {
 }
 
 // TransactionID contains the transaction ID sent by a client when sending an
-// event, along with the ID of that device.
+// event, along with the ID of the client session.
 type TransactionID struct {
-	DeviceID      string `json:"device_id"`
+	SessionID     int64  `json:"session_id"`
 	TransactionID string `json:"id"`
 }
 

roomserver/input/events.go

@@ -32,7 +32,7 @@ type RoomEventDatabase interface {
 	StoreEvent(
 		ctx context.Context,
 		event gomatrixserverlib.Event,
-		txnAndDeviceID *api.TransactionID,
+		txnAndSessionID *api.TransactionID,
 		authEventNIDs []types.EventNID,
 	) (types.RoomNID, types.StateAtEvent, error)
 	// Look up the state entries for a list of string event IDs
@@ -67,7 +67,7 @@ type RoomEventDatabase interface {
 	// Returns an empty string if no such event exists.
 	GetTransactionEventID(
 		ctx context.Context, transactionID string,
-		deviceID string, userID string,
+		sessionID int64, userID string,
 	) (string, error)
 }
 
@@ -100,7 +100,7 @@ func processRoomEvent(
 	if input.TransactionID != nil {
 		tdID := input.TransactionID
 		eventID, err = db.GetTransactionEventID(
-			ctx, tdID.TransactionID, tdID.DeviceID, input.Event.Sender(),
+			ctx, tdID.TransactionID, tdID.SessionID, input.Event.Sender(),
 		)
 		// On error OR event with the transaction already processed/processesing
 		if err != nil || eventID != "" {

roomserver/storage/storage.go

@@ -48,7 +48,7 @@ func Open(dataSourceName string) (*Database, error) {
 // StoreEvent implements input.EventDatabase
 func (d *Database) StoreEvent(
 	ctx context.Context, event gomatrixserverlib.Event,
-	txnAndDeviceID *api.TransactionID, authEventNIDs []types.EventNID,
+	txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID,
 ) (types.RoomNID, types.StateAtEvent, error) {
 	var (
 		roomNID          types.RoomNID
@@ -59,10 +59,10 @@ func (d *Database) StoreEvent(
 		err              error
 	)
 
-	if txnAndDeviceID != nil {
+	if txnAndSessionID != nil {
 		if err = d.statements.insertTransaction(
-			ctx, txnAndDeviceID.TransactionID,
+			ctx, txnAndSessionID.TransactionID,
-			txnAndDeviceID.DeviceID, event.Sender(), event.EventID(),
+			txnAndSessionID.SessionID, event.Sender(), event.EventID(),
 		); err != nil {
 			return 0, types.StateAtEvent{}, err
 		}
@@ -496,9 +496,9 @@ func (d *Database) GetLatestEventsForUpdate(
 // GetTransactionEventID implements input.EventDatabase
 func (d *Database) GetTransactionEventID(
 	ctx context.Context, transactionID string,
-	deviceID string, userID string,
+	sessionID int64, userID string,
 ) (string, error) {
-	eventID, err := d.statements.selectTransactionEventID(ctx, transactionID, deviceID, userID)
+	eventID, err := d.statements.selectTransactionEventID(ctx, transactionID, sessionID, userID)
 	if err == sql.ErrNoRows {
 		return "", nil
 	}

roomserver/storage/transactions_table.go

@@ -23,8 +23,8 @@ const transactionsSchema = `
 CREATE TABLE IF NOT EXISTS roomserver_transactions (
 	-- The transaction ID of the event.
 	transaction_id TEXT NOT NULL,
-	-- The device ID of the originating transaction.
+	-- The session ID of the originating transaction.
-	device_id TEXT NOT NULL,
+	session_id BIGINT NOT NULL,
 	-- User ID of the sender who authored the event
 	user_id TEXT NOT NULL,
 	-- Event ID corresponding to the transaction
@@ -32,16 +32,16 @@ CREATE TABLE IF NOT EXISTS roomserver_transactions (
 	event_id TEXT NOT NULL,
 	-- A transaction ID is unique for a user and device
 	-- This automatically creates an index.
-	PRIMARY KEY (transaction_id, device_id, user_id)
+	PRIMARY KEY (transaction_id, session_id, user_id)
 );
 `
 const insertTransactionSQL = "" +
-	"INSERT INTO roomserver_transactions (transaction_id, device_id, user_id, event_id)" +
+	"INSERT INTO roomserver_transactions (transaction_id, session_id, user_id, event_id)" +
 	" VALUES ($1, $2, $3, $4)"
 
 const selectTransactionEventIDSQL = "" +
 	"SELECT event_id FROM roomserver_transactions" +
-	" WHERE transaction_id = $1 AND device_id = $2 AND user_id = $3"
+	" WHERE transaction_id = $1 AND session_id = $2 AND user_id = $3"
 
 type transactionStatements struct {
 	insertTransactionStmt        *sql.Stmt
@@ -63,12 +63,12 @@ func (s *transactionStatements) prepare(db *sql.DB) (err error) {
 func (s *transactionStatements) insertTransaction(
 	ctx context.Context,
 	transactionID string,
-	deviceID string,
+	sessionID int64,
 	userID string,
 	eventID string,
 ) (err error) {
 	_, err = s.insertTransactionStmt.ExecContext(
-		ctx, transactionID, deviceID, userID, eventID,
+		ctx, transactionID, sessionID, userID, eventID,
 	)
 	return
 }
@@ -76,11 +76,11 @@ func (s *transactionStatements) insertTransaction(
 func (s *transactionStatements) selectTransactionEventID(
 	ctx context.Context,
 	transactionID string,
-	deviceID string,
+	sessionID int64,
 	userID string,
 ) (eventID string, err error) {
 	err = s.selectTransactionEventIDStmt.QueryRowContext(
-		ctx, transactionID, deviceID, userID,
+		ctx, transactionID, sessionID, userID,
 	).Scan(&eventID)
 	return
 }

syncapi/storage/output_room_events_table.go

@@ -54,7 +54,7 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
     -- if there is no delta.
     add_state_ids TEXT[],
     remove_state_ids TEXT[],
-    device_id TEXT,  -- The local device that sent the event, if any
+    session_id BIGINT,  -- The client session that sent the event, if any
     transaction_id TEXT  -- The transaction id used to send the event, if any
 );
 -- for event selection
@@ -63,7 +63,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_output_room_ev
 
 const insertEventSQL = "" +
 	"INSERT INTO syncapi_output_room_events (" +
-	"room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, device_id, transaction_id" +
+	"room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id" +
 	") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id"
 
 const selectEventsSQL = "" +
@@ -71,7 +71,7 @@ const selectEventsSQL = "" +
 	" FROM syncapi_output_room_events WHERE event_id = ANY($1)"
 
 const selectRecentEventsSQL = "" +
-	"SELECT id, event_json, device_id, transaction_id FROM syncapi_output_room_events" +
+	"SELECT id, event_json, session_id, transaction_id FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3" +
 	" ORDER BY id DESC LIMIT $4"
 
@@ -221,9 +221,10 @@ func (s *outputRoomEventsStatements) insertEvent(
 	event *gomatrixserverlib.Event, addState, removeState []string,
 	transactionID *api.TransactionID,
 ) (streamPos int64, err error) {
-	var deviceID, txnID *string
+	var txnID *string
+	var sessionID *int64
 	if transactionID != nil {
-		deviceID = &transactionID.DeviceID
+		sessionID = &transactionID.SessionID
 		txnID = &transactionID.TransactionID
 	}
 
@@ -246,7 +247,7 @@ func (s *outputRoomEventsStatements) insertEvent(
 		containsURL,
 		pq.StringArray(addState),
 		pq.StringArray(removeState),
-		deviceID,
+		sessionID,
 		txnID,
 	).Scan(&streamPos)
 	return
@@ -296,11 +297,11 @@ func rowsToStreamEvents(rows *sql.Rows) ([]streamEvent, error) {
 		var (
 			streamPos     int64
 			eventBytes    []byte
-			deviceID      *string
+			sessionID     *int64
 			txnID         *string
 			transactionID *api.TransactionID
 		)
-		if err := rows.Scan(&streamPos, &eventBytes, &deviceID, &txnID); err != nil {
+		if err := rows.Scan(&streamPos, &eventBytes, &sessionID, &txnID); err != nil {
 			return nil, err
 		}
 		ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false)
@@ -308,9 +309,9 @@ func rowsToStreamEvents(rows *sql.Rows) ([]streamEvent, error) {
 			return nil, err
 		}
 
-		if deviceID != nil && txnID != nil {
+		if sessionID != nil && txnID != nil {
 			transactionID = &api.TransactionID{
-				DeviceID:      *deviceID,
+				SessionID:     *sessionID,
 				TransactionID: *txnID,
 			}
 		}

syncapi/storage/syncserver.go

@@ -1126,7 +1126,7 @@ func streamEventsToEvents(device *authtypes.Device, in []streamEvent) []gomatrix
 	for i := 0; i < len(in); i++ {
 		out[i] = in[i].Event
 		if device != nil && in[i].transactionID != nil {
-			if device.UserID == in[i].Sender() && device.ID == in[i].transactionID.DeviceID {
+			if device.UserID == in[i].Sender() && device.SessionID == in[i].transactionID.SessionID {
 				err := out[i].SetUnsignedField(
 					"transaction_id", in[i].transactionID.TransactionID,
 				)

syncapi/sync/requestpool.go

@@ -196,13 +196,13 @@ func (rp *RequestPool) appendAccountData(
 		events := []gomatrixserverlib.ClientEvent{}
 		// Request the missing data from the database
 		for _, dataType := range dataTypes {
-			evs, err := rp.accountDB.GetAccountDataByType(
+			event, err := rp.accountDB.GetAccountDataByType(
 				req.ctx, localpart, roomID, dataType,
 			)
 			if err != nil {
 				return nil, err
 			}
-			events = append(events, evs...)
+			events = append(events, *event)
 		}
 
 		// Append the data to the response

testfile

@@ -143,7 +143,7 @@ Trying to get push rules with unknown rule_id fails with 404
 Events come down the correct room
 local user can join room with version 5
 User can invite local user to room with version 5
-Inbound federation can receive room-join requests
+Inbound federation can receive v1 room-join requests
 Typing events appear in initial sync
 Typing events appear in incremental sync
 Typing events appear in gapped sync
@@ -170,6 +170,8 @@ Deleted tags appear in an incremental v2 /sync
 Outbound federation can query profile data
 /event/ on joined room works
 /event/ does not allow access to events before the user joined
+Federation key API allows unsigned requests for keys
+Can paginate public room list
 POST /rooms/:room_id/redact/:event_id as original message sender redacts message
 POST /rooms/:room_id/redact/:event_id as random user does not redact message
 POST /redact disallows redaction of event in different room