From 717d16345cf9b93ef1652c34a70e6109f8f9c97e Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 8 Jul 2021 10:07:39 +0100
Subject: [PATCH 01/13] Improve error handling and close files post-tarring

---
 cmd/dendrite-upgrade-tests/main.go | 1 +
 cmd/dendrite-upgrade-tests/tar.go  | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index 5b5cf9d7e..46e33c1cd 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -104,6 +104,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
 	if resp.StatusCode != 200 {
 		return nil, fmt.Errorf("got HTTP %d", resp.StatusCode)
 	}
+	_ = os.RemoveAll(tmpDir)
 	if err = os.Mkdir(tmpDir, os.ModePerm); err != nil {
 		return nil, fmt.Errorf("failed to make temporary directory: %s", err)
 	}
diff --git a/cmd/dendrite-upgrade-tests/tar.go b/cmd/dendrite-upgrade-tests/tar.go
index 9eadbb3de..8c6402a84 100644
--- a/cmd/dendrite-upgrade-tests/tar.go
+++ b/cmd/dendrite-upgrade-tests/tar.go
@@ -17,7 +17,7 @@ func compress(src string, buf io.Writer) error {
 	tw := tar.NewWriter(zr)
 
 	// walk through every file in the folder
-	_ = filepath.Walk(src, func(file string, fi os.FileInfo, e error) error {
+	err := filepath.Walk(src, func(file string, fi os.FileInfo, e error) error {
 		// generate tar header
 		header, err := tar.FileInfoHeader(fi, file)
 		if err != nil {
@@ -40,9 +40,15 @@ func compress(src string, buf io.Writer) error {
 			if _, err := io.Copy(tw, data); err != nil {
 				return err
 			}
+			if err = data.Close(); err != nil {
+				return err
+			}
 		}
 		return nil
 	})
+	if err != nil {
+		return err
+	}
 
 	// produce tar
 	if err := tw.Close(); err != nil {

From 3fb5ee7e1cc9545b492abefa62980f96ee58364e Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 8 Jul 2021 10:17:13 +0100
Subject: [PATCH 02/13] linting

---
 cmd/dendrite-upgrade-tests/tar.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/dendrite-upgrade-tests/tar.go b/cmd/dendrite-upgrade-tests/tar.go
index 8c6402a84..fd45424db 100644
--- a/cmd/dendrite-upgrade-tests/tar.go
+++ b/cmd/dendrite-upgrade-tests/tar.go
@@ -37,7 +37,7 @@ func compress(src string, buf io.Writer) error {
 			if err != nil {
 				return err
 			}
-			if _, err := io.Copy(tw, data); err != nil {
+			if _, err = io.Copy(tw, data); err != nil {
 				return err
 			}
 			if err = data.Close(); err != nil {

From ef331c52af7606062d9d7584e6c004cf363d01eb Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Thu, 8 Jul 2021 12:28:04 +0100
Subject: [PATCH 03/13] dendrite-upgrade-test: tweaks to get it to run under CI
 in docker (#1905)

* dendrite-upgrade-test: tweaks to get it to run under CI in docker

* Linting
---
 cmd/dendrite-upgrade-tests/main.go | 49 ++++++++++++++++++++++--------
 1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index 46e33c1cd..a6cc2d3f9 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -35,9 +35,13 @@ var (
 	flagFrom             = flag.String("from", "HEAD-1", "The version to start from e.g '0.3.1'. If 'HEAD-N' then starts N versions behind HEAD.")
 	flagTo               = flag.String("to", "HEAD", "The version to end on e.g '0.3.3'.")
 	flagBuildConcurrency = flag.Int("build-concurrency", runtime.NumCPU(), "The amount of build concurrency when building images")
+	flagHead             = flag.String("head", "", "Location to a dendrite repository to treat as HEAD instead of Github")
+	flagDockerHost       = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.")
 	alphaNumerics        = regexp.MustCompile("[^a-zA-Z0-9]+")
 )
 
+const HEAD = "HEAD"
+
 // Embed the Dockerfile to use when building dendrite versions.
 // We cannot use the dockerfile associated with the repo with each version sadly due to changes in
 // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
@@ -135,15 +139,36 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
 
 // buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
 func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) {
-	log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir)
-	// pull an archive, this contains a top-level directory which screws with the build context
-	// which we need to fix up post download
-	u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName)
-	tarball, err := downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile))
-	if err != nil {
-		return "", fmt.Errorf("failed to download archive %s: %w", u, err)
+	var tarball *bytes.Buffer
+	var err error
+	// If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI
+	// where we want to use the working directory.
+	if branchOrTagName == HEAD && *flagHead != "" {
+		log.Printf("%s: Using %s as HEAD", branchOrTagName, *flagHead)
+		// add top level Dockerfile
+		err = ioutil.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
+		if err != nil {
+			return "", fmt.Errorf("Custom HEAD: failed to inject /Dockerfile: %w", err)
+		}
+		// now tarball it
+		var buffer bytes.Buffer
+		err = compress(*flagHead, &buffer)
+		if err != nil {
+			return "", fmt.Errorf("failed to tarball custom HEAD %s : %s", *flagHead, err)
+		}
+		tarball = &buffer
+	} else {
+		log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir)
+		// pull an archive, this contains a top-level directory which screws with the build context
+		// which we need to fix up post download
+		u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName)
+		tarball, err = downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile))
+		if err != nil {
+			return "", fmt.Errorf("failed to download archive %s: %w", u, err)
+		}
+		log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len())
 	}
-	log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len())
+
 	log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName)
 	res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
 		Tags: []string{"dendrite-upgrade"},
@@ -235,7 +260,7 @@ func calculateVersions(cli *http.Client, from, to string) []string {
 			semvers = semvers[i:]
 		}
 	}
-	if to != "" && to != "HEAD" {
+	if to != "" && to != HEAD {
 		toVer, err := semver.NewVersion(to)
 		if err != nil {
 			log.Fatalf("invalid --to: %s", err)
@@ -253,8 +278,8 @@ func calculateVersions(cli *http.Client, from, to string) []string {
 	for _, sv := range semvers {
 		versions = append(versions, sv.Original())
 	}
-	if to == "HEAD" {
-		versions = append(versions, "HEAD")
+	if to == HEAD {
+		versions = append(versions, HEAD)
 	}
 	return versions
 }
@@ -328,7 +353,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
 	if !ok {
 		return "", "", fmt.Errorf("port 8008 not exposed - exposed ports: %v", inspect.NetworkSettings.Ports)
 	}
-	baseURL := fmt.Sprintf("http://localhost:%s", csapiPortInfo[0].HostPort)
+	baseURL := fmt.Sprintf("http://%s:%s", *flagDockerHost, csapiPortInfo[0].HostPort)
 	versionsURL := fmt.Sprintf("%s/_matrix/client/versions", baseURL)
 	// hit /versions to check it is up
 	var lastErr error

From 70e4bbda3b0b3eebc8a1c3f8e1bb1b1a85988070 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Thu, 8 Jul 2021 13:13:27 +0100
Subject: [PATCH 04/13] Only log filename and not entire path (#1906)

---
 internal/log.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/log.go b/internal/log.go
index 0f374bd4a..d2b233c5b 100644
--- a/internal/log.go
+++ b/internal/log.go
@@ -73,9 +73,9 @@ func callerPrettyfier(f *runtime.Frame) (string, string) {
 	// Append a newline + tab to it to move the actual log content to its own line
 	funcname += "\n\t"
 
-	// Surround the filepath in brackets and append line number so IDEs can quickly
-	// navigate
-	filename := fmt.Sprintf(" [%s:%d]", f.File, f.Line)
+	// Use a shortened file path which just has the filename to avoid having lots of redundant
+	// directories which contribute significantly to overall log sizes!
+	filename := fmt.Sprintf(" [%s:%d]", path.Base(f.File), f.Line)
 
 	return funcname, filename
 }

From 816e1a402bdc69a681177c1994e07a2a907315b8 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 8 Jul 2021 14:54:03 +0100
Subject: [PATCH 05/13] Fix bug when rejecting invites (#1907)

* Fix rejecting invites maybe

* Remove comment that is no longer correct

* Review comment on performFederatedRejectInvite
---
 roomserver/internal/perform/perform_leave.go | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index 9d7c0816d..4d10dea67 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -64,7 +64,14 @@ func (r *Leaver) performLeaveRoomByID(
 	// that.
 	isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID)
 	if err == nil && isInvitePending {
-		return r.performRejectInvite(ctx, req, res, senderUser, eventID)
+		var host gomatrixserverlib.ServerName
+		_, host, err = gomatrixserverlib.SplitID('@', senderUser)
+		if err != nil {
+			return nil, fmt.Errorf("Sender %q is invalid", senderUser)
+		}
+		if host != r.Cfg.Matrix.ServerName {
+			return r.performFederatedRejectInvite(ctx, req, res, senderUser, eventID)
+		}
 	}
 
 	// There's no invite pending, so first of all we want to find out
@@ -94,9 +101,7 @@ func (r *Leaver) performLeaveRoomByID(
 	if err != nil {
 		return nil, fmt.Errorf("Error getting membership: %w", err)
 	}
-	if membership != gomatrixserverlib.Join {
-		// TODO: should be able to handle "invite" in this case too, if
-		// it's a case of kicking or banning or such
+	if membership != gomatrixserverlib.Join && membership != gomatrixserverlib.Invite {
 		return nil, fmt.Errorf("User %q is not joined to the room (membership is %q)", req.UserID, membership)
 	}
 
@@ -147,7 +152,7 @@ func (r *Leaver) performLeaveRoomByID(
 	return nil, nil
 }
 
-func (r *Leaver) performRejectInvite(
+func (r *Leaver) performFederatedRejectInvite(
 	ctx context.Context,
 	req *api.PerformLeaveRequest,
 	res *api.PerformLeaveResponse, // nolint:unparam

From 3e50bac9441ae43387fee510d5838039bc07e0bb Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Fri, 9 Jul 2021 10:49:49 +0100
Subject: [PATCH 06/13] bugfix: order the state blocks so recreating state
 snapshots works correctly (#1908)

* Logging

* Revert "Logging"

This reverts commit 23ce334182d8a70f8e0381786f9dea77a9b91ed8.

* bugfix: order the state blocks so recreating state snapshots works correctly
---
 roomserver/storage/postgres/state_block_table.go | 2 +-
 roomserver/storage/sqlite3/state_block_table.go  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go
index 4523d18bb..7ae987d6d 100644
--- a/roomserver/storage/postgres/state_block_table.go
+++ b/roomserver/storage/postgres/state_block_table.go
@@ -64,7 +64,7 @@ const insertStateDataSQL = "" +
 
 const bulkSelectStateBlockEntriesSQL = "" +
 	"SELECT state_block_nid, event_nids" +
-	" FROM roomserver_state_block WHERE state_block_nid = ANY($1)"
+	" FROM roomserver_state_block WHERE state_block_nid = ANY($1) ORDER BY state_block_nid ASC"
 
 type stateBlockStatements struct {
 	insertStateDataStmt             *sql.Stmt
diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go
index cfb2a49e5..5cb21e91c 100644
--- a/roomserver/storage/sqlite3/state_block_table.go
+++ b/roomserver/storage/sqlite3/state_block_table.go
@@ -57,7 +57,7 @@ const insertStateDataSQL = `
 
 const bulkSelectStateBlockEntriesSQL = "" +
 	"SELECT state_block_nid, event_nids" +
-	" FROM roomserver_state_block WHERE state_block_nid IN ($1)"
+	" FROM roomserver_state_block WHERE state_block_nid IN ($1) ORDER BY state_block_nid ASC"
 
 type stateBlockStatements struct {
 	db                              *sql.DB

From c8408a6387f6d155fe4e0547cbfdde7123832c84 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 9 Jul 2021 16:36:45 +0100
Subject: [PATCH 07/13] Add more optimised code path for checking if we're in a
 room (#1909)

* Add more optimised code path for checking if we're in a room

* Fix database queries

* Fix federation API test

* Fix logging

* Review comments

* Make separate API call for room membership
---
 federationapi/routing/send.go                 | 27 ++++++++++++-------
 federationapi/routing/send_test.go            |  4 ++-
 roomserver/api/query.go                       |  6 +++--
 roomserver/internal/api.go                    |  1 +
 roomserver/internal/query/query.go            | 11 ++++++++
 roomserver/storage/interface.go               |  2 ++
 .../storage/postgres/membership_table.go      | 23 ++++++++++++++++
 roomserver/storage/shared/storage.go          |  5 ++++
 .../storage/sqlite3/membership_table.go       | 23 ++++++++++++++++
 roomserver/storage/tables/interface.go        |  1 +
 10 files changed, 90 insertions(+), 13 deletions(-)

diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 1c9e72bff..5f214e0fc 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -573,6 +573,23 @@ func (t *txnReq) processEvent(ctx context.Context, e *gomatrixserverlib.Event) e
 	logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID())
 	t.work = "" // reset from previous event
 
+	// Ask the roomserver if we know about the room and/or if we're joined
+	// to it. If we aren't then we won't bother processing the event.
+	joinedReq := api.QueryServerJoinedToRoomRequest{
+		RoomID: e.RoomID(),
+	}
+	var joinedRes api.QueryServerJoinedToRoomResponse
+	if err := t.rsAPI.QueryServerJoinedToRoom(ctx, &joinedReq, &joinedRes); err != nil {
+		return fmt.Errorf("t.rsAPI.QueryServerJoinedToRoom: %w", err)
+	}
+
+	if !joinedRes.RoomExists || !joinedRes.IsInRoom {
+		// We don't believe we're a member of this room, therefore there's
+		// no point in wasting work trying to figure out what to do with
+		// missing auth or prev events. Drop the event.
+		return roomNotFoundError{e.RoomID()}
+	}
+
 	// Work out if the roomserver knows everything it needs to know to auth
 	// the event. This includes the prev_events and auth_events.
 	// NOTE! This is going to include prev_events that have an empty state
@@ -589,16 +606,6 @@ func (t *txnReq) processEvent(ctx context.Context, e *gomatrixserverlib.Event) e
 		return fmt.Errorf("t.rsAPI.QueryMissingAuthPrevEvents: %w", err)
 	}
 
-	if !stateResp.RoomExists {
-		// TODO: When synapse receives a message for a room it is not in it
-		// asks the remote server for the state of the room so that it can
-		// check if the remote server knows of a join "m.room.member" event
-		// that this server is unaware of.
-		// However generally speaking we should reject events for rooms we
-		// aren't a member of.
-		return roomNotFoundError{e.RoomID()}
-	}
-
 	// Prepare a map of all the events we already had before this point, so
 	// that we don't send them to the roomserver again.
 	for _, eventID := range append(e.AuthEventIDs(), e.PrevEventIDs()...) {
diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go
index 98ff1a0a3..0da06aa95 100644
--- a/federationapi/routing/send_test.go
+++ b/federationapi/routing/send_test.go
@@ -190,7 +190,9 @@ func (t *testRoomserverAPI) QueryServerJoinedToRoom(
 	request *api.QueryServerJoinedToRoomRequest,
 	response *api.QueryServerJoinedToRoomResponse,
 ) error {
-	return fmt.Errorf("not implemented")
+	response.RoomExists = true
+	response.IsInRoom = true
+	return nil
 }
 
 // Query whether a server is allowed to see an event
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index af35f7e72..c70db65c1 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -170,7 +170,8 @@ type QueryMembershipsForRoomResponse struct {
 
 // QueryServerJoinedToRoomRequest is a request to QueryServerJoinedToRoom
 type QueryServerJoinedToRoomRequest struct {
-	// Server name of the server to find
+	// Server name of the server to find. If not specified, we will
+	// default to checking if the local server is joined.
 	ServerName gomatrixserverlib.ServerName `json:"server_name"`
 	// ID of the room to see if we are still joined to
 	RoomID string `json:"room_id"`
@@ -182,7 +183,8 @@ type QueryServerJoinedToRoomResponse struct {
 	RoomExists bool `json:"room_exists"`
 	// True if we still believe that we are participating in the room
 	IsInRoom bool `json:"is_in_room"`
-	// List of servers that are also in the room
+	// List of servers that are also in the room. This will not be populated
+	// if the queried ServerName is the local server name.
 	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
 }
 
diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index c9f92f9ff..b05a931fe 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -59,6 +59,7 @@ func NewRoomserverAPI(
 		Queryer: &query.Queryer{
 			DB:         roomserverDB,
 			Cache:      caches,
+			ServerName: cfg.Matrix.ServerName,
 			ServerACLs: serverACLs,
 		},
 		Inputer: &input.Inputer{
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 408f9766e..ccd093726 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -36,6 +36,7 @@ import (
 type Queryer struct {
 	DB         storage.Database
 	Cache      caching.RoomServerCaches
+	ServerName gomatrixserverlib.ServerName
 	ServerACLs *acls.ServerACLs
 }
 
@@ -328,6 +329,16 @@ func (r *Queryer) QueryServerJoinedToRoom(
 	}
 	response.RoomExists = true
 
+	if request.ServerName == r.ServerName || request.ServerName == "" {
+		var joined bool
+		joined, err = r.DB.GetLocalServerInRoom(ctx, info.RoomNID)
+		if err != nil {
+			return fmt.Errorf("r.DB.GetLocalServerInRoom: %w", err)
+		}
+		response.IsInRoom = joined
+		return nil
+	}
+
 	eventNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, true, false)
 	if err != nil {
 		return fmt.Errorf("r.DB.GetMembershipEventNIDsForRoom: %w", err)
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index d2b0e75c9..c25820aac 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -154,6 +154,8 @@ type Database interface {
 	GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
 	// JoinedUsersSetInRooms returns all joined users in the rooms given, along with the count of how many times they appear.
 	JoinedUsersSetInRooms(ctx context.Context, roomIDs []string) (map[string]int, error)
+	// GetLocalServerInRoom returns true if we think we're in a given room or false otherwise.
+	GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error)
 	// GetKnownUsers searches all users that userID knows about.
 	GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error)
 	// GetKnownRooms returns a list of all rooms we know about.
diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go
index 3466da6d2..9102f26a3 100644
--- a/roomserver/storage/postgres/membership_table.go
+++ b/roomserver/storage/postgres/membership_table.go
@@ -124,6 +124,14 @@ var selectKnownUsersSQL = "" +
 	"  SELECT DISTINCT room_nid FROM roomserver_membership WHERE target_nid=$1 AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
 	") AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " AND event_state_key LIKE $2 LIMIT $3"
 
+// selectLocalServerInRoomSQL is an optimised case for checking if we, the local server,
+// are in the room by using the target_local column of the membership table. Normally when
+// we want to know if a server is in a room, we have to unmarshal the entire room state which
+// is expensive. The presence of a single row from this query suggests we're still in the
+// room, no rows returned suggests we aren't.
+const selectLocalServerInRoomSQL = "" +
+	"SELECT room_nid FROM roomserver_membership WHERE target_local = true AND membership_nid = $1 AND room_nid = $2 LIMIT 1"
+
 type membershipStatements struct {
 	insertMembershipStmt                            *sql.Stmt
 	selectMembershipForUpdateStmt                   *sql.Stmt
@@ -137,6 +145,7 @@ type membershipStatements struct {
 	selectJoinedUsersSetForRoomsStmt                *sql.Stmt
 	selectKnownUsersStmt                            *sql.Stmt
 	updateMembershipForgetRoomStmt                  *sql.Stmt
+	selectLocalServerInRoomStmt                     *sql.Stmt
 }
 
 func createMembershipTable(db *sql.DB) error {
@@ -160,6 +169,7 @@ func prepareMembershipTable(db *sql.DB) (tables.Membership, error) {
 		{&s.selectJoinedUsersSetForRoomsStmt, selectJoinedUsersSetForRoomsSQL},
 		{&s.selectKnownUsersStmt, selectKnownUsersSQL},
 		{&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom},
+		{&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL},
 	}.Prepare(db)
 }
 
@@ -324,3 +334,16 @@ func (s *membershipStatements) UpdateForgetMembership(
 	)
 	return err
 }
+
+func (s *membershipStatements) SelectLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) {
+	var nid types.RoomNID
+	err := s.selectLocalServerInRoomStmt.QueryRowContext(ctx, tables.MembershipStateJoin, roomNID).Scan(&nid)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return false, nil
+		}
+		return false, err
+	}
+	found := nid > 0
+	return found, nil
+}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index e77d62e06..9d9434cbb 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -1059,6 +1059,11 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs []string)
 	return result, nil
 }
 
+// GetLocalServerInRoom returns true if we think we're in a given room or false otherwise.
+func (d *Database) GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) {
+	return d.MembershipTable.SelectLocalServerInRoom(ctx, roomNID)
+}
+
 // GetKnownUsers searches all users that userID knows about.
 func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) {
 	stateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID)
diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go
index d9fe32cf8..82babe0d2 100644
--- a/roomserver/storage/sqlite3/membership_table.go
+++ b/roomserver/storage/sqlite3/membership_table.go
@@ -100,6 +100,14 @@ var selectKnownUsersSQL = "" +
 	"  SELECT DISTINCT room_nid FROM roomserver_membership WHERE target_nid=$1 AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
 	") AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " AND event_state_key LIKE $2 LIMIT $3"
 
+// selectLocalServerInRoomSQL is an optimised case for checking if we, the local server,
+// are in the room by using the target_local column of the membership table. Normally when
+// we want to know if a server is in a room, we have to unmarshal the entire room state which
+// is expensive. The presence of a single row from this query suggests we're still in the
+// room, no rows returned suggests we aren't.
+const selectLocalServerInRoomSQL = "" +
+	"SELECT room_nid FROM roomserver_membership WHERE target_local = 1 AND membership_nid = $1 AND room_nid = $2 LIMIT 1"
+
 type membershipStatements struct {
 	db                                              *sql.DB
 	insertMembershipStmt                            *sql.Stmt
@@ -113,6 +121,7 @@ type membershipStatements struct {
 	updateMembershipStmt                            *sql.Stmt
 	selectKnownUsersStmt                            *sql.Stmt
 	updateMembershipForgetRoomStmt                  *sql.Stmt
+	selectLocalServerInRoomStmt                     *sql.Stmt
 }
 
 func createMembershipTable(db *sql.DB) error {
@@ -137,6 +146,7 @@ func prepareMembershipTable(db *sql.DB) (tables.Membership, error) {
 		{&s.selectRoomsWithMembershipStmt, selectRoomsWithMembershipSQL},
 		{&s.selectKnownUsersStmt, selectKnownUsersSQL},
 		{&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom},
+		{&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL},
 	}.Prepare(db)
 }
 
@@ -304,3 +314,16 @@ func (s *membershipStatements) UpdateForgetMembership(
 	)
 	return err
 }
+
+func (s *membershipStatements) SelectLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) {
+	var nid types.RoomNID
+	err := s.selectLocalServerInRoomStmt.QueryRowContext(ctx, tables.MembershipStateJoin, roomNID).Scan(&nid)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return false, nil
+		}
+		return false, err
+	}
+	found := nid > 0
+	return found, nil
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index dd486873a..4a893663f 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -135,6 +135,7 @@ type Membership interface {
 	SelectJoinedUsersSetForRooms(ctx context.Context, roomNIDs []types.RoomNID) (map[types.EventStateKeyNID]int, error)
 	SelectKnownUsers(ctx context.Context, userID types.EventStateKeyNID, searchString string, limit int) ([]string, error)
 	UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error
+	SelectLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error)
 }
 
 type Published interface {

From 1ed732cc783e079feeaf637e23120b027fb7d70b Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Fri, 9 Jul 2021 16:52:31 +0100
Subject: [PATCH 08/13] Implement /_synapse/admin/v1/register (#1911)

* Implement /_synapse/admin/v1/register

This is implemented identically to Synapse, so scripts which work
with Synapse should work with Dendrite.

```
    Test 27 POST /_synapse/admin/v1/register with shared secret... OK
    Test 28 POST /_synapse/admin/v1/register admin with shared secret... OK
    Test 29 POST /_synapse/admin/v1/register with shared secret downcases capitals... OK
    Test 30 POST /_synapse/admin/v1/register with shared secret disallows symbols... OK
```

Sytest however has `implementation_specific => "synapse"` which stops these
tests from running.

* Add missing muxes to gobind

* Linting
---
 build/gobind-pinecone/monolith.go             |  1 +
 build/gobind-yggdrasil/monolith.go            |  1 +
 clientapi/clientapi.go                        |  3 +-
 clientapi/routing/register.go                 | 92 ++++++-----------
 clientapi/routing/register_secret.go          | 99 +++++++++++++++++++
 clientapi/routing/register_secret_test.go     | 43 ++++++++
 clientapi/routing/routing.go                  | 29 +++++-
 cmd/dendrite-demo-libp2p/main.go              |  1 +
 cmd/dendrite-demo-pinecone/main.go            |  1 +
 cmd/dendrite-demo-yggdrasil/main.go           |  1 +
 cmd/dendrite-monolith-server/main.go          |  1 +
 .../personalities/clientapi.go                |  2 +-
 cmd/dendritejs-pinecone/main.go               |  1 +
 cmd/dendritejs/main.go                        |  1 +
 go.mod                                        |  1 +
 go.sum                                        |  2 +
 setup/base.go                                 |  3 +
 setup/monolith.go                             |  4 +-
 18 files changed, 220 insertions(+), 66 deletions(-)
 create mode 100644 clientapi/routing/register_secret.go
 create mode 100644 clientapi/routing/register_secret_test.go

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 09af80f6c..e30057ed4 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -334,6 +334,7 @@ func (m *DendriteMonolith) Start() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 5074f6da4..338628049 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -173,6 +173,7 @@ func (m *DendriteMonolith) Start() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 
 	httpRouter := mux.NewRouter()
diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index 2c4fa5d64..562d89d28 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -35,6 +35,7 @@ import (
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
 func AddPublicRoutes(
 	router *mux.Router,
+	synapseAdminRouter *mux.Router,
 	cfg *config.ClientAPI,
 	accountsDB accounts.Database,
 	federation *gomatrixserverlib.FederationClient,
@@ -56,7 +57,7 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		router, cfg, eduInputAPI, rsAPI, asAPI,
+		router, synapseAdminRouter, cfg, eduInputAPI, rsAPI, asAPI,
 		accountsDB, userAPI, federation,
 		syncProducer, transactionsCache, fsAPI, keyAPI, extRoomsProvider, mscCfg,
 	)
diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go
index 526418669..8823a41e3 100644
--- a/clientapi/routing/register.go
+++ b/clientapi/routing/register.go
@@ -17,10 +17,7 @@ package routing
 
 import (
 	"context"
-	"crypto/hmac"
-	"crypto/sha1"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -594,7 +591,6 @@ func handleRegistrationFlow(
 	accessToken string,
 	accessTokenErr error,
 ) util.JSONResponse {
-	// TODO: Shared secret registration (create new user scripts)
 	// TODO: Enable registration config flag
 	// TODO: Guest account upgrading
 
@@ -643,20 +639,6 @@ func handleRegistrationFlow(
 		// Add Recaptcha to the list of completed registration stages
 		AddCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
 
-	case authtypes.LoginTypeSharedSecret:
-		// Check shared secret against config
-		valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Auth.Mac)
-
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("isValidMacLogin failed")
-			return jsonerror.InternalServerError()
-		} else if !valid {
-			return util.MessageResponse(http.StatusForbidden, "HMAC incorrect")
-		}
-
-		// Add SharedSecret to the list of completed registration stages
-		AddCompletedSessionStage(sessionID, authtypes.LoginTypeSharedSecret)
-
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
 		// Add Dummy to the list of completed registration stages
@@ -849,49 +831,6 @@ func completeRegistration(
 	}
 }
 
-// Used for shared secret registration.
-// Checks if the username, password and isAdmin flag matches the given mac.
-func isValidMacLogin(
-	cfg *config.ClientAPI,
-	username, password string,
-	isAdmin bool,
-	givenMac []byte,
-) (bool, error) {
-	sharedSecret := cfg.RegistrationSharedSecret
-
-	// Check that shared secret registration isn't disabled.
-	if cfg.RegistrationSharedSecret == "" {
-		return false, errors.New("Shared secret registration is disabled")
-	}
-
-	// Double check that username/password don't contain the HMAC delimiters. We should have
-	// already checked this.
-	if strings.Contains(username, "\x00") {
-		return false, errors.New("Username contains invalid character")
-	}
-	if strings.Contains(password, "\x00") {
-		return false, errors.New("Password contains invalid character")
-	}
-	if sharedSecret == "" {
-		return false, errors.New("Shared secret registration is disabled")
-	}
-
-	adminString := "notadmin"
-	if isAdmin {
-		adminString = "admin"
-	}
-	joined := strings.Join([]string{username, password, adminString}, "\x00")
-
-	mac := hmac.New(sha1.New, []byte(sharedSecret))
-	_, err := mac.Write([]byte(joined))
-	if err != nil {
-		return false, err
-	}
-	expectedMAC := mac.Sum(nil)
-
-	return hmac.Equal(givenMac, expectedMAC), nil
-}
-
 // checkFlows checks a single completed flow against another required one. If
 // one contains at least all of the stages that the other does, checkFlows
 // returns true.
@@ -995,3 +934,34 @@ func RegisterAvailable(
 		},
 	}
 }
+
+func handleSharedSecretRegistration(userAPI userapi.UserInternalAPI, sr *SharedSecretRegistration, req *http.Request) util.JSONResponse {
+	ssrr, err := NewSharedSecretRegistrationRequest(req.Body)
+	if err != nil {
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON(fmt.Sprintf("malformed json: %s", err)),
+		}
+	}
+	valid, err := sr.IsValidMacLogin(ssrr.Nonce, ssrr.User, ssrr.Password, ssrr.Admin, ssrr.MacBytes)
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	if !valid {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Forbidden("bad mac"),
+		}
+	}
+	// downcase capitals
+	ssrr.User = strings.ToLower(ssrr.User)
+
+	if resErr := validateUsername(ssrr.User); resErr != nil {
+		return *resErr
+	}
+	if resErr := validatePassword(ssrr.Password); resErr != nil {
+		return *resErr
+	}
+	deviceID := "shared_secret_registration"
+	return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), false, &ssrr.User, &deviceID)
+}
diff --git a/clientapi/routing/register_secret.go b/clientapi/routing/register_secret.go
new file mode 100644
index 000000000..f0436e322
--- /dev/null
+++ b/clientapi/routing/register_secret.go
@@ -0,0 +1,99 @@
+package routing
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/util"
+	cache "github.com/patrickmn/go-cache"
+)
+
+type SharedSecretRegistrationRequest struct {
+	User     string `json:"username"`
+	Password string `json:"password"`
+	Nonce    string `json:"nonce"`
+	MacBytes []byte
+	MacStr   string `json:"mac"`
+	Admin    bool   `json:"admin"`
+}
+
+func NewSharedSecretRegistrationRequest(reader io.ReadCloser) (*SharedSecretRegistrationRequest, error) {
+	defer internal.CloseAndLogIfError(context.Background(), reader, "NewSharedSecretRegistrationRequest: failed to close request body")
+	var ssrr SharedSecretRegistrationRequest
+	err := json.NewDecoder(reader).Decode(&ssrr)
+	if err != nil {
+		return nil, err
+	}
+	ssrr.MacBytes, err = hex.DecodeString(ssrr.MacStr)
+	return &ssrr, err
+}
+
+type SharedSecretRegistration struct {
+	sharedSecret string
+	nonces       *cache.Cache
+}
+
+func NewSharedSecretRegistration(sharedSecret string) *SharedSecretRegistration {
+	return &SharedSecretRegistration{
+		sharedSecret: sharedSecret,
+		// nonces live for 5mins, purge every 10mins
+		nonces: cache.New(5*time.Minute, 10*time.Minute),
+	}
+}
+
+func (r *SharedSecretRegistration) GenerateNonce() string {
+	nonce := util.RandomString(16)
+	r.nonces.Set(nonce, true, cache.DefaultExpiration)
+	return nonce
+}
+
+func (r *SharedSecretRegistration) validNonce(nonce string) bool {
+	_, exists := r.nonces.Get(nonce)
+	return exists
+}
+
+func (r *SharedSecretRegistration) IsValidMacLogin(
+	nonce, username, password string,
+	isAdmin bool,
+	givenMac []byte,
+) (bool, error) {
+	// Check that shared secret registration isn't disabled.
+	if r.sharedSecret == "" {
+		return false, errors.New("Shared secret registration is disabled")
+	}
+	if !r.validNonce(nonce) {
+		return false, fmt.Errorf("Incorrect or expired nonce: %s", nonce)
+	}
+
+	// Check that username/password don't contain the HMAC delimiters.
+	if strings.Contains(username, "\x00") {
+		return false, errors.New("Username contains invalid character")
+	}
+	if strings.Contains(password, "\x00") {
+		return false, errors.New("Password contains invalid character")
+	}
+
+	adminString := "notadmin"
+	if isAdmin {
+		adminString = "admin"
+	}
+	joined := strings.Join([]string{nonce, username, password, adminString}, "\x00")
+
+	mac := hmac.New(sha1.New, []byte(r.sharedSecret))
+	_, err := mac.Write([]byte(joined))
+	if err != nil {
+		return false, err
+	}
+	expectedMAC := mac.Sum(nil)
+
+	return hmac.Equal(givenMac, expectedMAC), nil
+}
diff --git a/clientapi/routing/register_secret_test.go b/clientapi/routing/register_secret_test.go
new file mode 100644
index 000000000..e702b2152
--- /dev/null
+++ b/clientapi/routing/register_secret_test.go
@@ -0,0 +1,43 @@
+package routing
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/patrickmn/go-cache"
+)
+
+func TestSharedSecretRegister(t *testing.T) {
+	// these values have come from a local synapse instance to ensure compatibility
+	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`)
+	sharedSecret := "dendritetest"
+
+	req, err := NewSharedSecretRegistrationRequest(ioutil.NopCloser(bytes.NewBuffer(jsonStr)))
+	if err != nil {
+		t.Fatalf("failed to read request: %s", err)
+	}
+
+	r := NewSharedSecretRegistration(sharedSecret)
+
+	// force the nonce to be known
+	r.nonces.Set(req.Nonce, true, cache.DefaultExpiration)
+
+	valid, err := r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes)
+	if err != nil {
+		t.Fatalf("failed to check for valid mac: %s", err)
+	}
+	if !valid {
+		t.Errorf("mac login failed, wanted success")
+	}
+
+	// modify the mac so it fails
+	req.MacBytes[0] = 0xff
+	valid, err = r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes)
+	if err != nil {
+		t.Fatalf("failed to check for valid mac: %s", err)
+	}
+	if valid {
+		t.Errorf("mac login succeeded, wanted failure")
+	}
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 9f980e0a9..37279e8ed 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -37,6 +37,7 @@ import (
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
@@ -46,7 +47,7 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	publicAPIMux *mux.Router, cfg *config.ClientAPI,
+	publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI,
 	eduAPI eduServerAPI.EDUServerInputAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
@@ -88,6 +89,32 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	if cfg.RegistrationSharedSecret != "" {
+		logrus.Info("Enabling shared secret registration at /_synapse/admin/v1/register")
+		sr := NewSharedSecretRegistration(cfg.RegistrationSharedSecret)
+		synapseAdminRouter.Handle("/admin/v1/register",
+			httputil.MakeExternalAPI("shared_secret_registration", func(req *http.Request) util.JSONResponse {
+				if req.Method == http.MethodGet {
+					return util.JSONResponse{
+						Code: 200,
+						JSON: struct {
+							Nonce string `json:"nonce"`
+						}{
+							Nonce: sr.GenerateNonce(),
+						},
+					}
+				}
+				if req.Method == http.MethodPost {
+					return handleSharedSecretRegistration(userAPI, sr, req)
+				}
+				return util.JSONResponse{
+					Code: http.StatusMethodNotAllowed,
+					JSON: jsonerror.NotFound("unknown method"),
+				}
+			}),
+		).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
+	}
+
 	r0mux := publicAPIMux.PathPrefix("/r0").Subrouter()
 	unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter()
 
diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go
index cc7dcf021..6b0e57d8b 100644
--- a/cmd/dendrite-demo-libp2p/main.go
+++ b/cmd/dendrite-demo-libp2p/main.go
@@ -197,6 +197,7 @@ func main() {
 		base.Base.PublicFederationAPIMux,
 		base.Base.PublicKeyAPIMux,
 		base.Base.PublicMediaAPIMux,
+		base.Base.SynapseAdminMux,
 	)
 	if err := mscs.Enable(&base.Base, &monolith); err != nil {
 		logrus.WithError(err).Fatalf("Failed to enable MSCs")
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 72936e42e..2712ed4a1 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -210,6 +210,7 @@ func main() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 
 	wsUpgrader := websocket.Upgrader{
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 2d710ae79..abeefbe5a 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -154,6 +154,7 @@ func main() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 	if err := mscs.Enable(base, &monolith); err != nil {
 		logrus.WithError(err).Fatalf("Failed to enable MSCs")
diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go
index ef349505c..5efbe8567 100644
--- a/cmd/dendrite-monolith-server/main.go
+++ b/cmd/dendrite-monolith-server/main.go
@@ -149,6 +149,7 @@ func main() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 
 	if len(base.Cfg.MSCs.MSCs) > 0 {
diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go
index ec445ceb7..5e0c43548 100644
--- a/cmd/dendrite-polylith-multi/personalities/clientapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go
@@ -33,7 +33,7 @@ func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	keyAPI := base.KeyServerHTTPClient()
 
 	clientapi.AddPublicRoutes(
-		base.PublicClientAPIMux, &base.Cfg.ClientAPI, accountDB, federation,
+		base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI, accountDB, federation,
 		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, keyAPI, nil,
 		&cfg.MSCs,
 	)
diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go
index 433e9bf82..25e496909 100644
--- a/cmd/dendritejs-pinecone/main.go
+++ b/cmd/dendritejs-pinecone/main.go
@@ -215,6 +215,7 @@ func main() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go
index 7ece94ff0..d5a845ae0 100644
--- a/cmd/dendritejs/main.go
+++ b/cmd/dendritejs/main.go
@@ -236,6 +236,7 @@ func main() {
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicMediaAPIMux,
+		base.SynapseAdminMux,
 	)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
diff --git a/go.mod b/go.mod
index 6e227e6c6..eeb4a7842 100644
--- a/go.mod
+++ b/go.mod
@@ -39,6 +39,7 @@ require (
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6
 	github.com/opentracing/opentracing-go v1.2.0
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pressly/goose v2.7.0+incompatible
 	github.com/prometheus/client_golang v1.9.0
diff --git a/go.sum b/go.sum
index 6bf4632ef..767826e7a 100644
--- a/go.sum
+++ b/go.sum
@@ -1256,6 +1256,8 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
diff --git a/setup/base.go b/setup/base.go
index 6bdeb80f7..7b691608d 100644
--- a/setup/base.go
+++ b/setup/base.go
@@ -77,6 +77,7 @@ type BaseDendrite struct {
 	PublicKeyAPIMux        *mux.Router
 	PublicMediaAPIMux      *mux.Router
 	InternalAPIMux         *mux.Router
+	SynapseAdminMux        *mux.Router
 	UseHTTPAPIs            bool
 	apiHttpClient          *http.Client
 	httpClient             *http.Client
@@ -199,6 +200,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 		PublicKeyAPIMux:        mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(),
 		PublicMediaAPIMux:      mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(),
 		InternalAPIMux:         mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(),
+		SynapseAdminMux:        mux.NewRouter().SkipClean(true).PathPrefix("/_synapse/").Subrouter().UseEncodedPath(),
 		apiHttpClient:          &apiClient,
 		httpClient:             &client,
 	}
@@ -391,6 +393,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.PublicKeyAPIMux)
 		externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(federationHandler)
 	}
+	externalRouter.PathPrefix("/_synapse/").Handler(b.SynapseAdminMux)
 	externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux)
 
 	if internalAddr != NoListener && internalAddr != externalAddr {
diff --git a/setup/monolith.go b/setup/monolith.go
index 235be4474..5ceb4ed30 100644
--- a/setup/monolith.go
+++ b/setup/monolith.go
@@ -57,9 +57,9 @@ type Monolith struct {
 }
 
 // AddAllPublicRoutes attaches all public paths to the given router
-func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, mediaMux *mux.Router) {
+func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, mediaMux, synapseMux *mux.Router) {
 	clientapi.AddPublicRoutes(
-		csMux, &m.Config.ClientAPI, m.AccountDB,
+		csMux, synapseMux, &m.Config.ClientAPI, m.AccountDB,
 		m.FedClient, m.RoomserverAPI,
 		m.EDUInternalAPI, m.AppserviceAPI, transactions.New(),
 		m.FederationSenderAPI, m.UserAPI, m.KeyAPI, m.ExtPublicRoomsProvider,

From acec6fa979a754c41fa83f7d887bfaf55946e17b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 9 Jul 2021 17:49:59 +0100
Subject: [PATCH 09/13] Move a couple of callers to
 helpers.IsServerCurrentlyInRoom over to the query API (#1912)

---
 roomserver/internal/api.go                  |  1 +
 roomserver/internal/helpers/helpers.go      |  8 ++++++++
 roomserver/internal/perform/perform_join.go | 11 ++++++++++-
 roomserver/internal/query/query.go          | 14 ++++++++++----
 4 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index b05a931fe..f39b26eaf 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -93,6 +93,7 @@ func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSen
 		FSAPI:      r.fsAPI,
 		RSAPI:      r,
 		Inputer:    r.Inputer,
+		Queryer:    r.Queryer,
 	}
 	r.Peeker = &perform.Peeker{
 		ServerName: r.Cfg.Matrix.ServerName,
diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index a829bffca..a389cc898 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -50,6 +50,10 @@ func UpdateToInviteMembership(
 	return updates, nil
 }
 
+// IsServerCurrentlyInRoom checks if a server is in a given room, based on the room
+// memberships. If the servername is not supplied then the local server will be
+// checked instead using a faster code path.
+// TODO: This should probably be replaced by an API call.
 func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) {
 	info, err := db.RoomInfo(ctx, roomID)
 	if err != nil {
@@ -59,6 +63,10 @@ func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverNam
 		return false, fmt.Errorf("unknown room %s", roomID)
 	}
 
+	if serverName == "" {
+		return db.GetLocalServerInRoom(ctx, info.RoomNID)
+	}
+
 	eventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, true, false)
 	if err != nil {
 		return false, err
diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index 048496d45..876888e29 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -28,6 +28,7 @@ import (
 	rsAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
+	"github.com/matrix-org/dendrite/roomserver/internal/query"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -42,6 +43,7 @@ type Joiner struct {
 	DB         storage.Database
 
 	Inputer *input.Inputer
+	Queryer *query.Queryer
 }
 
 // PerformJoin handles joining matrix rooms, including over federation by talking to the federationsender.
@@ -205,7 +207,14 @@ func (r *Joiner) performJoinRoomByID(
 
 	// Force a federated join if we aren't in the room and we've been
 	// given some server names to try joining by.
-	serverInRoom, _ := helpers.IsServerCurrentlyInRoom(ctx, r.DB, r.ServerName, req.RoomIDOrAlias)
+	inRoomReq := &api.QueryServerJoinedToRoomRequest{
+		RoomID: req.RoomIDOrAlias,
+	}
+	inRoomRes := &api.QueryServerJoinedToRoomResponse{}
+	if err = r.Queryer.QueryServerJoinedToRoom(ctx, inRoomReq, inRoomRes); err != nil {
+		return "", "", fmt.Errorf("r.Queryer.QueryServerJoinedToRoom: %w", err)
+	}
+	serverInRoom := inRoomRes.IsInRoom
 	forceFederatedJoin := len(req.ServerNames) > 0 && !serverInRoom
 
 	// Force a federated join if we're dealing with a pending invite
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index ccd093726..4af0e6397 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -388,10 +388,16 @@ func (r *Queryer) QueryServerAllowedToSeeEvent(
 		return
 	}
 	roomID := events[0].RoomID()
-	isServerInRoom, err := helpers.IsServerCurrentlyInRoom(ctx, r.DB, request.ServerName, roomID)
-	if err != nil {
-		return
+
+	inRoomReq := &api.QueryServerJoinedToRoomRequest{
+		RoomID:     roomID,
+		ServerName: request.ServerName,
 	}
+	inRoomRes := &api.QueryServerJoinedToRoomResponse{}
+	if err = r.QueryServerJoinedToRoom(ctx, inRoomReq, inRoomRes); err != nil {
+		return fmt.Errorf("r.Queryer.QueryServerJoinedToRoom: %w", err)
+	}
+
 	info, err := r.DB.RoomInfo(ctx, roomID)
 	if err != nil {
 		return err
@@ -400,7 +406,7 @@ func (r *Queryer) QueryServerAllowedToSeeEvent(
 		return fmt.Errorf("QueryServerAllowedToSeeEvent: no room info for room %s", roomID)
 	}
 	response.AllowedToSeeEvent, err = helpers.CheckServerAllowedToSeeEvent(
-		ctx, r.DB, *info, request.EventID, request.ServerName, isServerInRoom,
+		ctx, r.DB, *info, request.EventID, request.ServerName, inRoomRes.IsInRoom,
 	)
 	return
 }

From e48a08fef07e35a87d32591a44d2e416be3b12b1 Mon Sep 17 00:00:00 2001
From: Melroy van den Berg <melroy@melroy.org>
Date: Mon, 12 Jul 2021 11:13:17 +0200
Subject: [PATCH 10/13] Propose config better (#1758)

Better explain where the config file are located and how to deal with the yml file.

Co-authored-by: kegsay <kegan@matrix.org>
---
 build/docker/README.md | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/build/docker/README.md b/build/docker/README.md
index 6d3cd3dbd..19e4234c5 100644
--- a/build/docker/README.md
+++ b/build/docker/README.md
@@ -28,8 +28,8 @@ There are three sample `docker-compose` files:
 The `docker-compose` files refer to the `/etc/dendrite` volume as where the
 runtime config should come from. The mounted folder must contain:
 
-- `dendrite.yaml` configuration file (based on the sample `dendrite-config.yaml`
-   in the `docker/config` folder in the [Dendrite repository](https://github.com/matrix-org/dendrite)
+- `dendrite.yaml` configuration file (based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml)
+   sample in the `build/docker/config` folder of this repository.)
 - `matrix_key.pem` server key, as generated using `cmd/generate-keys`
 - `server.crt` certificate file
 - `server.key` private key file for the above certificate
@@ -50,8 +50,7 @@ The key files will now exist in your current working directory, and can be mount
 
 ## Starting Dendrite as a monolith deployment
 
-Create your config based on the `dendrite.yaml` configuration file in the `docker/config`
-folder in the [Dendrite repository](https://github.com/matrix-org/dendrite).
+Create your config based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml) configuration file in the `build/docker/config` folder of this repository. And rename the config file to `dendrite.yml` (and put it in your `config` directory).
 
 Once in place, start the PostgreSQL dependency:
 
@@ -67,8 +66,7 @@ docker-compose -f docker-compose.monolith.yml up
 
 ## Starting Dendrite as a polylith deployment
 
-Create your config based on the `dendrite.yaml` configuration file in the `docker/config`
-folder in the [Dendrite repository](https://github.com/matrix-org/dendrite).
+Create your config based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml) configuration file in the `build/docker/config` folder of this repository. And rename the config file to `dendrite.yml` (and put it in your `config` directory).
 
 Once in place, start all the dependencies:
 
@@ -84,10 +82,10 @@ docker-compose -f docker-compose.polylith.yml up
 
 ## Building the images
 
-The `docker/images-build.sh` script will build the base image, followed by
+The `build/docker/images-build.sh` script will build the base image, followed by
 all of the component images.
 
-The `docker/images-push.sh` script will push them to Docker Hub (subject
+The `build/docker/images-push.sh` script will push them to Docker Hub (subject
 to permissions).
 
 If you wish to build and push your own images, rename `matrixdotorg/dendrite` to

From 0530302cd6e028f890227309bd221115da637325 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Mon, 12 Jul 2021 11:48:08 +0100
Subject: [PATCH 11/13] Add shared secret sytests to whitelist

---
 sytest-whitelist | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/sytest-whitelist b/sytest-whitelist
index 8c4585716..55922f943 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -520,3 +520,7 @@ Inviting an AS-hosted user asks the AS server
 Can generate a openid access_token that can be exchanged for information about a user
 Invalid openid access tokens are rejected
 Requests to userinfo without access tokens are rejected
+POST /_synapse/admin/v1/register with shared secret
+POST /_synapse/admin/v1/register admin with shared secret
+POST /_synapse/admin/v1/register with shared secret downcases capitals
+POST /_synapse/admin/v1/register with shared secret disallows symbols

From 89a16bdcd9897e9410b0a4b70b4049ebfd087318 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 12 Jul 2021 11:48:29 +0100
Subject: [PATCH 12/13] Version 0.4.0

---
 CHANGES.md          | 41 +++++++++++++++++++++++++++++++++++++++++
 internal/version.go |  4 ++--
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index c909c5715..a11326844 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,46 @@
 # Changelog
 
+## Dendrite 0.4.0 (2021-07-12)
+
+### Features
+
+* All-new state storage in the roomserver, which dramatically reduces disk space utilisation
+  * State snapshots and blocks are now aggressively deduplicated and reused wherever possible, with state blocks being reduced by up to 15x and snapshot references being reduced up to 2x
+  * Dendrite will upgrade to the new state storage automatically on the first run after upgrade, although this may take some time depending on the size of the state storage
+* Appservice support has been improved significantly, with many bridges now working correctly with Dendrite
+  * Events are now correctly sent to appservices based on room memberships
+  * Aliases and namespaces are now handled correctly, calling the appservice to query for aliases as needed
+  * Appservice user registrations are no longer being subject to incorrect validation checks
+* Shared secret registration has now been implemented correctly
+* The roomserver input API implements a new queuing system to reduce backpressure across rooms
+* Checking if the local server is in a room has been optimised substantially, reducing CPU usage
+* State resolution v2 has been optimised further by improving the power level checks, reducing CPU usage
+* The federation API `/send` endpoint now deduplicates missing auth and prev events more aggressively to reduce memory usage
+* The federation API `/send` endpoint now uses workers to reduce backpressure across rooms
+* The bcrypt cost for password storage is now configurable with the `user_api.bcrypt_cost` option
+* The federation API will now use significantly less memory when calling `/get_missing_events`
+* MSC2946 Spaces endpoints have been updated to stable endpoint naming
+* The media API can now be configured without a maximum file size
+* A new `dendrite-upgrade-test` test has been added for verifying database schema upgrades across versions
+* Added Prometheus metrics for roomserver backpressure, excessive device list updates and federation API event processing summaries
+* Sentry support has been added for error reporting
+
+### Fixes
+
+* Removed the legacy `/v1` register endpoint. Dendrite only implements `/r0` of the CS API, and the legacy `/v1` endpoint had implementation errors which made it possible to bypass shared secret registration (thanks to Jakob Varmose Bentzen for reporting this)
+* Attempting to register an account that already exists now returns a sensible error code rather than a HTTP 500
+* Dendrite will no longer attempt to `/make_join` with itself if listed in the request `server_names`
+* `/sync` will no longer return immediately if there is nothing to sync, which happened particularly with new accounts, causing high CPU usage
+* Malicious media uploads can no longer exhaust all available memory (contributed by [S7evinK](https://github.com/S7evinK))
+* Selecting one-time keys from the database has been optimised (contributed by [S7evinK](https://github.com/S7evinK))
+* The return code when trying to fetch missing account data has been fixed (contributed by [adamgreig](https://github.com/adamgreig))
+* Dendrite will no longer attempt to use `/make_leave` over federation when rejecting a local invite
+* A panic has been fixed in `QueryMembershipsForRoom`
+* A panic on duplicate membership events has been fixed in the federation sender
+* A panic has been fixed in in `IsInterestedInRoomID` (contributed by [S7evinK](https://github.com/S7evinK))
+* A panic in the roomserver has been fixed when handling empty state sets
+* A panic in the federation API has been fixed when handling cached events
+
 ## Dendrite 0.3.11 (2021-03-02)
 
 ### Fixes
diff --git a/internal/version.go b/internal/version.go
index 0d3487799..37f0c30d3 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -16,8 +16,8 @@ var build string
 
 const (
 	VersionMajor = 0
-	VersionMinor = 3
-	VersionPatch = 11
+	VersionMinor = 4
+	VersionPatch = 0
 	VersionTag   = "" // example: "rc1"
 )
 

From 48bdd79bdec8f9f13cbc07762060b31d1a2cc6cd Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 12 Jul 2021 11:54:11 +0100
Subject: [PATCH 13/13] Fix attribution in changelog

---
 CHANGES.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index a11326844..27356b3cb 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -37,7 +37,7 @@
 * Dendrite will no longer attempt to use `/make_leave` over federation when rejecting a local invite
 * A panic has been fixed in `QueryMembershipsForRoom`
 * A panic on duplicate membership events has been fixed in the federation sender
-* A panic has been fixed in in `IsInterestedInRoomID` (contributed by [S7evinK](https://github.com/S7evinK))
+* A panic has been fixed in in `IsInterestedInRoomID` (contributed by [bodqhrohro](https://github.com/bodqhrohro))
 * A panic in the roomserver has been fixed when handling empty state sets
 * A panic in the federation API has been fixed when handling cached events