From 48600d554096d54d12a6d3834f8f5f45506e3026 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:00:07 +0200
Subject: [PATCH 01/19] Use `/admin/v1/register` in `create-account` (#2484)

* Get all account data on CompleteSync

* Revert "Get all account data on CompleteSync"

This reverts commit 44a3e566d8fb940b0b757aea9b8408fa19ea9f54.

* Use /_synapse/admin/v1/register to create account

* Linting

* Linter again :)

* Update docs

* Use HTTP API to reset password, add option to User API `PerformPasswordUpdate` to invalidate sessions

* Fix routing name

* Tell me more about what went wrong

* Deprecate the `-reset-password` flag, document the new API

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 clientapi/routing/admin.go           |  78 ++++++++++---
 clientapi/routing/routing.go         |  14 ++-
 cmd/create-account/main.go           | 166 ++++++++++++++++-----------
 docs/administration/1_createusers.md |  12 +-
 docs/administration/4_adminapi.md    |  19 ++-
 internal/httputil/httpapi.go         |  18 +++
 userapi/api/api.go                   |   5 +-
 userapi/internal/api.go              |   5 +
 8 files changed, 227 insertions(+), 90 deletions(-)

diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index a8dd0e64f..0c5f8c167 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -1,23 +1,20 @@
 package routing
 
 import (
+	"encoding/json"
 	"net/http"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
-func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	if device.AccountType != userapi.AccountTypeAdmin {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
-		}
-	}
+func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -50,13 +47,7 @@ func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserv
 	}
 }
 
-func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	if device.AccountType != userapi.AccountTypeAdmin {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
-		}
-	}
+func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -68,6 +59,16 @@ func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserv
 			JSON: jsonerror.MissingArgument("Expecting user ID."),
 		}
 	}
+	_, domain, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return util.MessageResponse(http.StatusBadRequest, err.Error())
+	}
+	if domain != cfg.Matrix.ServerName {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("User ID must belong to this server."),
+		}
+	}
 	res := &roomserverAPI.PerformAdminEvacuateUserResponse{}
 	if err := rsAPI.PerformAdminEvacuateUser(
 		req.Context(),
@@ -88,3 +89,52 @@ func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserv
 		},
 	}
 }
+
+func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	localpart, ok := vars["localpart"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting user localpart."),
+		}
+	}
+	request := struct {
+		Password string `json:"password"`
+	}{}
+	if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("Failed to decode request body: " + err.Error()),
+		}
+	}
+	if request.Password == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting non-empty password."),
+		}
+	}
+	updateReq := &userapi.PerformPasswordUpdateRequest{
+		Localpart:     localpart,
+		Password:      request.Password,
+		LogoutDevices: true,
+	}
+	updateRes := &userapi.PerformPasswordUpdateResponse{}
+	if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("Failed to perform password update: " + err.Error()),
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct {
+			Updated bool `json:"password_updated"`
+		}{
+			Updated: updateRes.PasswordUpdated,
+		},
+	}
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index ced4fdbcf..2063a008f 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -144,17 +144,23 @@ func Setup(
 	}
 
 	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
-		httputil.MakeAuthAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateRoom(req, device, rsAPI)
+		httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminEvacuateRoom(req, cfg, device, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}",
-		httputil.MakeAuthAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateUser(req, device, rsAPI)
+		httputil.MakeAdminAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminEvacuateUser(req, cfg, device, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/resetPassword/{localpart}",
+		httputil.MakeAdminAPI("admin_reset_password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminResetPassword(req, cfg, device, userAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index 92179a049..44d5691c2 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -15,20 +15,26 @@
 package main
 
 import (
-	"context"
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"regexp"
 	"strings"
+	"time"
+
+	"github.com/tidwall/gjson"
 
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/term"
+
+	"github.com/matrix-org/dendrite/setup"
 )
 
 const usage = `Usage: %s
@@ -58,12 +64,17 @@ var (
 	password           = flag.String("password", "", "The password to associate with the account")
 	pwdFile            = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
 	pwdStdin           = flag.Bool("passwordstdin", false, "Reads the password from stdin")
-	pwdLess            = flag.Bool("passwordless", false, "Create a passwordless account, e.g. if only an accesstoken is required")
 	isAdmin            = flag.Bool("admin", false, "Create an admin account")
 	resetPassword      = flag.Bool("reset-password", false, "Resets the password for the given username")
+	serverURL          = flag.String("url", "https://localhost:8448", "The URL to connect to.")
 	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 )
 
+var cl = http.Client{
+	Timeout:   time.Second * 10,
+	Transport: http.DefaultTransport,
+}
+
 func main() {
 	name := os.Args[0]
 	flag.Usage = func() {
@@ -72,15 +83,15 @@ func main() {
 	}
 	cfg := setup.ParseFlags(true)
 
+	if *resetPassword {
+		logrus.Fatalf("The reset-password flag has been replaced by the POST /_dendrite/admin/resetPassword/{localpart} admin API.")
+	}
+
 	if *username == "" {
 		flag.Usage()
 		os.Exit(1)
 	}
 
-	if *pwdLess && *resetPassword {
-		logrus.Fatalf("Can not reset to an empty password, unable to login afterwards.")
-	}
-
 	if !validUsernameRegex.MatchString(*username) {
 		logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='")
 		os.Exit(1)
@@ -90,67 +101,94 @@ func main() {
 		logrus.Fatalf("Username can not be longer than 255 characters: %s", fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName))
 	}
 
-	var pass string
-	var err error
-	if !*pwdLess {
-		pass, err = getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
-		if err != nil {
-			logrus.Fatalln(err)
-		}
-	}
-
-	// avoid warning about open registration
-	cfg.ClientAPI.RegistrationDisabled = true
-
-	b := base.NewBaseDendrite(cfg, "")
-	defer b.Close() // nolint: errcheck
-
-	accountDB, err := storage.NewUserAPIDatabase(
-		b,
-		&cfg.UserAPI.AccountDatabase,
-		cfg.Global.ServerName,
-		cfg.UserAPI.BCryptCost,
-		cfg.UserAPI.OpenIDTokenLifetimeMS,
-		0, // TODO
-		cfg.Global.ServerNotices.LocalPart,
-	)
+	pass, err := getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
 	if err != nil {
-		logrus.WithError(err).Fatalln("Failed to connect to the database")
+		logrus.Fatalln(err)
 	}
 
-	accType := api.AccountTypeUser
-	if *isAdmin {
-		accType = api.AccountTypeAdmin
-	}
-
-	available, err := accountDB.CheckAccountAvailability(context.Background(), *username)
-	if err != nil {
-		logrus.Fatalln("Unable check username existence.")
-	}
-	if *resetPassword {
-		if available {
-			logrus.Fatalln("Username could not be found.")
-		}
-		err = accountDB.SetPassword(context.Background(), *username, pass)
-		if err != nil {
-			logrus.Fatalf("Failed to update password for user %s: %s", *username, err.Error())
-		}
-		if _, err = accountDB.RemoveAllDevices(context.Background(), *username, ""); err != nil {
-			logrus.Fatalf("Failed to remove all devices: %s", err.Error())
-		}
-		logrus.Infof("Updated password for user %s and invalidated all logins\n", *username)
-		return
-	}
-	if !available {
-		logrus.Fatalln("Username is already in use.")
-	}
-
-	_, err = accountDB.CreateAccount(context.Background(), *username, pass, "", accType)
+	accessToken, err := sharedSecretRegister(cfg.ClientAPI.RegistrationSharedSecret, *serverURL, *username, pass, *isAdmin)
 	if err != nil {
 		logrus.Fatalln("Failed to create the account:", err.Error())
 	}
 
-	logrus.Infoln("Created account", *username)
+	logrus.Infof("Created account: %s (AccessToken: %s)", *username, accessToken)
+}
+
+type sharedSecretRegistrationRequest struct {
+	User     string `json:"username"`
+	Password string `json:"password"`
+	Nonce    string `json:"nonce"`
+	MacStr   string `json:"mac"`
+	Admin    bool   `json:"admin"`
+}
+
+func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, admin bool) (accesToken string, err error) {
+	registerURL := fmt.Sprintf("%s/_synapse/admin/v1/register", serverURL)
+	nonceReq, err := http.NewRequest(http.MethodGet, registerURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("unable to create http request: %w", err)
+	}
+	nonceResp, err := cl.Do(nonceReq)
+	if err != nil {
+		return "", fmt.Errorf("unable to get nonce: %w", err)
+	}
+	body, err := io.ReadAll(nonceResp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body: %w", err)
+	}
+	defer nonceResp.Body.Close() // nolint: errcheck
+
+	nonce := gjson.GetBytes(body, "nonce").Str
+
+	adminStr := "notadmin"
+	if admin {
+		adminStr = "admin"
+	}
+	reg := sharedSecretRegistrationRequest{
+		User:     localpart,
+		Password: password,
+		Nonce:    nonce,
+		Admin:    admin,
+	}
+	macStr, err := getRegisterMac(sharedSecret, nonce, localpart, password, adminStr)
+	if err != nil {
+		return "", err
+	}
+	reg.MacStr = macStr
+
+	js, err := json.Marshal(reg)
+	if err != nil {
+		return "", fmt.Errorf("unable to marshal json: %w", err)
+	}
+	registerReq, err := http.NewRequest(http.MethodPost, registerURL, bytes.NewBuffer(js))
+	if err != nil {
+		return "", fmt.Errorf("unable to create http request: %w", err)
+
+	}
+	regResp, err := cl.Do(registerReq)
+	if err != nil {
+		return "", fmt.Errorf("unable to create account: %w", err)
+	}
+	defer regResp.Body.Close() // nolint: errcheck
+	if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
+		body, _ = io.ReadAll(regResp.Body)
+		return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
+	}
+	r, _ := io.ReadAll(regResp.Body)
+
+	return gjson.GetBytes(r, "access_token").Str, nil
+}
+
+func getRegisterMac(sharedSecret, nonce, localpart, password, adminStr string) (string, error) {
+	joined := strings.Join([]string{nonce, localpart, password, adminStr}, "\x00")
+	mac := hmac.New(sha1.New, []byte(sharedSecret))
+	_, err := mac.Write([]byte(joined))
+	if err != nil {
+		return "", fmt.Errorf("unable to construct mac: %w", err)
+	}
+	regMac := mac.Sum(nil)
+
+	return hex.EncodeToString(regMac), nil
 }
 
 func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string, error) {
diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md
index 61ec2299b..3468398ac 100644
--- a/docs/administration/1_createusers.md
+++ b/docs/administration/1_createusers.md
@@ -14,9 +14,8 @@ User accounts can be created on a Dendrite instance in a number of ways.
 The `create-account` tool is built in the `bin` folder when building Dendrite with
 the `build.sh` script.
 
-It uses the `dendrite.yaml` configuration file to connect to the Dendrite user database
-and create the account entries directly. It can therefore be used even if Dendrite is not
-running yet, as long as the database is up.
+It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires 
+shared secret registration to be enabled as explained below.
 
 An example of using `create-account` to create a **normal account**:
 
@@ -32,6 +31,13 @@ To create a new **admin account**, add the `-admin` flag:
 ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
 ```
 
+By default `create-account` uses `https://localhost:8448` to connect to Dendrite, this can be overwritten using 
+the `-url` flag:
+
+```bash
+./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url http://localhost:8008
+```
+
 An example of using `create-account` when running in **Docker**, having found the `CONTAINERNAME` from `docker ps`:
 
 ```bash
diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md
index 51f56374b..783fee95a 100644
--- a/docs/administration/4_adminapi.md
+++ b/docs/administration/4_adminapi.md
@@ -13,19 +13,32 @@ without warning.
 
 More endpoints will be added in the future.
 
-## `/_dendrite/admin/evacuateRoom/{roomID}`
+## GET `/_dendrite/admin/evacuateRoom/{roomID}`
 
 This endpoint will instruct Dendrite to part all local users from the given `roomID`
 in the URL. It may take some time to complete. A JSON body will be returned containing
 the user IDs of all affected users.
 
-## `/_dendrite/admin/evacuateUser/{userID}`
+## GET `/_dendrite/admin/evacuateUser/{userID}`
 
 This endpoint will instruct Dendrite to part the given local `userID` in the URL from
 all rooms which they are currently joined. A JSON body will be returned containing
 the room IDs of all affected rooms.
 
-## `/_synapse/admin/v1/register`
+## POST `/_dendrite/admin/resetPassword/{localpart}`
+
+Request body format:
+
+```
+{
+    "password": "new_password_here"
+}
+```
+
+Reset the password of a local user. The `localpart` is the username only, i.e. if
+the full user ID is `@alice:domain.com` then the local part is `alice`.
+
+## GET `/_synapse/admin/v1/register`
 
 Shared secret registration — please see the [user creation page](createusers) for
 guidance on configuring and using this endpoint.
diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go
index aba50ae4d..e0436c60a 100644
--- a/internal/httputil/httpapi.go
+++ b/internal/httputil/httpapi.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -83,6 +84,23 @@ func MakeAuthAPI(
 	return MakeExternalAPI(metricsName, h)
 }
 
+// MakeAdminAPI is a wrapper around MakeAuthAPI which enforces that the request can only be
+// completed by a user that is a server administrator.
+func MakeAdminAPI(
+	metricsName string, userAPI userapi.QueryAcccessTokenAPI,
+	f func(*http.Request, *userapi.Device) util.JSONResponse,
+) http.Handler {
+	return MakeAuthAPI(metricsName, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		if device.AccountType != userapi.AccountTypeAdmin {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden("This API can only be used by admin users."),
+			}
+		}
+		return f(req, device)
+	})
+}
+
 // MakeExternalAPI turns a util.JSONRequestHandler function into an http.Handler.
 // This is used for APIs that are called from the internet.
 func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse) http.Handler {
diff --git a/userapi/api/api.go b/userapi/api/api.go
index 388f97cb4..66ee9c7c8 100644
--- a/userapi/api/api.go
+++ b/userapi/api/api.go
@@ -334,8 +334,9 @@ type PerformAccountCreationResponse struct {
 
 // PerformAccountCreationRequest is the request for PerformAccountCreation
 type PerformPasswordUpdateRequest struct {
-	Localpart string // Required: The localpart for this account.
-	Password  string // Required: The new password to set.
+	Localpart     string // Required: The localpart for this account.
+	Password      string // Required: The new password to set.
+	LogoutDevices bool   // Optional: Whether to log out all user devices.
 }
 
 // PerformAccountCreationResponse is the response for PerformAccountCreation
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 78b226d46..6ba469327 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -139,6 +139,11 @@ func (a *UserInternalAPI) PerformPasswordUpdate(ctx context.Context, req *api.Pe
 	if err := a.DB.SetPassword(ctx, req.Localpart, req.Password); err != nil {
 		return err
 	}
+	if req.LogoutDevices {
+		if _, err := a.DB.RemoveAllDevices(context.Background(), req.Localpart, ""); err != nil {
+			return err
+		}
+	}
 	res.PasswordUpdated = true
 	return nil
 }

From b4647fbb7e36014c01d4ec8a65e2644133103683 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:33:31 +0200
Subject: [PATCH 02/19] Show/hide users in user directory (#2637)

* CS API changes

* Query remote profiles

* Add passing tests

* Don't create a new FullyQualifiedProfile

* Handle sql.ErrNoRows

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 clientapi/routing/routing.go       |   4 +-
 clientapi/routing/userdirectory.go | 108 +++++++++++++++++------------
 roomserver/internal/query/query.go |   9 +--
 sytest-whitelist                   |   6 +-
 4 files changed, 75 insertions(+), 52 deletions(-)

diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 2063a008f..6904a2b34 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -935,12 +935,12 @@ func Setup(
 			return SearchUserDirectory(
 				req.Context(),
 				device,
-				userAPI,
 				rsAPI,
 				userDirectoryProvider,
-				cfg.Matrix.ServerName,
 				postContent.SearchString,
 				postContent.Limit,
+				federation,
+				cfg.Matrix.ServerName,
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
diff --git a/clientapi/routing/userdirectory.go b/clientapi/routing/userdirectory.go
index f311457a0..d3d1c22e4 100644
--- a/clientapi/routing/userdirectory.go
+++ b/clientapi/routing/userdirectory.go
@@ -18,10 +18,13 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"net/http"
+	"strings"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -34,12 +37,12 @@ type UserDirectoryResponse struct {
 func SearchUserDirectory(
 	ctx context.Context,
 	device *userapi.Device,
-	userAPI userapi.ClientUserAPI,
 	rsAPI api.ClientRoomserverAPI,
 	provider userapi.QuerySearchProfilesAPI,
-	serverName gomatrixserverlib.ServerName,
 	searchString string,
 	limit int,
+	federation *gomatrixserverlib.FederationClient,
+	localServerName gomatrixserverlib.ServerName,
 ) util.JSONResponse {
 	if limit < 10 {
 		limit = 10
@@ -51,59 +54,74 @@ func SearchUserDirectory(
 		Limited: false,
 	}
 
-	// First start searching local users.
-	userReq := &userapi.QuerySearchProfilesRequest{
-		SearchString: searchString,
-		Limit:        limit,
+	// Get users we share a room with
+	knownUsersReq := &api.QueryKnownUsersRequest{
+		UserID: device.UserID,
+		Limit:  limit,
 	}
-	userRes := &userapi.QuerySearchProfilesResponse{}
-	if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
-		return util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
+	knownUsersRes := &api.QueryKnownUsersResponse{}
+	if err := rsAPI.QueryKnownUsers(ctx, knownUsersReq, knownUsersRes); err != nil && err != sql.ErrNoRows {
+		return util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
 	}
 
-	for _, user := range userRes.Profiles {
+knownUsersLoop:
+	for _, profile := range knownUsersRes.Users {
 		if len(results) == limit {
 			response.Limited = true
 			break
 		}
-
-		var userID string
-		if user.ServerName != "" {
-			userID = fmt.Sprintf("@%s:%s", user.Localpart, user.ServerName)
+		userID := profile.UserID
+		// get the full profile of the local user
+		localpart, serverName, _ := gomatrixserverlib.SplitID('@', userID)
+		if serverName == localServerName {
+			userReq := &userapi.QuerySearchProfilesRequest{
+				SearchString: localpart,
+				Limit:        limit,
+			}
+			userRes := &userapi.QuerySearchProfilesResponse{}
+			if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
+				return util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
+			}
+			for _, p := range userRes.Profiles {
+				if strings.Contains(p.DisplayName, searchString) ||
+					strings.Contains(p.Localpart, searchString) {
+					profile.DisplayName = p.DisplayName
+					profile.AvatarURL = p.AvatarURL
+					results[userID] = profile
+					if len(results) == limit {
+						response.Limited = true
+						break knownUsersLoop
+					}
+				}
+			}
 		} else {
-			userID = fmt.Sprintf("@%s:%s", user.Localpart, serverName)
-		}
-		if _, ok := results[userID]; !ok {
-			results[userID] = authtypes.FullyQualifiedProfile{
-				UserID:      userID,
-				DisplayName: user.DisplayName,
-				AvatarURL:   user.AvatarURL,
+			// If the username already contains the search string, don't bother hitting federation.
+			// This will result in missing avatars and displaynames, but saves the federation roundtrip.
+			if strings.Contains(localpart, searchString) {
+				results[userID] = profile
+				if len(results) == limit {
+					response.Limited = true
+					break knownUsersLoop
+				}
+				continue
 			}
-		}
-	}
-
-	// Then, if we have enough room left in the response,
-	// start searching for known users from joined rooms.
-
-	if len(results) <= limit {
-		stateReq := &api.QueryKnownUsersRequest{
-			UserID:       device.UserID,
-			SearchString: searchString,
-			Limit:        limit - len(results),
-		}
-		stateRes := &api.QueryKnownUsersResponse{}
-		if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil && err != sql.ErrNoRows {
-			return util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
-		}
-
-		for _, user := range stateRes.Users {
-			if len(results) == limit {
-				response.Limited = true
-				break
+			// TODO: We should probably cache/store this
+			fedProfile, fedErr := federation.LookupProfile(ctx, serverName, userID, "")
+			if fedErr != nil {
+				if x, ok := fedErr.(gomatrix.HTTPError); ok {
+					if x.Code == http.StatusNotFound {
+						continue
+					}
+				}
 			}
-
-			if _, ok := results[user.UserID]; !ok {
-				results[user.UserID] = user
+			if strings.Contains(fedProfile.DisplayName, searchString) {
+				profile.DisplayName = fedProfile.DisplayName
+				profile.AvatarURL = fedProfile.AvatarURL
+				results[userID] = profile
+				if len(results) == limit {
+					response.Limited = true
+					break knownUsersLoop
+				}
 			}
 		}
 	}
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index c41e1ea67..f3061c222 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -21,6 +21,10 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/acls"
@@ -30,9 +34,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/roomserver/version"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 type Queryer struct {
@@ -732,7 +733,7 @@ func (r *Queryer) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForU
 
 func (r *Queryer) QueryKnownUsers(ctx context.Context, req *api.QueryKnownUsersRequest, res *api.QueryKnownUsersResponse) error {
 	users, err := r.DB.GetKnownUsers(ctx, req.UserID, req.SearchString, req.Limit)
-	if err != nil {
+	if err != nil && err != sql.ErrNoRows {
 		return err
 	}
 	for _, user := range users {
diff --git a/sytest-whitelist b/sytest-whitelist
index 88dfe920b..dcffeaffb 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -739,4 +739,8 @@ Only see history_visibility changes on boundaries
 Current state appears in timeline in private history
 Current state appears in timeline in private history with many messages before
 Local users can peek into world_readable rooms by room ID
-Newly joined room includes presence in incremental sync
\ No newline at end of file
+Newly joined room includes presence in incremental sync
+User in private room doesn't appear in user directory
+User joining then leaving public room appears and dissappears from directory
+User in remote room doesn't appear in user directory after server left room
+User in shared private room does appear in user directory until leave
\ No newline at end of file

From f1e9108db8768b37999ee35b2550e795e53edaef Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 12 Aug 2022 12:53:29 +0100
Subject: [PATCH 03/19] Update `create-account` help text

---
 cmd/create-account/main.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index 44d5691c2..fbd14b8d7 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -52,8 +52,6 @@ Example:
 	# read password from stdin
 	%s --config dendrite.yaml -username alice -passwordstdin < my.pass
 	cat my.pass | %s --config dendrite.yaml -username alice -passwordstdin
-	# reset password for a user, can be used with a combination above to read the password
-	%s --config dendrite.yaml -reset-password -username alice -password foobarbaz
 
 Arguments:
 
@@ -65,7 +63,7 @@ var (
 	pwdFile            = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
 	pwdStdin           = flag.Bool("passwordstdin", false, "Reads the password from stdin")
 	isAdmin            = flag.Bool("admin", false, "Create an admin account")
-	resetPassword      = flag.Bool("reset-password", false, "Resets the password for the given username")
+	resetPassword      = flag.Bool("reset-password", false, "Deprecated")
 	serverURL          = flag.String("url", "https://localhost:8448", "The URL to connect to.")
 	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 )
@@ -78,7 +76,7 @@ var cl = http.Client{
 func main() {
 	name := os.Args[0]
 	flag.Usage = func() {
-		_, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name, name)
+		_, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name)
 		flag.PrintDefaults()
 	}
 	cfg := setup.ParseFlags(true)

From 20bf00b743c20a8ea57a46737ab366580d219d9e Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:55:45 +0200
Subject: [PATCH 04/19] Update README

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 10e2b1b86..638161bf2 100644
--- a/README.md
+++ b/README.md
@@ -89,12 +89,12 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of August 2022 we're at around 83% CS API coverage and 95% Federation coverage, though check
+updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well, although there are still some missing features (like Search).
 
 We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather
-than features that massive deployments may be interested in (User Directory, OpenID, Guests, Admin APIs, AS API).
+than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API).
 This means Dendrite supports amongst others:
 
 - Core room functionality (creating rooms, invites, auth rules)

From 9fa30f5d3cadcfd15394c61b8058e1e6cda49afc Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:29:09 +0100
Subject: [PATCH 05/19] Update NATS Server and nats.go libraries

---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 79ef5c3e8..108165ba2 100644
--- a/go.mod
+++ b/go.mod
@@ -25,8 +25,8 @@ require (
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
-	github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee
-	github.com/nats-io/nats.go v1.16.1-0.20220731182438-87bbea85922b
+	github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c
+	github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
diff --git a/go.sum b/go.sum
index 9d5c50d2c..e925d6415 100644
--- a/go.sum
+++ b/go.sum
@@ -385,10 +385,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
 github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
-github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee h1:vAtoZ+LW6eIUjkCWWwO1DZ6o16UGrVOG+ot/AkwejO8=
-github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee/go.mod h1:3Yg3ApyQxPlAs1KKHKV5pobV5VtZk+TtOiUJx/iqkkg=
-github.com/nats-io/nats.go v1.16.1-0.20220731182438-87bbea85922b h1:CE9wSYLvwq8aC/0+6zH8lhhtZYvJ9p8PzwvZeYgdBc0=
-github.com/nats-io/nats.go v1.16.1-0.20220731182438-87bbea85922b/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
+github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c h1:U5qngWGZ7E/nQxz0544IpIEdKFUUaOJxQN2LHCYLGhg=
+github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c/go.mod h1:+f++B/5jpr71JATt7b5KCX+G7bt43iWx1OYWGkpE/Kk=
+github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995 h1:CUcSQR8jwa9//qNgN/t3tW53DObnTPQ/G/K+qnS7yRc=
+github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
 github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=

From 6d0d7a0bc34563e272dbc045c13cfadc826946d1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:34:05 +0100
Subject: [PATCH 06/19] Update example in readme

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 638161bf2..34a08be94 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co
 
 # Create an user account (add -admin for an admin user).
 # Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
-$ ./bin/create-account --config dendrite.yaml -username alice
+$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice
 ```
 
 Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.

From e55cd6ea785e927996cd1e0461d5c1826aa89ef5 Mon Sep 17 00:00:00 2001
From: texuf <texuf.eth@gmail.com>
Date: Fri, 12 Aug 2022 06:07:45 -0700
Subject: [PATCH 07/19] /hierarchy - return public and knockable rooms for
 authed users (#2578)

When requesting the room hierarchy with an authenticated user, return public and knockable rooms.

According to the spec, https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/2946-spaces-summary.md

```
Any child room that the user is joined or is potentially joinable is included in the response.
```

This is currently not the case. See discussion here: https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$t2Csj-6y1PVsn8GOnFZfXzeQW13NfqvrFCxB-XI_uhA?via=matrix.org&via=libera.chat&via=element.io and here: https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$EHp1x1DY7tnYZtx_PVEb-sKB9lmJajqHx2uGlhrRh6k?via=matrix.org&via=libera.chat&via=element.io

Test Plan:
create and register clients bob and alice
have bob create a public space
have bob create a public room parented to the space
have alice join the space(room)
have alice sync the space
expect alice to see two rooms in the space hierarchy, the space and the child room

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 setup/mscs/msc2946/msc2946.go | 41 ++++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go
index 4cffa82ad..a92a16a27 100644
--- a/setup/mscs/msc2946/msc2946.go
+++ b/setup/mscs/msc2946/msc2946.go
@@ -479,7 +479,7 @@ func (w *walker) authorised(roomID, parentRoomID string) (authed, isJoinedOrInvi
 	return w.authorisedServer(roomID), false
 }
 
-// authorisedServer returns true iff the server is joined this room or the room is world_readable
+// authorisedServer returns true iff the server is joined this room or the room is world_readable, public, or knockable
 func (w *walker) authorisedServer(roomID string) bool {
 	// Check history visibility / join rules first
 	hisVisTuple := gomatrixserverlib.StateKeyTuple{
@@ -513,8 +513,21 @@ func (w *walker) authorisedServer(roomID string) bool {
 	// in addition to the actual room ID (but always do the actual one first as it's quicker in the common case)
 	allowJoinedToRoomIDs := []string{roomID}
 	joinRuleEv := queryRoomRes.StateEvents[joinRuleTuple]
+
 	if joinRuleEv != nil {
-		allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")...)
+		rule, ruleErr := joinRuleEv.JoinRule()
+		if ruleErr != nil {
+			util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule")
+			return false
+		}
+
+		if rule == gomatrixserverlib.Public || rule == gomatrixserverlib.Knock {
+			return true
+		}
+
+		if rule == gomatrixserverlib.Restricted {
+			allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")...)
+		}
 	}
 
 	// check if server is joined to any allowed room
@@ -537,7 +550,8 @@ func (w *walker) authorisedServer(roomID string) bool {
 	return false
 }
 
-// authorisedUser returns true iff the user is invited/joined this room or the room is world_readable.
+// authorisedUser returns true iff the user is invited/joined this room or the room is world_readable
+// or if the room has a public or knock join rule.
 // Failing that, if the room has a restricted join rule and belongs to the space parent listed, it will return true.
 func (w *walker) authorisedUser(roomID, parentRoomID string) (authed bool, isJoinedOrInvited bool) {
 	hisVisTuple := gomatrixserverlib.StateKeyTuple{
@@ -579,13 +593,20 @@ func (w *walker) authorisedUser(roomID, parentRoomID string) (authed bool, isJoi
 	}
 	joinRuleEv := queryRes.StateEvents[joinRuleTuple]
 	if parentRoomID != "" && joinRuleEv != nil {
-		allowedRoomIDs := w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")
-		// check parent is in the allowed set
 		var allowed bool
-		for _, a := range allowedRoomIDs {
-			if parentRoomID == a {
-				allowed = true
-				break
+		rule, ruleErr := joinRuleEv.JoinRule()
+		if ruleErr != nil {
+			util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", parentRoomID).Warn("failed to get join rule")
+		} else if rule == gomatrixserverlib.Public || rule == gomatrixserverlib.Knock {
+			allowed = true
+		} else if rule == gomatrixserverlib.Restricted {
+			allowedRoomIDs := w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")
+			// check parent is in the allowed set
+			for _, a := range allowedRoomIDs {
+				if parentRoomID == a {
+					allowed = true
+					break
+				}
 			}
 		}
 		if allowed {
@@ -615,7 +636,7 @@ func (w *walker) authorisedUser(roomID, parentRoomID string) (authed bool, isJoi
 
 func (w *walker) restrictedJoinRuleAllowedRooms(joinRuleEv *gomatrixserverlib.HeaderedEvent, allowType string) (allows []string) {
 	rule, _ := joinRuleEv.JoinRule()
-	if rule != "restricted" {
+	if rule != gomatrixserverlib.Restricted {
 		return nil
 	}
 	var jrContent gomatrixserverlib.JoinRuleContent

From ef7262a7a21f6106b8130a7ad8a1911cbd46a82f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 12 Aug 2022 15:18:31 +0100
Subject: [PATCH 08/19] Version 0.9.2 (#2638)

---
 CHANGES.md          | 27 ++++++++++++++++++++++++++-
 internal/version.go |  2 +-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 5dd8da362..041816ae3 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,30 @@
 # Changelog
 
+## Dendrite 0.9.2 (2022-08-12)
+
+### Features
+
+* Dendrite now supports history visibility on the `/sync`, `/messages` and `/context` endpoints
+  * It should now be possible to view the history of a room in more cases (as opposed to limiting scrollback to the join event or defaulting to the restrictive `"join"` visibility rule as before)
+* The default room version for newly created rooms is now room version 9
+* New admin endpoint `/_dendrite/admin/resetPassword/{userID}` has been added, which replaces the `-reset-password` flag in `create-account`
+* The `create-account` binary now uses shared secret registration over HTTP to create new accounts, which fixes a number of problems with account data and push rules not being configured correctly for new accounts
+* The internal HTTP APIs for polylith deployments have been refactored for correctness and consistency
+* The federation API will now automatically clean up some EDUs that have failed to send within a certain period of time
+* The `/hierarchy` endpoint will now return potentially joinable rooms (contributed by [texuf](https://github.com/texuf))
+* The user directory will now show or hide users correctly
+
+### Fixes
+
+* Send-to-device messages should no longer be incorrectly duplicated in `/sync`
+* The federation sender will no longer create unnecessary destination queues as a result of a logic error
+* A bug where database migrations may not execute properly when upgrading from older versions has been fixed
+* A crash when failing to update user account data has been fixed
+* A race condition when generating notification counts has been fixed
+* A race condition when setting up NATS has been fixed (contributed by [brianathere](https://github.com/brianathere))
+* Stale cache data for membership lazy-loading is now correctly invalidated when doing a complete sync
+* Data races within user-interactive authentication have been fixed (contributed by [tak-hntlabs](https://github.com/tak-hntlabs))
+
 ## Dendrite 0.9.1 (2022-08-03)
 
 ### Fixes
@@ -10,7 +35,7 @@
 * The media endpoint now sets the `Cache-Control` header correctly to prevent web-based clients from hitting media endpoints excessively
 * The sync API will now advance the PDU stream position correctly in all cases (contributed by [sergekh2](https://github.com/sergekh2))
 * The sync API will now delete the correct range of send-to-device messages when advancing the stream position
-* The device list `changed` key in the `/sync` response should now return the correct users 
+* The device list `changed` key in the `/sync` response should now return the correct users
 * A data race when looking up missing state has been fixed
 * The `/send_join` API is now applying stronger validation to the received membership event
 
diff --git a/internal/version.go b/internal/version.go
index 38d0864e7..18db2dcc8 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 1
+	VersionPatch = 2
 	VersionTag   = "" // example: "rc1"
 )
 

From 30dbc5ac448e30f69305c81b7049d6bd30ae76d0 Mon Sep 17 00:00:00 2001
From: timaeos <104638316+timaeos@users.noreply.github.com>
Date: Sat, 13 Aug 2022 13:24:52 -0500
Subject: [PATCH 09/19] Adding FAQ Questions with common questions (#2641)

* Adding FAQ Questions with common questions

* changing order of the FAQ to reflect suggestions
---
 docs/FAQ.md                              | 26 +++++++++++++++++++++++-
 docs/administration/5_troubleshooting.md |  9 +++++++-
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/docs/FAQ.md b/docs/FAQ.md
index f8255684e..85ad83db5 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -12,7 +12,13 @@ Mostly, although there are still bugs and missing features. If you are a confide
 
 ## Is Dendrite feature-complete?
 
-No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the readme at the root of the repository for more information.
+No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](../README.md) at the root of the repository for more information.
+
+## Why doesn't Dendrite have <blank> yet?
+
+Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be 
+specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting 
+the development efforts through [contributing](https://matrix-org.github.io/dendrite/development/contributing). 
 
 ## Is there a migration path from Synapse to Dendrite?
 
@@ -43,6 +49,20 @@ It should do, although we are aware of some minor issues:
 * **Element Android**: registration does not work, but logging in with an existing account does
 * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
 
+## Does Dendrite support Space Summaries?
+
+Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration:
+
+```
+mscs:
+  mscs:
+    - msc2946
+```
+
+Similarly, [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported.
+
+Please note that MSCs should be considered experimental and can result in significant usability issues when enabled. If you'd like more details on how MSCs are ratified or the current status of MSCs, please see the [Matrix specification documentation](https://spec.matrix.org/proposals/) on the subject.
+
 ## Does Dendrite support push notifications?
 
 Yes, we have experimental support for push notifications. Configure them in the usual way in your Matrix client.
@@ -86,6 +106,10 @@ would be a huge help too, as that will help us to understand where the memory us
 
 You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!
 
+## VOIP and Video Calls don't appear to work on Dendrite
+
+There is likely an issue with your STUN/TURN configuration on the server. If you believe your configuration to be correct, please see the [troubleshooting](administration/5_troubleshooting.md) for troubleshooting recommendations.
+
 ## What is being reported when enabling phone-home statistics?
 
 Phone-home statistics contain your server's domain name, some configuration information about
diff --git a/docs/administration/5_troubleshooting.md b/docs/administration/5_troubleshooting.md
index 14df2e3fb..8ba510ef6 100644
--- a/docs/administration/5_troubleshooting.md
+++ b/docs/administration/5_troubleshooting.md
@@ -77,5 +77,12 @@ If there aren't, you will see a log lines like this:
 level=warning msg="IMPORTANT: Process file descriptor limit is currently 65535, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues"
 ```
 
-Follow the [Optimisation](../installation/10_optimisation.md) instructions to correct the
+Follow the [Optimisation](../installation/11_optimisation.md) instructions to correct the
 available number of file descriptors.
+
+## 6. STUN/TURN Server tester
+
+If you are experiencing problems with VoIP or video calls, you should check that Dendrite
+is able to successfully connect your TURN server using 
+[Matrix VoIP Tester](https://test.voip.librepush.net/). This can highlight any issues
+that the server may encounter so that you can begin the troubleshooting process.

From da7d209b10c01ea3762bf5e524dcc17549de934d Mon Sep 17 00:00:00 2001
From: timaeos <104638316+timaeos@users.noreply.github.com>
Date: Sat, 13 Aug 2022 13:25:39 -0500
Subject: [PATCH 10/19] Admin endpoint documentation (#2640)

* Adding documentation for administration end points that aren't currently documented

* Additional information on using the administration API

* Fix inaccurate information on server notice documentation

* Fix typo in the curl command

* Add details about using the whois endpoint

* fix formatting in documentation for admin whois endpoint

* Fix more documentation formatting

* Additional formatting updates to match other formats

* Add a link to the server notice information in the Matrix Spec

* Fix broken link on server notices

* Adding access token information and admin info

* adding a warning regarding access_token

Co-authored-by: Timothy Arnold <Yourmove@gmail.com>
---
 docs/administration/4_adminapi.md | 46 +++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md
index 783fee95a..a34bfde1f 100644
--- a/docs/administration/4_adminapi.md
+++ b/docs/administration/4_adminapi.md
@@ -13,6 +13,25 @@ without warning.
 
 More endpoints will be added in the future.
 
+Endpoints may be used directly through curl:
+
+```
+curl --header "Authorization: Bearer <access_token>" -X <POST|GET|PUT> <Endpoint URI> -d '<Request Body Contents>'
+```
+
+An `access_token` can be obtained through most Element-based matrix clients by going to `Settings` -> `Help & About` -> `Advanced` -> `Access Token`.
+Be aware that an `access_token` allows a client to perform actions as an user and should be kept **secret**.
+
+The user must be an administrator in the `account_accounts` table in order to use these endpoints.
+
+Existing user accounts can be set to administrative accounts by changing `account_type` to `3` in `account_accounts`
+
+```
+UPDATE account_accounts SET account_type = 3 WHERE localpart = '$localpart';
+```
+
+Where `$localpart` is the username only (e.g. `alice`).
+
 ## GET `/_dendrite/admin/evacuateRoom/{roomID}`
 
 This endpoint will instruct Dendrite to part all local users from the given `roomID`
@@ -38,7 +57,34 @@ Request body format:
 Reset the password of a local user. The `localpart` is the username only, i.e. if
 the full user ID is `@alice:domain.com` then the local part is `alice`.
 
+## POST `/_synapse/admin/v1/send_server_notice`
+
+Request body format:
+```
+{
+    "user_id": "@target_user:server_name",
+    "content": {
+       "msgtype": "m.text",
+       "body": "This is my message"
+    }
+}
+```
+
+Send a server notice to a specific user. See the [Matrix Spec](https://spec.matrix.org/v1.3/client-server-api/#server-notices) for additional details on server notice behaviour.
+If successfully sent, the API will return the following response:
+
+```
+{
+     "event_id": "<event_id>"
+}
+```
+
 ## GET `/_synapse/admin/v1/register`
 
 Shared secret registration — please see the [user creation page](createusers) for
 guidance on configuring and using this endpoint.
+
+## GET `/_matrix/client/v3/admin/whois/{userId}`
+
+From the [Matrix Spec](https://spec.matrix.org/v1.3/client-server-api/#get_matrixclientv3adminwhoisuserid). 
+Gets information about a particular user. `userId` is the full user ID (e.g. `@alice:domain.com`)

From 65b7db633c07311fea109ca045fcdb20c87d22a3 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Sat, 13 Aug 2022 20:31:43 +0200
Subject: [PATCH 11/19] Fix FAQ question

---
 docs/FAQ.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/FAQ.md b/docs/FAQ.md
index 85ad83db5..ca72b151d 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -14,7 +14,7 @@ Mostly, although there are still bugs and missing features. If you are a confide
 
 No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](../README.md) at the root of the repository for more information.
 
-## Why doesn't Dendrite have <blank> yet?
+## Why doesn't Dendrite have "x" yet?
 
 Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be 
 specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting 

From bcdbd5c00a757971f753622b5a9f78347d5160d4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 15 Aug 2022 10:56:54 +0100
Subject: [PATCH 12/19] Version 0.9.3

---
 CHANGES.md          | 10 ++++++++++
 go.mod              |  2 +-
 go.sum              |  4 ++--
 internal/version.go |  2 +-
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 041816ae3..36a1a6311 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,15 @@
 # Changelog
 
+## Dendrite 0.9.3 (2022-08-15)
+
+### Important
+
+* This is a **security release** to fix a vulnerability within event auth, affecting all versions of Dendrite before 0.9.3. Upgrading to this version is highly recommended. For more information, [see here](https://github.com/matrix-org/gomatrixserverlib/security/advisories/GHSA-grvv-h2f9-7v9c).
+
+### Fixes
+
+* Dendrite will now correctly parse the `"events_default"` power level value for event auth.
+
 ## Dendrite 0.9.2 (2022-08-12)
 
 ### Features
diff --git a/go.mod b/go.mod
index 108165ba2..fe4604a03 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220801083850-5ff38e2c2839
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220815094957-74b7ff4ae09c
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index e925d6415..7ac9fc6e2 100644
--- a/go.sum
+++ b/go.sum
@@ -343,8 +343,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220801083850-5ff38e2c2839 h1:QEFxKWH8PlEt3ZQKl31yJNAm8lvpNUwT51IMNTl9v1k=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220801083850-5ff38e2c2839/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220815094957-74b7ff4ae09c h1:GhKmb8s9iXA9qsFD1SbiRo6Ee7cnbfcgJQ/iy43wczM=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220815094957-74b7ff4ae09c/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/internal/version.go b/internal/version.go
index 18db2dcc8..561e6c06f 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 2
+	VersionPatch = 3
 	VersionTag   = "" // example: "rc1"
 )
 

From 0642ffc0f6c6d21af0e4a17acd84200e35bf279a Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 16 Aug 2022 10:29:36 +0200
Subject: [PATCH 13/19] Only return non-retired invites (#2643)

* Only return non-retired invites

* Revert "Only return non-retired invites"

This reverts commit 1150aa7f385b7d7cf5378297f3e17566d5aabcc6.

* Check if we're doing an initial sync in the stream
---
 syncapi/streams/stream_invite.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index ddac9be2c..925da32f2 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -7,8 +7,9 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 type InviteStreamProvider struct {
@@ -62,6 +63,11 @@ func (p *InviteStreamProvider) IncrementalSync(
 		req.Response.Rooms.Invite[roomID] = *ir
 	}
 
+	// When doing an initial sync, we don't want to add retired invites, as this
+	// can add rooms we were invited to, but already left.
+	if from == 0 {
+		return to
+	}
 	for roomID := range retiredInvites {
 		if _, ok := req.Response.Rooms.Join[roomID]; !ok {
 			lr := types.NewLeaveResponse()

From 5424b88f3061dddbeae53df8b67bd064e8cbc400 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 16 Aug 2022 11:55:06 +0100
Subject: [PATCH 14/19] Use `is_direct` flag from `/createRoom`, update
 stripped state (#2644)

* Use `is_direct` flag from `/createRoom`, update stripped state

* Add comment
---
 clientapi/routing/createroom.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index 874908639..3e837c864 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -49,6 +49,7 @@ type createRoomRequest struct {
 	GuestCanJoin              bool                          `json:"guest_can_join"`
 	RoomVersion               gomatrixserverlib.RoomVersion `json:"room_version"`
 	PowerLevelContentOverride json.RawMessage               `json:"power_level_content_override"`
+	IsDirect                  bool                          `json:"is_direct"`
 }
 
 const (
@@ -499,9 +500,17 @@ func createRoom(
 		// Build some stripped state for the invite.
 		var globalStrippedState []gomatrixserverlib.InviteV2StrippedState
 		for _, event := range builtEvents {
+			// Chosen events from the spec:
+			// https://spec.matrix.org/v1.3/client-server-api/#stripped-state
 			switch event.Type() {
+			case gomatrixserverlib.MRoomCreate:
+				fallthrough
 			case gomatrixserverlib.MRoomName:
 				fallthrough
+			case gomatrixserverlib.MRoomAvatar:
+				fallthrough
+			case gomatrixserverlib.MRoomTopic:
+				fallthrough
 			case gomatrixserverlib.MRoomCanonicalAlias:
 				fallthrough
 			case gomatrixserverlib.MRoomEncryption:
@@ -522,7 +531,7 @@ func createRoom(
 			// Build the invite event.
 			inviteEvent, err := buildMembershipEvent(
 				ctx, invitee, "", profileAPI, device, gomatrixserverlib.Invite,
-				roomID, true, cfg, evTime, rsAPI, asAPI,
+				roomID, r.IsDirect, cfg, evTime, rsAPI, asAPI,
 			)
 			if err != nil {
 				util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")

From 804653e55118a8155dfbfe5187e941a2f8ce8337 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 16 Aug 2022 13:21:22 +0200
Subject: [PATCH 15/19] Verify a shared secret is set in `create-account`
 (#2645)

---
 cmd/create-account/main.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index fbd14b8d7..bd053f2f7 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -85,6 +85,10 @@ func main() {
 		logrus.Fatalf("The reset-password flag has been replaced by the POST /_dendrite/admin/resetPassword/{localpart} admin API.")
 	}
 
+	if cfg.ClientAPI.RegistrationSharedSecret == "" {
+		logrus.Fatalln("Shared secret registration is not enabled, enable it by setting a shared secret in the config: 'client_api.registration_shared_secret'")
+	}
+
 	if *username == "" {
 		flag.Usage()
 		os.Exit(1)

From ec16c944eb646c6ef61f0b7783f2f3869b9bc10c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 16 Aug 2022 14:42:00 +0100
Subject: [PATCH 16/19] Lazy-loading fixes (#2646)

* Use existing current room state if we have it

* Don't dedupe before applying the history vis filter

* Revert "Don't dedupe before applying the history vis filter"

This reverts commit d27c4a0874dabb77c2eda6b23eb7c00478bc9e90.

* Revert "Use existing current room state if we have it"

This reverts commit 5819b4a7ce511204c4fb48d3c4741612b136e2ea.

* Tweaks
---
 syncapi/streams/stream_pdu.go | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 136cbea5a..0e9dda577 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -560,14 +560,13 @@ func (p *PDUStreamProvider) lazyLoadMembers(
 			// If this is a gapped incremental sync, we still want this membership
 			isGappedIncremental := limited && incremental
 			// We want this users membership event, keep it in the list
-			_, ok := timelineUsers[event.Sender()]
-			wantMembership := ok || isGappedIncremental
-			if wantMembership {
+			stateKey := *event.StateKey()
+			if _, ok := timelineUsers[stateKey]; ok || isGappedIncremental {
 				newStateEvents = append(newStateEvents, event)
 				if !includeRedundant {
-					p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, event.Sender(), event.EventID())
+					p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, stateKey, event.EventID())
 				}
-				delete(timelineUsers, event.Sender())
+				delete(timelineUsers, stateKey)
 			}
 		} else {
 			newStateEvents = append(newStateEvents, event)
@@ -578,17 +577,16 @@ func (p *PDUStreamProvider) lazyLoadMembers(
 		wantUsers = append(wantUsers, userID)
 	}
 	// Query missing membership events
-	memberships, err := p.DB.GetStateEventsForRoom(ctx, roomID, &gomatrixserverlib.StateFilter{
-		Limit:   100,
-		Senders: &wantUsers,
-		Types:   &[]string{gomatrixserverlib.MRoomMember},
-	})
+	filter := gomatrixserverlib.DefaultStateFilter()
+	filter.Senders = &wantUsers
+	filter.Types = &[]string{gomatrixserverlib.MRoomMember}
+	memberships, err := p.DB.GetStateEventsForRoom(ctx, roomID, &filter)
 	if err != nil {
 		return stateEvents, err
 	}
 	// cache the membership events
 	for _, membership := range memberships {
-		p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, membership.Sender(), membership.EventID())
+		p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, *membership.StateKey(), membership.EventID())
 	}
 	stateEvents = append(newStateEvents, memberships...)
 	return stateEvents, nil

From ad4ac2c016d2916c51695fd11d3a7b52e38d0ca2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 16 Aug 2022 14:42:35 +0100
Subject: [PATCH 17/19] Stop spamming the logs with `StateBetween: ignoring
 deleted state` event IDs

---
 syncapi/storage/postgres/output_room_events_table.go | 4 ++--
 syncapi/storage/sqlite3/output_room_events_table.go  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index 8f633640e..9c758cac1 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -279,8 +279,8 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 			log.WithFields(log.Fields{
 				"since":   r.From,
 				"current": r.To,
-				"adds":    addIDs,
-				"dels":    delIDs,
+				"adds":    len(addIDs),
+				"dels":    len(delIDs),
 			}).Warn("StateBetween: ignoring deleted state")
 		}
 
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 91fd35b5b..cd9a46d4c 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -234,8 +234,8 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 			log.WithFields(log.Fields{
 				"since":   r.From,
 				"current": r.To,
-				"adds":    addIDsJSON,
-				"dels":    delIDsJSON,
+				"adds":    len(addIDsJSON),
+				"dels":    len(delIDsJSON),
 			}).Warn("StateBetween: ignoring deleted state")
 		}
 

From 8d9c8f11c5c5673d48b8e5308e1f565927bdd341 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 18 Aug 2022 08:56:57 +0200
Subject: [PATCH 18/19] Add a delay after sending events to the roomserver

---
 syncapi/syncapi_test.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index dc073a16e..089bdafaf 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -10,6 +10,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/tidwall/gjson"
+
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver"
@@ -21,9 +25,6 @@ import (
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	"github.com/tidwall/gjson"
 )
 
 type syncRoomserverAPI struct {
@@ -422,6 +423,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) {
 				if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", nil, false); err != nil {
 					t.Fatalf("failed to send events: %v", err)
 				}
+				time.Sleep(100 * time.Millisecond) // TODO: find a better way
 
 				// There is only one event, we expect only to be able to see this, if the room is world_readable
 				w := httptest.NewRecorder()
@@ -454,6 +456,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) {
 				if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", nil, false); err != nil {
 					t.Fatalf("failed to send events: %v", err)
 				}
+				time.Sleep(100 * time.Millisecond) // TODO: find a better way
 
 				// Verify the messages after/before invite are visible or not
 				w = httptest.NewRecorder()

From 59bc0a6f4ed0324da0387118e1761b4551aaf103 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 18 Aug 2022 10:37:47 +0100
Subject: [PATCH 19/19] Reprocess rejected input events (#2647)

* Reprocess outliers that were previously rejected

* Might as well do all events this way

* More useful errors

* Fix queries

* Tweak condition

* Don't wrap errors

* Report more useful error

* Flatten error on `r.Queryer.QueryStateAfterEvents`

* Some more debug logging

* Flatten error in `QueryRestrictedJoinAllowed`

* Revert "Flatten error in `QueryRestrictedJoinAllowed`"

This reverts commit 1238b4184c30e0c31ffb0f364806fa1275aba483.

* Tweak `QueryStateAfterEvents`

* Handle MissingStateError too

* Scope to room

* Clean up

* Fix the error

* Only apply rejection check to outliers
---
 roomserver/internal/input/input_events.go   | 47 +++++++++++----------
 roomserver/internal/query/query.go          | 13 +++---
 roomserver/storage/interface.go             |  2 +
 roomserver/storage/postgres/events_table.go | 13 ++++++
 roomserver/storage/shared/storage.go        |  4 ++
 roomserver/storage/sqlite3/events_table.go  | 13 ++++++
 roomserver/storage/tables/interface.go      |  1 +
 7 files changed, 65 insertions(+), 28 deletions(-)

diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 81541260c..53ccd5973 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -17,8 +17,8 @@
 package input
 
 import (
-	"bytes"
 	"context"
+	"database/sql"
 	"fmt"
 	"time"
 
@@ -107,28 +107,6 @@ func (r *Inputer) processRoomEvent(
 		})
 	}
 
-	// if we have already got this event then do not process it again, if the input kind is an outlier.
-	// Outliers contain no extra information which may warrant a re-processing.
-	if input.Kind == api.KindOutlier {
-		evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
-		if err2 == nil && len(evs) == 1 {
-			// check hash matches if we're on early room versions where the event ID was a random string
-			idFormat, err2 := headered.RoomVersion.EventIDFormat()
-			if err2 == nil {
-				switch idFormat {
-				case gomatrixserverlib.EventIDFormatV1:
-					if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {
-						logger.Debugf("Already processed event; ignoring")
-						return nil
-					}
-				default:
-					logger.Debugf("Already processed event; ignoring")
-					return nil
-				}
-			}
-		}
-	}
-
 	// Don't waste time processing the event if the room doesn't exist.
 	// A room entry locally will only be created in response to a create
 	// event.
@@ -141,6 +119,29 @@ func (r *Inputer) processRoomEvent(
 		return fmt.Errorf("room %s does not exist for event %s", event.RoomID(), event.EventID())
 	}
 
+	// If we already know about this outlier and it hasn't been rejected
+	// then we won't attempt to reprocess it. If it was rejected or has now
+	// arrived as a different kind of event, then we can attempt to reprocess,
+	// in case we have learned something new or need to weave the event into
+	// the DAG now.
+	if input.Kind == api.KindOutlier && roomInfo != nil {
+		wasRejected, werr := r.DB.IsEventRejected(ctx, roomInfo.RoomNID, event.EventID())
+		switch {
+		case werr == sql.ErrNoRows:
+			// We haven't seen this event before so continue.
+		case werr != nil:
+			// Something has gone wrong trying to find out if we rejected
+			// this event already.
+			logger.WithError(werr).Errorf("Failed to check if event %q is already seen", event.EventID())
+			return werr
+		case !wasRejected:
+			// We've seen this event before and it wasn't rejected so we
+			// should ignore it.
+			logger.Debugf("Already processed event %q, ignoring", event.EventID())
+			return nil
+		}
+	}
+
 	var missingAuth, missingPrev bool
 	serverRes := &fedapi.QueryJoinedHostServerNamesInRoomResponse{}
 	if !isCreateEvent {
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index f3061c222..f5d8c2d49 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -72,13 +72,10 @@ func (r *Queryer) QueryStateAfterEvents(
 
 	prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs)
 	if err != nil {
-		switch err.(type) {
-		case types.MissingEventError:
-			util.GetLogger(ctx).Errorf("QueryStateAfterEvents: MissingEventError: %s", err)
+		if _, ok := err.(types.MissingEventError); ok {
 			return nil
-		default:
-			return err
 		}
+		return err
 	}
 	response.PrevEventsExist = true
 
@@ -95,6 +92,12 @@ func (r *Queryer) QueryStateAfterEvents(
 		)
 	}
 	if err != nil {
+		if _, ok := err.(types.MissingEventError); ok {
+			return nil
+		}
+		if _, ok := err.(types.MissingStateError); ok {
+			return nil
+		}
 		return err
 	}
 
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index b12025c41..5c068873a 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -94,6 +94,8 @@ type Database interface {
 	// Opens and returns a room updater, which locks the room and opens a transaction.
 	// The GetRoomUpdater must have Commit or Rollback called on it if this doesn't return an error.
 	// If this returns an error then no further action is required.
+	// IsEventRejected returns true if the event is known and rejected.
+	IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (rejected bool, err error)
 	GetRoomUpdater(ctx context.Context, roomInfo *types.RoomInfo) (*shared.RoomUpdater, error)
 	// Look up event references for the latest events in the room and the current state snapshot.
 	// Returns the latest events, the current state and the maximum depth of the latest events plus 1.
diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go
index a4d05756d..c7748d2be 100644
--- a/roomserver/storage/postgres/events_table.go
+++ b/roomserver/storage/postgres/events_table.go
@@ -136,6 +136,9 @@ const selectMaxEventDepthSQL = "" +
 const selectRoomNIDsForEventNIDsSQL = "" +
 	"SELECT event_nid, room_nid FROM roomserver_events WHERE event_nid = ANY($1)"
 
+const selectEventRejectedSQL = "" +
+	"SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2"
+
 type eventStatements struct {
 	insertEventStmt                        *sql.Stmt
 	selectEventStmt                        *sql.Stmt
@@ -153,6 +156,7 @@ type eventStatements struct {
 	bulkSelectUnsentEventNIDStmt           *sql.Stmt
 	selectMaxEventDepthStmt                *sql.Stmt
 	selectRoomNIDsForEventNIDsStmt         *sql.Stmt
+	selectEventRejectedStmt                *sql.Stmt
 }
 
 func CreateEventsTable(db *sql.DB) error {
@@ -180,6 +184,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 		{&s.bulkSelectUnsentEventNIDStmt, bulkSelectUnsentEventNIDSQL},
 		{&s.selectMaxEventDepthStmt, selectMaxEventDepthSQL},
 		{&s.selectRoomNIDsForEventNIDsStmt, selectRoomNIDsForEventNIDsSQL},
+		{&s.selectEventRejectedStmt, selectEventRejectedSQL},
 	}.Prepare(db)
 }
 
@@ -540,3 +545,11 @@ func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array {
 	}
 	return nids
 }
+
+func (s *eventStatements) SelectEventRejected(
+	ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventID string,
+) (rejected bool, err error) {
+	stmt := sqlutil.TxStmt(txn, s.selectEventRejectedStmt)
+	err = stmt.QueryRowContext(ctx, roomNID, eventID).Scan(&rejected)
+	return
+}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index cbf9c8b20..4f92adf1f 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -567,6 +567,10 @@ func (d *Database) GetRoomUpdater(
 	return updater, err
 }
 
+func (d *Database) IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error) {
+	return d.EventsTable.SelectEventRejected(ctx, nil, roomNID, eventID)
+}
+
 func (d *Database) StoreEvent(
 	ctx context.Context, event *gomatrixserverlib.Event,
 	authEventNIDs []types.EventNID, isRejected bool,
diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go
index 1dda34c36..174e3a9a7 100644
--- a/roomserver/storage/sqlite3/events_table.go
+++ b/roomserver/storage/sqlite3/events_table.go
@@ -109,6 +109,9 @@ const selectMaxEventDepthSQL = "" +
 const selectRoomNIDsForEventNIDsSQL = "" +
 	"SELECT event_nid, room_nid FROM roomserver_events WHERE event_nid IN ($1)"
 
+const selectEventRejectedSQL = "" +
+	"SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2"
+
 type eventStatements struct {
 	db                                     *sql.DB
 	insertEventStmt                        *sql.Stmt
@@ -122,6 +125,7 @@ type eventStatements struct {
 	bulkSelectStateAtEventAndReferenceStmt *sql.Stmt
 	bulkSelectEventReferenceStmt           *sql.Stmt
 	bulkSelectEventIDStmt                  *sql.Stmt
+	selectEventRejectedStmt                *sql.Stmt
 	//bulkSelectEventNIDStmt               *sql.Stmt
 	//bulkSelectUnsentEventNIDStmt         *sql.Stmt
 	//selectRoomNIDsForEventNIDsStmt       *sql.Stmt
@@ -152,6 +156,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 		//{&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL},
 		//{&s.bulkSelectUnsentEventNIDStmt, bulkSelectUnsentEventNIDSQL},
 		//{&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL},
+		{&s.selectEventRejectedStmt, selectEventRejectedSQL},
 	}.Prepare(db)
 }
 
@@ -614,3 +619,11 @@ func eventNIDsAsArray(eventNIDs []types.EventNID) string {
 	b, _ := json.Marshal(eventNIDs)
 	return string(b)
 }
+
+func (s *eventStatements) SelectEventRejected(
+	ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventID string,
+) (rejected bool, err error) {
+	stmt := sqlutil.TxStmt(txn, s.selectEventRejectedStmt)
+	err = stmt.QueryRowContext(ctx, roomNID, eventID).Scan(&rejected)
+	return
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 0bc389b80..ed67c43d8 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -66,6 +66,7 @@ type Events interface {
 	BulkSelectUnsentEventNID(ctx context.Context, txn *sql.Tx, eventIDs []string) (map[string]types.EventNID, error)
 	SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error)
 	SelectRoomNIDsForEventNIDs(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (roomNIDs map[types.EventNID]types.RoomNID, err error)
+	SelectEventRejected(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventID string) (rejected bool, err error)
 }
 
 type Rooms interface {