Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/fts

CHANGES.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## Dendrite 0.8.6 (2022-05-26)
+
+### Features
+
+* Room versions 8 and 9 are now marked as stable
+* Dendrite can now assist remote users to join restricted rooms via `/make_join` and `/send_join`
+
+### Fixes
+
+* The sync API no longer returns immediately on `/sync` requests unnecessarily if it can be avoided
+* A race condition has been fixed in the sync API when updating presence via `/sync`
+* A race condition has been fixed sending E2EE keys to remote servers over federation when joining rooms
+* The `trusted_private_chat` preset should now grant power level 100 to all participant users, which should improve the user experience of direct messages
+* Invited users are now authed correctly in restricted rooms
+* The `join_authorised_by_users_server` key is now correctly stripped in restricted rooms when updating the membership event
+* Appservices should now receive invite events correctly
+* Device list updates should no longer contain optional fields with `null` values
+* The `/deactivate` endpoint has been fixed to no longer confuse Element with incorrect completed flows
+
 ## Dendrite 0.8.5 (2022-05-13)
 
 ### Features

README.md

@@ -96,10 +96,9 @@ than features that massive deployments may be interested in (User Directory, Ope
 This means Dendrite supports amongst others:
 
 - Core room functionality (creating rooms, invites, auth rules)
-- Full support for room versions 1 to 7
-- Experimental support for room versions 8 to 9
+- Room versions 1 to 10 supported
 - Backfilling locally and via federation
-- Accounts, Profiles and Devices
+- Accounts, profiles and devices
 - Published room lists
 - Typing
 - Media APIs

appservice/consumers/roomserver.go

@@ -83,29 +83,38 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 		return true
 	}
 
-	if output.Type != api.OutputTypeNewRoomEvent || output.NewRoomEvent == nil {
-		return true
-	}
+	log.WithFields(log.Fields{
+		"type": output.Type,
+	}).Debug("Got a message in OutputRoomEventConsumer")
 
-	newEventID := output.NewRoomEvent.Event.EventID()
-	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(output.NewRoomEvent.AddsStateEventIDs))
-	events = append(events, output.NewRoomEvent.Event)
-	if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
-		eventsReq := &api.QueryEventsByIDRequest{
-			EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
-		}
-		eventsRes := &api.QueryEventsByIDResponse{}
-		for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
-			if eventID != newEventID {
-				eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
+	events := []*gomatrixserverlib.HeaderedEvent{}
+	if output.Type == api.OutputTypeNewRoomEvent && output.NewRoomEvent != nil {
+		newEventID := output.NewRoomEvent.Event.EventID()
+		events = append(events, output.NewRoomEvent.Event)
+		if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
+			eventsReq := &api.QueryEventsByIDRequest{
+				EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
+			}
+			eventsRes := &api.QueryEventsByIDResponse{}
+			for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
+				if eventID != newEventID {
+					eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
+				}
+			}
+			if len(eventsReq.EventIDs) > 0 {
+				if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
+					return false
+				}
+				events = append(events, eventsRes.Events...)
 			}
 		}
-		if len(eventsReq.EventIDs) > 0 {
-			if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
-				return false
-			}
-			events = append(events, eventsRes.Events...)
-		}
+	} else if output.Type == api.OutputTypeNewInviteEvent && output.NewInviteEvent != nil {
+		events = append(events, output.NewInviteEvent.Event)
+	} else {
+		log.WithFields(log.Fields{
+			"type": output.Type,
+		}).Debug("appservice OutputRoomEventConsumer ignoring event", string(msg.Data))
+		return true
 	}
 
 	// Send event to any relevant application services

clientapi/jsonerror/jsonerror.go

@@ -154,6 +154,12 @@ func MissingParam(msg string) *MatrixError {
 	return &MatrixError{"M_MISSING_PARAM", msg}
 }
 
+// UnableToAuthoriseJoin is an error that is returned when a server can't
+// determine whether to allow a restricted join or not.
+func UnableToAuthoriseJoin(msg string) *MatrixError {
+	return &MatrixError{"M_UNABLE_TO_AUTHORISE_JOIN", msg}
+}
+
 // LeaveServerNoticeError is an error returned when trying to reject an invite
 // for a server notice room.
 func LeaveServerNoticeError() *MatrixError {

clientapi/routing/createroom.go

@@ -245,7 +245,9 @@ func createRoom(
 	case presetTrustedPrivateChat:
 		joinRuleContent.JoinRule = gomatrixserverlib.Invite
 		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
-		// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
+		for _, invitee := range r.Invite {
+			powerLevelContent.Users[invitee] = 100
+		}
 	case presetPublicChat:
 		joinRuleContent.JoinRule = gomatrixserverlib.Public
 		historyVisibilityContent.HistoryVisibility = historyVisibilityShared

clientapi/routing/sendevent.go

@@ -104,6 +104,13 @@ func SendEvent(
 		return *resErr
 	}
 
+	// If we're sending a membership update, make sure to strip the authorised
+	// via key if it is present, otherwise other servers won't be able to auth
+	// the event if the room is set to the "restricted" join rule.
+	if eventType == gomatrixserverlib.MRoomMember {
+		delete(r, "join_authorised_via_users_server")
+	}
+
 	evTime, err := httputil.ParseTSParam(req)
 	if err != nil {
 		return util.JSONResponse{

cmd/resolve-state/main.go

@@ -4,13 +4,17 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"os"
+	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -23,11 +27,17 @@ import (
 //   e.g. ./resolve-state --roomversion=5 1254 1235 1282
 
 var roomVersion = flag.String("roomversion", "5", "the room version to parse events as")
+var filterType = flag.String("filtertype", "", "the event types to filter on")
 
 func main() {
 	ctx := context.Background()
 	cfg := setup.ParseFlags(true)
-	args := os.Args[1:]
+	cfg.Logging = append(cfg.Logging[:0], config.LogrusHook{
+		Type:  "std",
+		Level: "error",
+	})
+	base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics)
+	args := flag.Args()
 
 	fmt.Println("Room version", *roomVersion)
 
@@ -45,30 +55,28 @@ func main() {
 		panic(err)
 	}
 
-	roomserverDB, err := storage.Open(nil, &cfg.RoomServer.Database, cache)
+	roomserverDB, err := storage.Open(base, &cfg.RoomServer.Database, cache)
 	if err != nil {
 		panic(err)
 	}
 
-	blockNIDs, err := roomserverDB.StateBlockNIDs(ctx, snapshotNIDs)
-	if err != nil {
-		panic(err)
-	}
+	stateres := state.NewStateResolution(roomserverDB, &types.RoomInfo{
+		RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion),
+	})
 
-	var stateEntries []types.StateEntryList
-	for _, list := range blockNIDs {
-		entries, err2 := roomserverDB.StateEntries(ctx, list.StateBlockNIDs)
-		if err2 != nil {
-			panic(err2)
+	var stateEntries []types.StateEntry
+	for _, snapshotNID := range snapshotNIDs {
+		var entries []types.StateEntry
+		entries, err = stateres.LoadStateAtSnapshot(ctx, snapshotNID)
+		if err != nil {
+			panic(err)
 		}
 		stateEntries = append(stateEntries, entries...)
 	}
 
 	var eventNIDs []types.EventNID
 	for _, entry := range stateEntries {
-		for _, e := range entry.StateEntries {
-			eventNIDs = append(eventNIDs, e.EventNID)
-		}
+		eventNIDs = append(eventNIDs, entry.EventNID)
 	}
 
 	fmt.Println("Fetching", len(eventNIDs), "state events")
@@ -103,7 +111,8 @@ func main() {
 	}
 
 	fmt.Println("Resolving state")
-	resolved, err := gomatrixserverlib.ResolveConflicts(
+	var resolved Events
+	resolved, err = gomatrixserverlib.ResolveConflicts(
 		gomatrixserverlib.RoomVersion(*roomVersion),
 		events,
 		authEvents,
@@ -113,9 +122,41 @@ func main() {
 	}
 
 	fmt.Println("Resolved state contains", len(resolved), "events")
+	sort.Sort(resolved)
+	filteringEventType := *filterType
+	count := 0
 	for _, event := range resolved {
+		if filteringEventType != "" && event.Type() != filteringEventType {
+			continue
+		}
+		count++
 		fmt.Println()
 		fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey())
 		fmt.Printf("  %s\n", string(event.Content()))
 	}
+
+	fmt.Println()
+	fmt.Println("Returned", count, "state events after filtering")
+}
+
+type Events []*gomatrixserverlib.Event
+
+func (e Events) Len() int {
+	return len(e)
+}
+
+func (e Events) Swap(i, j int) {
+	e[i], e[j] = e[j], e[i]
+}
+
+func (e Events) Less(i, j int) bool {
+	typeDelta := strings.Compare(e[i].Type(), e[j].Type())
+	if typeDelta < 0 {
+		return true
+	}
+	if typeDelta > 0 {
+		return false
+	}
+	stateKeyDelta := strings.Compare(*e[i].StateKey(), *e[j].StateKey())
+	return stateKeyDelta < 0
 }

docs/installation/1_planning.md

@@ -9,21 +9,19 @@ permalink: /installation/planning
 
 ## Modes
 
-Dendrite can be run in one of two configurations:
+Dendrite consists of several components, each responsible for a different aspect of the Matrix protocol.
+Users can run Dendrite in one of two modes which dictate how these components are executed and communicate.
 
-* **Monolith mode**: All components run in the same process. In this mode,
-   it is possible to run an in-process NATS Server instead of running a standalone deployment.
-   This will usually be the preferred model for low-to-mid volume deployments, providing the best
-   balance between performance and resource usage.
+* **Monolith mode** runs all components in a single process. Components communicate through an internal NATS
+  server with generally low overhead. This mode dramatically simplifies deployment complexity and offers the
+  best balance between performance and resource usage for low-to-mid volume deployments.
 
-* **Polylith mode**: A cluster of individual components running in their own processes, dealing
-  with different aspects of the Matrix protocol. Components communicate with each other using
-  internal HTTP APIs and NATS Server. This will almost certainly be the preferred model for very
-  large deployments but scalability comes with a cost. API calls are expensive and therefore a
-  polylith deployment may end up using disproportionately more resources for a smaller number of
-  users compared to a monolith deployment.
+* **Polylith mode** runs all components in isolated processes. Components communicate through an external NATS
+  server and HTTP APIs, which incur considerable overhead. While this mode allows for more granular control of
+  resources dedicated toward individual processes, given the additional communications overhead, it is only
+  necessary for very large deployments.
 
-At present, we **recommend monolith mode deployments** in all cases.
+Given our current state of development, **we recommend monolith mode** for all deployments.
 
 ## Databases
 

federationapi/federationapi_keys_test.go

@@ -55,60 +55,62 @@ var servers = map[string]*server{
 func TestMain(m *testing.M) {
 	// Set up the server key API for each "server" that we
 	// will use in our tests.
-	for _, s := range servers {
-		// Generate a new key.
-		_, testPriv, err := ed25519.GenerateKey(nil)
-		if err != nil {
-			panic("can't generate identity key: " + err.Error())
+	os.Exit(func() int {
+		for _, s := range servers {
+			// Generate a new key.
+			_, testPriv, err := ed25519.GenerateKey(nil)
+			if err != nil {
+				panic("can't generate identity key: " + err.Error())
+			}
+
+			// Create a new cache but don't enable prometheus!
+			s.cache, err = caching.NewInMemoryLRUCache(false)
+			if err != nil {
+				panic("can't create cache: " + err.Error())
+			}
+
+			// Create a temporary directory for JetStream.
+			d, err := ioutil.TempDir("./", "jetstream*")
+			if err != nil {
+				panic(err)
+			}
+			defer os.RemoveAll(d)
+
+			// Draw up just enough Dendrite config for the server key
+			// API to work.
+			cfg := &config.Dendrite{}
+			cfg.Defaults(true)
+			cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
+			cfg.Global.PrivateKey = testPriv
+			cfg.Global.JetStream.InMemory = true
+			cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
+			cfg.Global.JetStream.StoragePath = config.Path(d)
+			cfg.Global.KeyID = serverKeyID
+			cfg.Global.KeyValidityPeriod = s.validity
+			cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
+			s.config = &cfg.FederationAPI
+
+			// Create a transport which redirects federation requests to
+			// the mock round tripper. Since we're not *really* listening for
+			// federation requests then this will return the key instead.
+			transport := &http.Transport{}
+			transport.RegisterProtocol("matrix", &MockRoundTripper{})
+
+			// Create the federation client.
+			s.fedclient = gomatrixserverlib.NewFederationClient(
+				s.config.Matrix.ServerName, serverKeyID, testPriv,
+				gomatrixserverlib.WithTransport(transport),
+			)
+
+			// Finally, build the server key APIs.
+			sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
+			s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
 		}
 
-		// Create a new cache but don't enable prometheus!
-		s.cache, err = caching.NewInMemoryLRUCache(false)
-		if err != nil {
-			panic("can't create cache: " + err.Error())
-		}
-
-		// Create a temporary directory for JetStream.
-		d, err := ioutil.TempDir("./", "jetstream*")
-		if err != nil {
-			panic(err)
-		}
-		defer os.RemoveAll(d)
-
-		// Draw up just enough Dendrite config for the server key
-		// API to work.
-		cfg := &config.Dendrite{}
-		cfg.Defaults(true)
-		cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
-		cfg.Global.PrivateKey = testPriv
-		cfg.Global.JetStream.InMemory = true
-		cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
-		cfg.Global.JetStream.StoragePath = config.Path(d)
-		cfg.Global.KeyID = serverKeyID
-		cfg.Global.KeyValidityPeriod = s.validity
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
-		s.config = &cfg.FederationAPI
-
-		// Create a transport which redirects federation requests to
-		// the mock round tripper. Since we're not *really* listening for
-		// federation requests then this will return the key instead.
-		transport := &http.Transport{}
-		transport.RegisterProtocol("matrix", &MockRoundTripper{})
-
-		// Create the federation client.
-		s.fedclient = gomatrixserverlib.NewFederationClient(
-			s.config.Matrix.ServerName, serverKeyID, testPriv,
-			gomatrixserverlib.WithTransport(transport),
-		)
-
-		// Finally, build the server key APIs.
-		sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
-		s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
-	}
-
-	// Now that we have built our server key APIs, start the
-	// rest of the tests.
-	os.Exit(m.Run())
+		// Now that we have built our server key APIs, start the
+		// rest of the tests.
+		return m.Run()
+	}())
 }
 
 type MockRoundTripper struct{}

federationapi/internal/perform.go

@@ -166,7 +166,8 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	if content == nil {
 		content = map[string]interface{}{}
 	}
-	content["membership"] = "join"
+	_ = json.Unmarshal(respMakeJoin.JoinEvent.Content, &content)
+	content["membership"] = gomatrixserverlib.Join
 	if err = respMakeJoin.JoinEvent.SetContent(content); err != nil {
 		return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err)
 	}
@@ -209,10 +210,22 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	}
 	r.statistics.ForServer(serverName).Success()
 
-	authEvents := respSendJoin.AuthEvents.UntrustedEvents(respMakeJoin.RoomVersion)
+	// If the remote server returned an event in the "event" key of
+	// the send_join request then we should use that instead. It may
+	// contain signatures that we don't know about.
+	if len(respSendJoin.Event) > 0 {
+		var remoteEvent *gomatrixserverlib.Event
+		remoteEvent, err = respSendJoin.Event.UntrustedEvent(respMakeJoin.RoomVersion)
+		if err == nil && isWellFormedMembershipEvent(
+			remoteEvent, roomID, userID, r.cfg.Matrix.ServerName,
+		) {
+			event = remoteEvent
+		}
+	}
 
 	// Sanity-check the join response to ensure that it has a create
 	// event, that the room version is known, etc.
+	authEvents := respSendJoin.AuthEvents.UntrustedEvents(respMakeJoin.RoomVersion)
 	if err = sanityCheckAuthChain(authEvents); err != nil {
 		return fmt.Errorf("sanityCheckAuthChain: %w", err)
 	}
@@ -270,6 +283,26 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	return nil
 }
 
+// isWellFormedMembershipEvent returns true if the event looks like a legitimate
+// membership event.
+func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string, origin gomatrixserverlib.ServerName) bool {
+	if membership, err := event.Membership(); err != nil {
+		return false
+	} else if membership != gomatrixserverlib.Join {
+		return false
+	}
+	if event.RoomID() != roomID {
+		return false
+	}
+	if event.Origin() != origin {
+		return false
+	}
+	if !event.StateKeyEquals(userID) {
+		return false
+	}
+	return true
+}
+
 // PerformOutboundPeekRequest implements api.FederationInternalAPI
 func (r *FederationInternalAPI) PerformOutboundPeek(
 	ctx context.Context,

federationapi/routing/join.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"sort"
@@ -103,6 +104,16 @@ func MakeJoin(
 		}
 	}
 
+	// Check if the restricted join is allowed. If the room doesn't
+	// support restricted joins then this is effectively a no-op.
+	res, authorisedVia, err := checkRestrictedJoin(httpReq, rsAPI, verRes.RoomVersion, roomID, userID)
+	if err != nil {
+		util.GetLogger(httpReq.Context()).WithError(err).Error("checkRestrictedJoin failed")
+		return jsonerror.InternalServerError()
+	} else if res != nil {
+		return *res
+	}
+
 	// Try building an event for the server
 	builder := gomatrixserverlib.EventBuilder{
 		Sender:   userID,
@@ -110,8 +121,11 @@ func MakeJoin(
 		Type:     "m.room.member",
 		StateKey: &userID,
 	}
-	err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Join})
-	if err != nil {
+	content := gomatrixserverlib.MemberContent{
+		Membership:    gomatrixserverlib.Join,
+		AuthorisedVia: authorisedVia,
+	}
+	if err = builder.SetContent(content); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("builder.SetContent failed")
 		return jsonerror.InternalServerError()
 	}
@@ -161,6 +175,7 @@ func MakeJoin(
 // SendJoin implements the /send_join API
 // The make-join send-join dance makes much more sense as a single
 // flow so the cyclomatic complexity is high:
+// nolint:gocyclo
 func SendJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
@@ -314,6 +329,40 @@ func SendJoin(
 		}
 	}
 
+	// If the membership content contains a user ID for a server that is not
+	// ours then we should kick it back.
+	var memberContent gomatrixserverlib.MemberContent
+	if err := json.Unmarshal(event.Content(), &memberContent); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	}
+	if memberContent.AuthorisedVia != "" {
+		_, domain, err := gomatrixserverlib.SplitID('@', memberContent.AuthorisedVia)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON(fmt.Sprintf("The authorising username %q is invalid.", memberContent.AuthorisedVia)),
+			}
+		}
+		if domain != cfg.Matrix.ServerName {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON(fmt.Sprintf("The authorising username %q does not belong to this server.", memberContent.AuthorisedVia)),
+			}
+		}
+	}
+
+	// Sign the membership event. This is required for restricted joins to work
+	// in the case that the authorised via user is one of our own users. It also
+	// doesn't hurt to do it even if it isn't a restricted join.
+	signed := event.Sign(
+		string(cfg.Matrix.ServerName),
+		cfg.Matrix.KeyID,
+		cfg.Matrix.PrivateKey,
+	)
+
 	// Send the events to the room server.
 	// We are responsible for notifying other servers that the user has joined
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
@@ -323,7 +372,7 @@ func SendJoin(
 			InputRoomEvents: []api.InputRoomEvent{
 				{
 					Kind:          api.KindNew,
-					Event:         event.Headered(stateAndAuthChainResponse.RoomVersion),
+					Event:         signed.Headered(stateAndAuthChainResponse.RoomVersion),
 					SendAsServer:  string(cfg.Matrix.ServerName),
 					TransactionID: nil,
 				},
@@ -354,10 +403,77 @@ func SendJoin(
 			StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.StateEvents),
 			AuthEvents:  gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.AuthChainEvents),
 			Origin:      cfg.Matrix.ServerName,
+			Event:       signed.JSON(),
 		},
 	}
 }
 
+// checkRestrictedJoin finds out whether or not we can assist in processing
+// a restricted room join. If the room version does not support restricted
+// joins then this function returns with no side effects. This returns three
+// values:
+// * an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which
+//   should always be sent back to the client if one is specified
+// * a user ID of an authorising user, typically a user that has power to
+//   issue invites in the room, if one has been found
+// * an error if there was a problem finding out if this was allowable,
+//   like if the room version isn't known or a problem happened talking to
+//   the roomserver
+func checkRestrictedJoin(
+	httpReq *http.Request,
+	rsAPI api.FederationRoomserverAPI,
+	roomVersion gomatrixserverlib.RoomVersion,
+	roomID, userID string,
+) (*util.JSONResponse, string, error) {
+	if allowRestricted, err := roomVersion.MayAllowRestrictedJoinsInEventAuth(); err != nil {
+		return nil, "", err
+	} else if !allowRestricted {
+		return nil, "", nil
+	}
+	req := &api.QueryRestrictedJoinAllowedRequest{
+		RoomID: roomID,
+		UserID: userID,
+	}
+	res := &api.QueryRestrictedJoinAllowedResponse{}
+	if err := rsAPI.QueryRestrictedJoinAllowed(httpReq.Context(), req, res); err != nil {
+		return nil, "", err
+	}
+
+	switch {
+	case !res.Restricted:
+		// The join rules for the room don't restrict membership.
+		return nil, "", nil
+
+	case !res.Resident:
+		// The join rules restrict membership but our server isn't currently
+		// joined to all of the allowed rooms, so we can't actually decide
+		// whether or not to allow the user to join. This error code should
+		// tell the joining server to try joining via another resident server
+		// instead.
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnableToAuthoriseJoin("This server cannot authorise the join."),
+		}, "", nil
+
+	case !res.Allowed:
+		// The join rules restrict membership, our server is in the relevant
+		// rooms and the user wasn't joined to join any of the allowed rooms
+		// and therefore can't join this room.
+		return &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You are not joined to any matching rooms."),
+		}, "", nil
+
+	default:
+		// The join rules restrict membership, our server is in the relevant
+		// rooms and the user was allowed to join because they belong to one
+		// of the allowed rooms. We now need to pick one of our own local users
+		// from within the room to use as the authorising user ID, so that it
+		// can be referred to from within the membership content.
+		return nil, res.AuthorisedVia, nil
+	}
+}
+
 type eventsByDepth []*gomatrixserverlib.HeaderedEvent
 
 func (e eventsByDepth) Len() int {

federationapi/storage/postgres/queue_edus_table.go

@@ -36,6 +36,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
     ON federationsender_queue_edus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_nid_idx
+    ON federationsender_queue_edus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
+    ON federationsender_queue_edus (server_name);
 `
 
 const insertQueueEDUSQL = "" +

federationapi/storage/postgres/queue_json_table.go

@@ -33,6 +33,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_json (
 	-- The JSON body. Text so that we preserve UTF-8.
 	json_body TEXT NOT NULL
 );
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_json_json_nid_idx
+    ON federationsender_queue_json (json_nid);
 `
 
 const insertJSONSQL = "" +

federationapi/storage/postgres/queue_pdus_table.go

@@ -36,6 +36,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
     ON federationsender_queue_pdus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_json_nid_idx
+    ON federationsender_queue_pdus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_server_name_idx
+    ON federationsender_queue_pdus (server_name);
 `
 
 const insertQueuePDUSQL = "" +

federationapi/storage/sqlite3/queue_edus_table.go

@@ -37,6 +37,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
     ON federationsender_queue_edus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_nid_idx
+    ON federationsender_queue_edus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
+    ON federationsender_queue_edus (server_name);
 `
 
 const insertQueueEDUSQL = "" +

federationapi/storage/sqlite3/queue_json_table.go

@@ -35,6 +35,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_json (
 	-- The JSON body. Text so that we preserve UTF-8.
 	json_body TEXT NOT NULL
 );
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_json_json_nid_idx
+    ON federationsender_queue_json (json_nid);
 `
 
 const insertJSONSQL = "" +

federationapi/storage/sqlite3/queue_pdus_table.go

@@ -38,6 +38,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
     ON federationsender_queue_pdus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_json_nid_idx
+    ON federationsender_queue_pdus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_server_name_idx
+    ON federationsender_queue_pdus (server_name);
 `
 
 const insertQueuePDUSQL = "" +

go.mod

@@ -5,21 +5,25 @@ replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-serve
 replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e
 
 require (
-	github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0
+	github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98
 	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
+	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/blevesearch/bleve/v2 v2.3.2
 	github.com/codeclysm/extract v2.2.0+incompatible
-	github.com/containerd/containerd v1.6.2 // indirect
-	github.com/docker/docker v20.10.14+incompatible
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.16+incompatible
 	github.com/docker/go-connections v0.4.0
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/frankban/quicktest v1.14.3 // indirect
 	github.com/getsentry/sentry-go v0.13.0
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gologme/log v1.3.0
-	github.com/google/go-cmp v0.5.7
+	github.com/google/go-cmp v0.5.8
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
@@ -31,38 +35,47 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220530084946-3a4b148706bc
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
-	github.com/mattn/go-sqlite3 v1.14.10
-	github.com/miekg/dns v1.1.31 // indirect
+	github.com/mattn/go-sqlite3 v1.14.13
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/miekg/dns v1.1.49 // indirect
+	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb
 	github.com/nats-io/nats.go v1.14.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
-	github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31
-	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
+	github.com/onsi/gomega v1.17.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pressly/goose v2.7.0+incompatible
-	github.com/prometheus/client_golang v1.12.1
+	github.com/prometheus/client_golang v1.12.2
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.7.1
 	github.com/tidwall/gjson v1.14.1
 	github.com/tidwall/sjson v1.2.4
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.3
 	go.uber.org/atomic v1.9.0
-	golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122
-	golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
-	golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2
-	golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3
-	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
+	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
+	golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
+	golang.org/x/net v0.0.0-20220524220425-1d687d428aca
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0 // indirect
+	gotest.tools/v3 v3.0.3 // indirect
 	nhooyr.io/websocket v1.8.7
 )
 

internal/sqlutil/trace.go

@@ -37,7 +37,7 @@ type traceInterceptor struct {
 	sqlmw.NullInterceptor
 }
 
-func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (driver.Rows, error) {
+func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (context.Context, driver.Rows, error) {
 	startedAt := time.Now()
 	rows, err := stmt.QueryContext(ctx, args)
 
@@ -45,7 +45,7 @@ func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.St
 
 	logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args)
 
-	return rows, err
+	return ctx, rows, err
 }
 
 func (in *traceInterceptor) StmtExecContext(ctx context.Context, stmt driver.StmtExecContext, query string, args []driver.NamedValue) (driver.Result, error) {

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 5
+	VersionPatch = 6
 	VersionTag   = "" // example: "rc1"
 )
 

roomserver/api/api.go

@@ -184,6 +184,7 @@ type FederationRoomserverAPI interface {
 	// Query whether a server is allowed to see an event
 	QueryServerAllowedToSeeEvent(ctx context.Context, req *QueryServerAllowedToSeeEventRequest, res *QueryServerAllowedToSeeEventResponse) error
 	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
+	QueryRestrictedJoinAllowed(ctx context.Context, req *QueryRestrictedJoinAllowedRequest, res *QueryRestrictedJoinAllowedResponse) error
 	PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error
 	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error
 	// Query a given amount (or less) of events prior to a given set of events.

roomserver/api/api_trace.go

@@ -354,6 +354,16 @@ func (t *RoomserverInternalAPITrace) QueryAuthChain(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) QueryRestrictedJoinAllowed(
+	ctx context.Context,
+	request *QueryRestrictedJoinAllowedRequest,
+	response *QueryRestrictedJoinAllowedResponse,
+) error {
+	err := t.Impl.QueryRestrictedJoinAllowed(ctx, request, response)
+	util.GetLogger(ctx).WithError(err).Infof("QueryRestrictedJoinAllowed req=%+v res=%+v", js(request), js(response))
+	return err
+}
+
 func js(thing interface{}) string {
 	b, err := json.Marshal(thing)
 	if err != nil {

roomserver/api/query.go

@@ -348,6 +348,26 @@ type QueryServerBannedFromRoomResponse struct {
 	Banned bool `json:"banned"`
 }
 
+type QueryRestrictedJoinAllowedRequest struct {
+	UserID string `json:"user_id"`
+	RoomID string `json:"room_id"`
+}
+
+type QueryRestrictedJoinAllowedResponse struct {
+	// True if the room membership is restricted by the join rule being set to "restricted"
+	Restricted bool `json:"restricted"`
+	// True if our local server is joined to all of the allowed rooms specified in the "allow"
+	// key of the join rule, false if we are missing from some of them and therefore can't
+	// reliably decide whether or not we can satisfy the join
+	Resident bool `json:"resident"`
+	// True if the restricted join is allowed because we found the membership in one of the
+	// allowed rooms from the join rule, false if not
+	Allowed bool `json:"allowed"`
+	// Contains the user ID of the selected user ID that has power to issue invites, this will
+	// get populated into the "join_authorised_via_users_server" content in the membership
+	AuthorisedVia string `json:"authorised_via,omitempty"`
+}
+
 // MarshalJSON stringifies the room ID and StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
 func (r *QueryBulkStateContentResponse) MarshalJSON() ([]byte, error) {
 	se := make(map[string]string)

roomserver/internal/input/input_events.go

@@ -535,8 +535,6 @@ func (r *Inputer) calculateAndSetState(
 	roomState := state.NewStateResolution(updater, roomInfo)
 
 	if input.HasState {
-		stateAtEvent.Overwrite = true
-
 		// We've been told what the state at the event is so we don't need to calculate it.
 		// Check that those state events are in the database and store the state.
 		var entries []types.StateEntry
@@ -549,8 +547,6 @@ func (r *Inputer) calculateAndSetState(
 			return fmt.Errorf("updater.AddState: %w", err)
 		}
 	} else {
-		stateAtEvent.Overwrite = false
-
 		// We haven't been told what the state at the event is so we need to calculate it from the prev_events
 		if stateAtEvent.BeforeStateSnapshotNID, err = roomState.CalculateAndStoreStateBeforeEvent(ctx, event, isRejected); err != nil {
 			return fmt.Errorf("roomState.CalculateAndStoreStateBeforeEvent: %w", err)

roomserver/internal/input/input_latest_events.go

@@ -27,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // updateLatestEvents updates the list of latest events for this room in the database and writes the
@@ -101,8 +102,8 @@ type latestEventsUpdater struct {
 	// The eventID of the event that was processed before this one.
 	lastEventIDSent string
 	// The latest events in the room after processing this event.
-	oldLatest []types.StateAtEventAndReference
-	latest    []types.StateAtEventAndReference
+	oldLatest types.StateAtEventAndReferences
+	latest    types.StateAtEventAndReferences
 	// The state entries removed from and added to the current state of the
 	// room as a result of processing this event. They are sorted lists.
 	removed []types.StateEntry
@@ -125,7 +126,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// state snapshot from somewhere else, e.g. a federated room join,
 	// then start with an empty set - none of the forward extremities
 	// that we knew about before matter anymore.
-	u.oldLatest = []types.StateAtEventAndReference{}
+	u.oldLatest = types.StateAtEventAndReferences{}
 	if !u.rewritesState {
 		u.oldStateNID = u.updater.CurrentStateSnapshotNID()
 		u.oldLatest = u.updater.LatestEvents()
@@ -258,13 +259,6 @@ func (u *latestEventsUpdater) latestState() error {
 		return fmt.Errorf("roomState.CalculateAndStoreStateAfterEvents: %w", err)
 	}
 
-	// If we are overwriting the state then we should make sure that we
-	// don't send anything out over federation again, it will very likely
-	// be a repeat.
-	if u.stateAtEvent.Overwrite {
-		u.sendAsServer = ""
-	}
-
 	// Now that we have a new state snapshot based on the latest events,
 	// we can compare that new snapshot to the previous one and see what
 	// has changed. This gives us one list of removed state events and
@@ -277,6 +271,17 @@ func (u *latestEventsUpdater) latestState() error {
 		return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
 	}
 
+	if removed := len(u.removed) - len(u.added); removed > 0 {
+		logrus.WithFields(logrus.Fields{
+			"event_id":      u.event.EventID(),
+			"room_id":       u.event.RoomID(),
+			"old_state_nid": u.oldStateNID,
+			"new_state_nid": u.newStateNID,
+			"old_latest":    u.oldLatest.EventIDs(),
+			"new_latest":    u.latest.EventIDs(),
+		}).Errorf("Unexpected state deletion (removing %d events)", removed)
+	}
+
 	// Also work out the state before the event removes and the event
 	// adds.
 	u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = roomState.DifferenceBetweeenStateSnapshots(

roomserver/internal/perform/perform_join.go

@@ -24,6 +24,7 @@ import (
 	"github.com/getsentry/sentry-go"
 	fsAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	rsAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
@@ -160,6 +161,7 @@ func (r *Joiner) performJoinRoomByAlias(
 }
 
 // TODO: Break this function up a bit
+// nolint:gocyclo
 func (r *Joiner) performJoinRoomByID(
 	ctx context.Context,
 	req *rsAPI.PerformJoinRequest,
@@ -210,6 +212,11 @@ func (r *Joiner) performJoinRoomByID(
 		req.Content = map[string]interface{}{}
 	}
 	req.Content["membership"] = gomatrixserverlib.Join
+	if authorisedVia, aerr := r.populateAuthorisedViaUserForRestrictedJoin(ctx, req); aerr != nil {
+		return "", "", aerr
+	} else if authorisedVia != "" {
+		req.Content["join_authorised_via_users_server"] = authorisedVia
+	}
 	if err = eb.SetContent(req.Content); err != nil {
 		return "", "", fmt.Errorf("eb.SetContent: %w", err)
 	}
@@ -350,6 +357,33 @@ func (r *Joiner) performFederatedJoinRoomByID(
 	return fedRes.JoinedVia, nil
 }
 
+func (r *Joiner) populateAuthorisedViaUserForRestrictedJoin(
+	ctx context.Context,
+	joinReq *rsAPI.PerformJoinRequest,
+) (string, error) {
+	req := &api.QueryRestrictedJoinAllowedRequest{
+		UserID: joinReq.UserID,
+		RoomID: joinReq.RoomIDOrAlias,
+	}
+	res := &api.QueryRestrictedJoinAllowedResponse{}
+	if err := r.Queryer.QueryRestrictedJoinAllowed(ctx, req, res); err != nil {
+		return "", fmt.Errorf("r.Queryer.QueryRestrictedJoinAllowed: %w", err)
+	}
+	if !res.Restricted {
+		return "", nil
+	}
+	if !res.Resident {
+		return "", nil
+	}
+	if !res.Allowed {
+		return "", &rsAPI.PerformError{
+			Code: rsAPI.PerformErrorNotAllowed,
+			Msg:  fmt.Sprintf("The join to room %s was not allowed.", joinReq.RoomIDOrAlias),
+		}
+	}
+	return res.AuthorisedVia, nil
+}
+
 func buildEvent(
 	ctx context.Context, db storage.Database, cfg *config.Global, builder *gomatrixserverlib.EventBuilder,
 ) (*gomatrixserverlib.HeaderedEvent, *rsAPI.QueryLatestEventsAndStateResponse, error) {

roomserver/internal/query/query.go

@@ -16,6 +16,7 @@ package query
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 
@@ -757,3 +758,128 @@ func (r *Queryer) QueryAuthChain(ctx context.Context, req *api.QueryAuthChainReq
 	res.AuthChain = hchain
 	return nil
 }
+
+// nolint:gocyclo
+func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse) error {
+	// Look up if we know anything about the room. If it doesn't exist
+	// or is a stub entry then we can't do anything.
+	roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID)
+	if err != nil {
+		return fmt.Errorf("r.DB.RoomInfo: %w", err)
+	}
+	if roomInfo == nil || roomInfo.IsStub {
+		return nil // fmt.Errorf("room %q doesn't exist or is stub room", req.RoomID)
+	}
+	// If the room version doesn't allow restricted joins then don't
+	// try to process any further.
+	allowRestrictedJoins, err := roomInfo.RoomVersion.MayAllowRestrictedJoinsInEventAuth()
+	if err != nil {
+		return fmt.Errorf("roomInfo.RoomVersion.AllowRestrictedJoinsInEventAuth: %w", err)
+	} else if !allowRestrictedJoins {
+		return nil
+	}
+	// Get the join rules to work out if the join rule is "restricted".
+	joinRulesEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomJoinRules, "")
+	if err != nil {
+		return fmt.Errorf("r.DB.GetStateEvent: %w", err)
+	}
+	var joinRules gomatrixserverlib.JoinRuleContent
+	if err = json.Unmarshal(joinRulesEvent.Content(), &joinRules); err != nil {
+		return fmt.Errorf("json.Unmarshal: %w", err)
+	}
+	// If the join rule isn't "restricted" then there's nothing more to do.
+	res.Restricted = joinRules.JoinRule == gomatrixserverlib.Restricted
+	if !res.Restricted {
+		return nil
+	}
+	// Start off by populating the "resident" flag in the response. If we
+	// come across any rooms in the request that are missing, we will unset
+	// the flag.
+	res.Resident = true
+	// If the user is already invited to the room then the join is allowed
+	// but we don't specify an authorised via user, since the event auth
+	// will allow the join anyway.
+	var pending bool
+	if pending, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil {
+		return fmt.Errorf("helpers.IsInvitePending: %w", err)
+	} else if pending {
+		res.Allowed = true
+		return nil
+	}
+	// We need to get the power levels content so that we can determine which
+	// users in the room are entitled to issue invites. We need to use one of
+	// these users as the authorising user.
+	powerLevelsEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomPowerLevels, "")
+	if err != nil {
+		return fmt.Errorf("r.DB.GetStateEvent: %w", err)
+	}
+	var powerLevels gomatrixserverlib.PowerLevelContent
+	if err = json.Unmarshal(powerLevelsEvent.Content(), &powerLevels); err != nil {
+		return fmt.Errorf("json.Unmarshal: %w", err)
+	}
+	// Step through the join rules and see if the user matches any of them.
+	for _, rule := range joinRules.Allow {
+		// We only understand "m.room_membership" rules at this point in
+		// time, so skip any rule that doesn't match those.
+		if rule.Type != gomatrixserverlib.MRoomMembership {
+			continue
+		}
+		// See if the room exists. If it doesn't exist or if it's a stub
+		// room entry then we can't check memberships.
+		targetRoomInfo, err := r.DB.RoomInfo(ctx, rule.RoomID)
+		if err != nil || targetRoomInfo == nil || targetRoomInfo.IsStub {
+			res.Resident = false
+			continue
+		}
+		// First of all work out if *we* are still in the room, otherwise
+		// it's possible that the memberships will be out of date.
+		isIn, err := r.DB.GetLocalServerInRoom(ctx, targetRoomInfo.RoomNID)
+		if err != nil || !isIn {
+			// If we aren't in the room, we can no longer tell if the room
+			// memberships are up-to-date.
+			res.Resident = false
+			continue
+		}
+		// At this point we're happy that we are in the room, so now let's
+		// see if the target user is in the room.
+		_, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
+		if err != nil {
+			continue
+		}
+		// If the user is not in the room then we will skip them.
+		if !isIn {
+			continue
+		}
+		// The user is in the room, so now we will need to authorise the
+		// join using the user ID of one of our own users in the room. Pick
+		// one.
+		joinNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, targetRoomInfo.RoomNID, true, true)
+		if err != nil || len(joinNIDs) == 0 {
+			// There should always be more than one join NID at this point
+			// because we are gated behind GetLocalServerInRoom, but y'know,
+			// sometimes strange things happen.
+			continue
+		}
+		// For each of the joined users, let's see if we can get a valid
+		// membership event.
+		for _, joinNID := range joinNIDs {
+			events, err := r.DB.Events(ctx, []types.EventNID{joinNID})
+			if err != nil || len(events) != 1 {
+				continue
+			}
+			event := events[0]
+			if event.Type() != gomatrixserverlib.MRoomMember || event.StateKey() == nil {
+				continue // shouldn't happen
+			}
+			// Only users that have the power to invite should be chosen.
+			if powerLevels.UserLevel(*event.StateKey()) < powerLevels.Invite {
+				continue
+			}
+			res.Resident = true
+			res.Allowed = true
+			res.AuthorisedVia = *event.StateKey()
+			return nil
+		}
+	}
+	return nil
+}

roomserver/inthttp/client.go

@@ -61,6 +61,7 @@ const (
 	RoomserverQueryKnownUsersPath              = "/roomserver/queryKnownUsers"
 	RoomserverQueryServerBannedFromRoomPath    = "/roomserver/queryServerBannedFromRoom"
 	RoomserverQueryAuthChainPath               = "/roomserver/queryAuthChain"
+	RoomserverQueryRestrictedJoinAllowed       = "/roomserver/queryRestrictedJoinAllowed"
 )
 
 type httpRoomserverInternalAPI struct {
@@ -557,6 +558,16 @@ func (h *httpRoomserverInternalAPI) QueryServerBannedFromRoom(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
 }
 
+func (h *httpRoomserverInternalAPI) QueryRestrictedJoinAllowed(
+	ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRestrictedJoinAllowed")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverQueryRestrictedJoinAllowed
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
 func (h *httpRoomserverInternalAPI) PerformForget(ctx context.Context, req *api.PerformForgetRequest, res *api.PerformForgetResponse) error {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformForget")
 	defer span.Finish()

roomserver/inthttp/server.go

@@ -472,4 +472,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverQueryRestrictedJoinAllowed,
+		httputil.MakeInternalAPI("queryRestrictedJoinAllowed", func(req *http.Request) util.JSONResponse {
+			request := api.QueryRestrictedJoinAllowedRequest{}
+			response := api.QueryRestrictedJoinAllowedResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := r.QueryRestrictedJoinAllowed(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }

roomserver/storage/tables/events_table_test.go

@@ -85,7 +85,6 @@ func Test_EventsTable(t *testing.T) {
 				assert.NoError(t, err)
 			}
 			stateAtEvent := types.StateAtEvent{
-				Overwrite:              false,
 				BeforeStateSnapshotNID: types.StateSnapshotNID(stateSnapshot),
 				IsRejected:             false,
 				StateEntry: types.StateEntry{

roomserver/types/types.go

@@ -173,10 +173,6 @@ func DeduplicateStateEntries(a []StateEntry) []StateEntry {
 
 // StateAtEvent is the state before and after a matrix event.
 type StateAtEvent struct {
-	// Should this state overwrite the latest events and memberships of the room?
-	// This might be necessary when rejoining a federated room after a period of
-	// absence, as our state and latest events will be out of date.
-	Overwrite bool
 	// The state before the event.
 	BeforeStateSnapshotNID StateSnapshotNID
 	// True if this StateEntry is rejected. State resolution should then treat this
@@ -214,6 +210,14 @@ func (s StateAtEventAndReferences) Swap(a, b int) {
 	s[a], s[b] = s[b], s[a]
 }
 
+func (s StateAtEventAndReferences) EventIDs() string {
+	strs := make([]string, 0, len(s))
+	for _, r := range s {
+		strs = append(strs, r.EventID)
+	}
+	return "[" + strings.Join(strs, " ") + "]"
+}
+
 // An Event is a gomatrixserverlib.Event with the numeric event ID attached.
 // It is when performing bulk event lookup in the database.
 type Event struct {

userapi/storage/postgres/accounts_table.go

@@ -65,7 +65,7 @@ const selectPasswordHashSQL = "" +
 	"SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = FALSE"
 
 const selectNewNumericLocalpartSQL = "" +
-	"SELECT COALESCE(MAX(localpart::integer), 0) FROM account_accounts WHERE localpart ~ '^[0-9]*$'"
+	"SELECT COALESCE(MAX(localpart::bigint), 0) FROM account_accounts WHERE localpart ~ '^[0-9]{1,}$'"
 
 type accountsStatements struct {
 	insertAccountStmt             *sql.Stmt

userapi/storage/storage_test.go

@@ -124,6 +124,23 @@ func Test_Accounts(t *testing.T) {
 
 		_, err = db.GetAccountByLocalpart(ctx, "unusename")
 		assert.Error(t, err, "expected an error for non existent localpart")
+
+		// create an empty localpart; this should never happen, but is required to test getting a numeric localpart
+		// if there's already a user without a localpart in the database
+		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeUser)
+		assert.NoError(t, err)
+
+		// test getting a numeric localpart, with an existing user without a localpart
+		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeGuest)
+		assert.NoError(t, err)
+
+		// Create a user with a high numeric localpart, out of range for the Postgres integer (2147483647) type
+		_, err = db.CreateAccount(ctx, "2147483650", "", "", api.AccountTypeUser)
+		assert.NoError(t, err)
+
+		// Now try to create a new guest user
+		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeGuest)
+		assert.NoError(t, err)
 	})
 }