Merge branch 'matrix-org:main' into main

CHANGES.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## Dendrite 0.8.7 (2022-06-01)
+
+### Features
+
+* Support added for room version 10
+
+### Fixes
+
+* A number of state handling bugs have been fixed, which previously resulted in missing state events, unexpected state deletions, reverted memberships and unexpectedly rejected/soft-failed events in some specific cases
+* Fixed destination queue performance issues as a result of missing indexes, which speeds up outbound federation considerably
+* A bug which could cause the `/register` endpoint to return HTTP 500 has been fixed
+
 ## Dendrite 0.8.6 (2022-05-26)
 
 ### Features

README.md

@@ -96,10 +96,9 @@ than features that massive deployments may be interested in (User Directory, Ope
 This means Dendrite supports amongst others:
 
 - Core room functionality (creating rooms, invites, auth rules)
-- Full support for room versions 1 to 7
-- Experimental support for room versions 8 to 9
+- Room versions 1 to 10 supported
 - Backfilling locally and via federation
-- Accounts, Profiles and Devices
+- Accounts, profiles and devices
 - Published room lists
 - Typing
 - Media APIs

clientapi/routing/routing.go

@@ -89,6 +89,9 @@ func Setup(
 					"r0.4.0",
 					"r0.5.0",
 					"r0.6.1",
+					"v1.0",
+					"v1.1",
+					"v1.2",
 				}, UnstableFeatures: unstableFeatures},
 			}
 		}),

clientapi/routing/sendevent.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"reflect"
 	"sync"
 	"time"
 
@@ -96,14 +97,21 @@ func SendEvent(
 	mutex.(*sync.Mutex).Lock()
 	defer mutex.(*sync.Mutex).Unlock()
 
-	startedGeneratingEvent := time.Now()
-
 	var r map[string]interface{} // must be a JSON object
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
 		return *resErr
 	}
 
+	if stateKey != nil {
+		// If the existing/new state content are equal, return the existing event_id, making the request idempotent.
+		if resp := stateEqual(req.Context(), rsAPI, eventType, *stateKey, roomID, r); resp != nil {
+			return *resp
+		}
+	}
+
+	startedGeneratingEvent := time.Now()
+
 	// If we're sending a membership update, make sure to strip the authorised
 	// via key if it is present, otherwise other servers won't be able to auth
 	// the event if the room is set to the "restricted" join rule.
@@ -208,6 +216,37 @@ func SendEvent(
 	return res
 }
 
+// stateEqual compares the new and the existing state event content. If they are equal, returns a *util.JSONResponse
+// with the existing event_id, making this an idempotent request.
+func stateEqual(ctx context.Context, rsAPI api.ClientRoomserverAPI, eventType, stateKey, roomID string, newContent map[string]interface{}) *util.JSONResponse {
+	stateRes := api.QueryCurrentStateResponse{}
+	tuple := gomatrixserverlib.StateKeyTuple{
+		EventType: eventType,
+		StateKey:  stateKey,
+	}
+	err := rsAPI.QueryCurrentState(ctx, &api.QueryCurrentStateRequest{
+		RoomID:      roomID,
+		StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
+	}, &stateRes)
+	if err != nil {
+		return nil
+	}
+	if existingEvent, ok := stateRes.StateEvents[tuple]; ok {
+		var existingContent map[string]interface{}
+		if err = json.Unmarshal(existingEvent.Content(), &existingContent); err != nil {
+			return nil
+		}
+		if reflect.DeepEqual(existingContent, newContent) {
+			return &util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: sendEventResponse{existingEvent.EventID()},
+			}
+		}
+
+	}
+	return nil
+}
+
 func generateSendEvent(
 	ctx context.Context,
 	r map[string]interface{},

cmd/resolve-state/main.go

@@ -4,9 +4,12 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup"
@@ -57,25 +60,23 @@ func main() {
 		panic(err)
 	}
 
-	blockNIDs, err := roomserverDB.StateBlockNIDs(ctx, snapshotNIDs)
-	if err != nil {
-		panic(err)
-	}
+	stateres := state.NewStateResolution(roomserverDB, &types.RoomInfo{
+		RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion),
+	})
 
-	var stateEntries []types.StateEntryList
-	for _, list := range blockNIDs {
-		entries, err2 := roomserverDB.StateEntries(ctx, list.StateBlockNIDs)
-		if err2 != nil {
-			panic(err2)
+	var stateEntries []types.StateEntry
+	for _, snapshotNID := range snapshotNIDs {
+		var entries []types.StateEntry
+		entries, err = stateres.LoadStateAtSnapshot(ctx, snapshotNID)
+		if err != nil {
+			panic(err)
 		}
 		stateEntries = append(stateEntries, entries...)
 	}
 
 	var eventNIDs []types.EventNID
 	for _, entry := range stateEntries {
-		for _, e := range entry.StateEntries {
-			eventNIDs = append(eventNIDs, e.EventNID)
-		}
+		eventNIDs = append(eventNIDs, entry.EventNID)
 	}
 
 	fmt.Println("Fetching", len(eventNIDs), "state events")
@@ -110,7 +111,8 @@ func main() {
 	}
 
 	fmt.Println("Resolving state")
-	resolved, err := gomatrixserverlib.ResolveConflicts(
+	var resolved Events
+	resolved, err = gomatrixserverlib.ResolveConflicts(
 		gomatrixserverlib.RoomVersion(*roomVersion),
 		events,
 		authEvents,
@@ -120,6 +122,7 @@ func main() {
 	}
 
 	fmt.Println("Resolved state contains", len(resolved), "events")
+	sort.Sort(resolved)
 	filteringEventType := *filterType
 	count := 0
 	for _, event := range resolved {
@@ -135,3 +138,25 @@ func main() {
 	fmt.Println()
 	fmt.Println("Returned", count, "state events after filtering")
 }
+
+type Events []*gomatrixserverlib.Event
+
+func (e Events) Len() int {
+	return len(e)
+}
+
+func (e Events) Swap(i, j int) {
+	e[i], e[j] = e[j], e[i]
+}
+
+func (e Events) Less(i, j int) bool {
+	typeDelta := strings.Compare(e[i].Type(), e[j].Type())
+	if typeDelta < 0 {
+		return true
+	}
+	if typeDelta > 0 {
+		return false
+	}
+	stateKeyDelta := strings.Compare(*e[i].StateKey(), *e[j].StateKey())
+	return stateKeyDelta < 0
+}

federationapi/storage/postgres/queue_edus_table.go

@@ -36,6 +36,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
     ON federationsender_queue_edus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_nid_idx
+    ON federationsender_queue_edus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
+    ON federationsender_queue_edus (server_name);
 `
 
 const insertQueueEDUSQL = "" +

federationapi/storage/postgres/queue_json_table.go

@@ -33,6 +33,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_json (
 	-- The JSON body. Text so that we preserve UTF-8.
 	json_body TEXT NOT NULL
 );
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_json_json_nid_idx
+    ON federationsender_queue_json (json_nid);
 `
 
 const insertJSONSQL = "" +

federationapi/storage/postgres/queue_pdus_table.go

@@ -36,6 +36,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
     ON federationsender_queue_pdus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_json_nid_idx
+    ON federationsender_queue_pdus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_server_name_idx
+    ON federationsender_queue_pdus (server_name);
 `
 
 const insertQueuePDUSQL = "" +

federationapi/storage/sqlite3/queue_edus_table.go

@@ -37,6 +37,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
     ON federationsender_queue_edus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_nid_idx
+    ON federationsender_queue_edus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
+    ON federationsender_queue_edus (server_name);
 `
 
 const insertQueueEDUSQL = "" +

federationapi/storage/sqlite3/queue_json_table.go

@@ -35,6 +35,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_json (
 	-- The JSON body. Text so that we preserve UTF-8.
 	json_body TEXT NOT NULL
 );
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_json_json_nid_idx
+    ON federationsender_queue_json (json_nid);
 `
 
 const insertJSONSQL = "" +

federationapi/storage/sqlite3/queue_pdus_table.go

@@ -38,6 +38,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
     ON federationsender_queue_pdus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_json_nid_idx
+    ON federationsender_queue_pdus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_server_name_idx
+    ON federationsender_queue_pdus (server_name);
 `
 
 const insertQueuePDUSQL = "" +

go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13

go.sum

@@ -542,8 +542,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d h1:IwyG/58rFn0/ugD0A/IdSIo7D/oLJ4+k3NznlYhzyHs=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee h1:56sxEWrwB3eOmwjP2S4JsrQf29uBUaf+8WrbQJmjaGE=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 6
+	VersionPatch = 7
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/internal/device_list_update.go

@@ -374,7 +374,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 	// fetch stale device lists
 	userIDs, err := u.db.StaleDeviceLists(ctx, []gomatrixserverlib.ServerName{serverName})
 	if err != nil {
-		logger.WithError(err).Error("failed to load stale device lists")
+		logger.WithError(err).Error("Failed to load stale device lists")
 		return waitTime, true
 	}
 	failCount := 0
@@ -399,7 +399,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 				}
 			} else {
 				waitTime = time.Hour
-				logger.WithError(err).WithField("user_id", userID).Warn("GetUserDevices returned unknown error type")
+				logger.WithError(err).WithField("user_id", userID).Debug("GetUserDevices returned unknown error type")
 			}
 			continue
 		}
@@ -422,12 +422,12 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 		}
 		err = u.updateDeviceList(&res)
 		if err != nil {
-			logger.WithError(err).WithField("user_id", userID).Error("fetched device list but failed to store/emit it")
+			logger.WithError(err).WithField("user_id", userID).Error("Fetched device list but failed to store/emit it")
 			failCount += 1
 		}
 	}
 	if failCount > 0 {
-		logger.WithField("total", len(userIDs)).WithField("failed", failCount).WithField("wait", waitTime).Error("failed to query device keys for some users")
+		logger.WithField("total", len(userIDs)).WithField("failed", failCount).WithField("wait", waitTime).Warn("Failed to query device keys for some users")
 	}
 	for _, userID := range userIDs {
 		// always clear the channel to unblock Update calls regardless of success/failure

roomserver/internal/input/input_latest_events.go

@@ -206,7 +206,7 @@ func (u *latestEventsUpdater) latestState() error {
 	// Work out if the state at the extremities has actually changed
 	// or not. If they haven't then we won't bother doing all of the
 	// hard work.
-	if u.event.StateKey() == nil {
+	if !u.stateAtEvent.IsStateEvent() {
 		stateChanged := false
 		oldStateNIDs := make([]types.StateSnapshotNID, 0, len(u.oldLatest))
 		newStateNIDs := make([]types.StateSnapshotNID, 0, len(u.latest))
@@ -234,19 +234,12 @@ func (u *latestEventsUpdater) latestState() error {
 		}
 	}
 
-	// Take the old set of extremities and the new set of extremities and
-	// mash them together into a list. This may or may not include the new event
-	// from the input path, depending on whether it became a forward extremity
-	// or not. We'll then run state resolution across all of them to determine
-	// the new current state of the room. Including the old extremities here
-	// ensures that new forward extremities with bad state snapshots (from
-	// possible malicious actors) can't completely corrupt the room state
-	// away from what it was before.
-	combinedExtremities := types.StateAtEventAndReferences(append(u.oldLatest, u.latest...))
-	combinedExtremities = combinedExtremities[:util.SortAndUnique(combinedExtremities)]
-	latestStateAtEvents := make([]types.StateAtEvent, len(combinedExtremities))
-	for i := range combinedExtremities {
-		latestStateAtEvents[i] = combinedExtremities[i].StateAtEvent
+	// Get a list of the current latest events. This may or may not
+	// include the new event from the input path, depending on whether
+	// it is a forward extremity or not.
+	latestStateAtEvents := make([]types.StateAtEvent, len(u.latest))
+	for i := range u.latest {
+		latestStateAtEvents[i] = u.latest[i].StateAtEvent
 	}
 
 	// Takes the NIDs of the latest events and creates a state snapshot

roomserver/state/state.go

@@ -39,6 +39,7 @@ type StateResolutionStorage interface {
 	StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error)
 	AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error)
 	Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error)
+	EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error)
 }
 
 type StateResolution struct {
@@ -659,15 +660,13 @@ func (v *StateResolution) calculateStateAfterManyEvents(
 	}
 
 	// Collect all the entries with the same type and key together.
-	// We don't care about the order here because the conflict resolution
-	// algorithm doesn't depend on the order of the prev events.
-	// Remove duplicate entires.
+	// This is done so findDuplicateStateKeys can work in groups.
+	// We remove duplicates (same type, state key and event NID) too.
 	combined = combined[:util.SortAndUnique(stateEntrySorter(combined))]
 
 	// Find the conflicts
-	conflicts := findDuplicateStateKeys(combined)
-
-	if len(conflicts) > 0 {
+	if conflicts := findDuplicateStateKeys(combined); len(conflicts) > 0 {
+		conflictMap := stateEntryMap(conflicts)
 		conflictLength = len(conflicts)
 
 		// 5) There are conflicting state events, for each conflict workout
@@ -676,7 +675,7 @@ func (v *StateResolution) calculateStateAfterManyEvents(
 		// Work out which entries aren't conflicted.
 		var notConflicted []types.StateEntry
 		for _, entry := range combined {
-			if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok {
+			if _, ok := conflictMap.lookup(entry.StateKeyTuple); !ok {
 				notConflicted = append(notConflicted, entry)
 			}
 		}
@@ -689,7 +688,7 @@ func (v *StateResolution) calculateStateAfterManyEvents(
 			return
 		}
 		algorithm = "full_state_with_conflicts"
-		state = resolved[:util.SortAndUnique(stateEntrySorter(resolved))]
+		state = resolved
 	} else {
 		algorithm = "full_state_no_conflicts"
 		// 6) There weren't any conflicts
@@ -818,39 +817,19 @@ func (v *StateResolution) resolveConflictsV2(
 	authDifference := make([]*gomatrixserverlib.Event, 0, estimate)
 
 	// For each conflicted event, let's try and get the needed auth events.
-	neededStateKeys := make([]string, 16)
-	authEntries := make([]types.StateEntry, 16)
 	for _, conflictedEvent := range conflictedEvents {
 		// Work out which auth events we need to load.
 		key := conflictedEvent.EventID()
-		needed := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{conflictedEvent})
-
-		// Find the numeric IDs for the necessary state keys.
-		neededStateKeys = neededStateKeys[:0]
-		neededStateKeys = append(neededStateKeys, needed.Member...)
-		neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...)
-		stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys)
-		if err != nil {
-			return nil, err
-		}
-
-		// Load the necessary auth events.
-		tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed)
-		authEntries = authEntries[:0]
-		for _, tuple := range tuplesNeeded {
-			if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok {
-				authEntries = append(authEntries, types.StateEntry{
-					StateKeyTuple: tuple,
-					EventNID:      eventNID,
-				})
-			}
-		}
 
 		// Store the newly found auth events in the auth set for this event.
-		authSets[key], _, err = v.loadStateEvents(ctx, authEntries)
+		var authEventMap map[string]types.StateEntry
+		authSets[key], authEventMap, err = v.loadAuthEvents(ctx, conflictedEvent)
 		if err != nil {
 			return nil, err
 		}
+		for k, v := range authEventMap {
+			eventIDMap[k] = v
+		}
 
 		// Only add auth events into the authEvents slice once, otherwise the
 		// check for the auth difference can become expensive and produce
@@ -909,7 +888,7 @@ func (v *StateResolution) resolveConflictsV2(
 	for _, resolvedEvent := range resolvedEvents {
 		entry, ok := eventIDMap[resolvedEvent.EventID()]
 		if !ok {
-			panic(fmt.Errorf("missing state entry for event ID %q", resolvedEvent.EventID()))
+			return nil, fmt.Errorf("missing state entry for event ID %q", resolvedEvent.EventID())
 		}
 		notConflicted = append(notConflicted, entry)
 	}
@@ -996,6 +975,84 @@ func (v *StateResolution) loadStateEvents(
 	return result, eventIDMap, nil
 }
 
+// loadAuthEvents loads all of the auth events for a given event recursively,
+// along with a map that contains state entries for all of the auth events.
+func (v *StateResolution) loadAuthEvents(
+	ctx context.Context, event *gomatrixserverlib.Event,
+) ([]*gomatrixserverlib.Event, map[string]types.StateEntry, error) {
+	eventMap := map[string]struct{}{}
+	var lookup []string
+	var authEvents []types.Event
+	queue := event.AuthEventIDs()
+	for i := 0; i < len(queue); i++ {
+		lookup = lookup[:0]
+		for _, authEventID := range queue {
+			if _, ok := eventMap[authEventID]; ok {
+				continue
+			}
+			lookup = append(lookup, authEventID)
+		}
+		if len(lookup) == 0 {
+			break
+		}
+		events, err := v.db.EventsFromIDs(ctx, lookup)
+		if err != nil {
+			return nil, nil, fmt.Errorf("v.db.EventsFromIDs: %w", err)
+		}
+		add := map[string]struct{}{}
+		for _, event := range events {
+			eventMap[event.EventID()] = struct{}{}
+			authEvents = append(authEvents, event)
+			for _, authEventID := range event.AuthEventIDs() {
+				if _, ok := eventMap[authEventID]; ok {
+					continue
+				}
+				add[authEventID] = struct{}{}
+			}
+			for authEventID := range add {
+				queue = append(queue, authEventID)
+			}
+		}
+	}
+	authEventTypes := map[string]struct{}{}
+	authEventStateKeys := map[string]struct{}{}
+	for _, authEvent := range authEvents {
+		authEventTypes[authEvent.Type()] = struct{}{}
+		authEventStateKeys[*authEvent.StateKey()] = struct{}{}
+	}
+	lookupAuthEventTypes := make([]string, 0, len(authEventTypes))
+	lookupAuthEventStateKeys := make([]string, 0, len(authEventStateKeys))
+	for eventType := range authEventTypes {
+		lookupAuthEventTypes = append(lookupAuthEventTypes, eventType)
+	}
+	for eventStateKey := range authEventStateKeys {
+		lookupAuthEventStateKeys = append(lookupAuthEventStateKeys, eventStateKey)
+	}
+	eventTypes, err := v.db.EventTypeNIDs(ctx, lookupAuthEventTypes)
+	if err != nil {
+		return nil, nil, fmt.Errorf("v.db.EventTypeNIDs: %w", err)
+	}
+	eventStateKeys, err := v.db.EventStateKeyNIDs(ctx, lookupAuthEventStateKeys)
+	if err != nil {
+		return nil, nil, fmt.Errorf("v.db.EventStateKeyNIDs: %w", err)
+	}
+	stateEntryMap := map[string]types.StateEntry{}
+	for _, authEvent := range authEvents {
+		stateEntryMap[authEvent.EventID()] = types.StateEntry{
+			EventNID: authEvent.EventNID,
+			StateKeyTuple: types.StateKeyTuple{
+				EventTypeNID:     eventTypes[authEvent.Type()],
+				EventStateKeyNID: eventStateKeys[*authEvent.StateKey()],
+			},
+		}
+	}
+	nakedEvents := make([]*gomatrixserverlib.Event, 0, len(authEvents))
+	for _, authEvent := range authEvents {
+		nakedEvents = append(nakedEvents, authEvent.Event)
+	}
+	return nakedEvents, stateEntryMap, nil
+}
+
 // findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list.
 // Returns a sorted list of those state entries.
 func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry {

roomserver/storage/shared/room_updater.go

@@ -192,6 +192,10 @@ func (u *RoomUpdater) StateAtEventIDs(
 	return u.d.EventsTable.BulkSelectStateAtEventByID(ctx, u.txn, eventIDs)
 }
 
+func (u *RoomUpdater) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) {
+	return u.d.eventsFromIDs(ctx, u.txn, eventIDs, false)
+}
+
 func (u *RoomUpdater) UnsentEventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) {
 	return u.d.eventsFromIDs(ctx, u.txn, eventIDs, true)
 }

sytest-whitelist

@@ -715,4 +715,7 @@ Presence can be set from sync
 PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
 Unnamed room comes with a name summary
 Named room comes with just joined member count summary
-Room summary only has 5 heroes
+Room summary only has 5 heroes
+Setting state twice is idempotent
+Joining room twice is idempotent
+Inbound federation can return missing events for shared visibility

userapi/storage/postgres/accounts_table.go

@@ -65,7 +65,7 @@ const selectPasswordHashSQL = "" +
 	"SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = FALSE"
 
 const selectNewNumericLocalpartSQL = "" +
-	"SELECT COALESCE(MAX(localpart::integer), 0) FROM account_accounts WHERE localpart ~ '^[0-9]*$'"
+	"SELECT COALESCE(MAX(localpart::bigint), 0) FROM account_accounts WHERE localpart ~ '^[0-9]{1,}$'"
 
 type accountsStatements struct {
 	insertAccountStmt             *sql.Stmt

userapi/storage/storage_test.go

@@ -124,6 +124,23 @@ func Test_Accounts(t *testing.T) {
 
 		_, err = db.GetAccountByLocalpart(ctx, "unusename")
 		assert.Error(t, err, "expected an error for non existent localpart")
+
+		// create an empty localpart; this should never happen, but is required to test getting a numeric localpart
+		// if there's already a user without a localpart in the database
+		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeUser)
+		assert.NoError(t, err)
+
+		// test getting a numeric localpart, with an existing user without a localpart
+		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeGuest)
+		assert.NoError(t, err)
+
+		// Create a user with a high numeric localpart, out of range for the Postgres integer (2147483647) type
+		_, err = db.CreateAccount(ctx, "2147483650", "", "", api.AccountTypeUser)
+		assert.NoError(t, err)
+
+		// Now try to create a new guest user
+		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeGuest)
+		assert.NoError(t, err)
 	})
 }