merge latest dendrite main. Resolve merge conflicts

CHANGES.md

@@ -1,5 +1,39 @@
 # Changelog
 
+## Dendrite 0.8.6 (2022-05-26)
+
+### Features
+
+* Room versions 8 and 9 are now marked as stable
+* Dendrite can now assist remote users to join restricted rooms via `/make_join` and `/send_join`
+
+### Fixes
+
+* The sync API no longer returns immediately on `/sync` requests unnecessarily if it can be avoided
+* A race condition has been fixed in the sync API when updating presence via `/sync`
+* A race condition has been fixed sending E2EE keys to remote servers over federation when joining rooms
+* The `trusted_private_chat` preset should now grant power level 100 to all participant users, which should improve the user experience of direct messages
+* Invited users are now authed correctly in restricted rooms
+* The `join_authorised_by_users_server` key is now correctly stripped in restricted rooms when updating the membership event
+* Appservices should now receive invite events correctly
+* Device list updates should no longer contain optional fields with `null` values
+* The `/deactivate` endpoint has been fixed to no longer confuse Element with incorrect completed flows
+
+## Dendrite 0.8.5 (2022-05-13)
+
+### Features
+
+* New living documentation available at <https://matrix-org.github.io/dendrite/>, including new installation instructions
+* The built-in NATS Server has been updated to version 2.8.2
+
+### Fixes
+
+* Monolith deployments will no longer panic at startup if given a config file that does not include the `internal_api` and `external_api` options
+* State resolution v2 now correctly identifies other events related to power events, which should fix some event auth issues
+* The latest events updater will no longer implicitly trust the new forward extremities when calculating the current room state, which may help to avoid some state resets
+* The one-time key count is now correctly returned in `/sync` even if the request otherwise timed out, which should reduce the chance that unnecessary one-time keys will be uploaded by clients
+* The `create-account` tool should now work properly when the database is configured using the global connection pool
+
 ## Dendrite 0.8.4 (2022-05-10)
 
 ### Fixes

README.md

@@ -53,7 +53,7 @@ The [Federation Tester](https://federationtester.matrix.org) can be used to veri
 
 ## Get started
 
-If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](docs/installation). For running in Docker, see [build/docker](build/docker).
+If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](https://matrix-org.github.io/dendrite/installation). For running in Docker, see [build/docker](build/docker).
 
 The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases:
 

appservice/consumers/roomserver.go

@@ -83,29 +83,38 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 		return true
 	}
 
-	if output.Type != api.OutputTypeNewRoomEvent || output.NewRoomEvent == nil {
+	log.WithFields(log.Fields{
-		return true
+		"type": output.Type,
-	}
+	}).Debug("Got a message in OutputRoomEventConsumer")
 
-	newEventID := output.NewRoomEvent.Event.EventID()
+	events := []*gomatrixserverlib.HeaderedEvent{}
-	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(output.NewRoomEvent.AddsStateEventIDs))
+	if output.Type == api.OutputTypeNewRoomEvent && output.NewRoomEvent != nil {
-	events = append(events, output.NewRoomEvent.Event)
+		newEventID := output.NewRoomEvent.Event.EventID()
-	if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
+		events = append(events, output.NewRoomEvent.Event)
-		eventsReq := &api.QueryEventsByIDRequest{
+		if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
-			EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
+			eventsReq := &api.QueryEventsByIDRequest{
-		}
+				EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
-		eventsRes := &api.QueryEventsByIDResponse{}
+			}
-		for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
+			eventsRes := &api.QueryEventsByIDResponse{}
-			if eventID != newEventID {
+			for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
-				eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
+				if eventID != newEventID {
+					eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
+				}
+			}
+			if len(eventsReq.EventIDs) > 0 {
+				if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
+					return false
+				}
+				events = append(events, eventsRes.Events...)
 			}
 		}
-		if len(eventsReq.EventIDs) > 0 {
+	} else if output.Type == api.OutputTypeNewInviteEvent && output.NewInviteEvent != nil {
-			if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
+		events = append(events, output.NewInviteEvent.Event)
-				return false
+	} else {
-			}
+		log.WithFields(log.Fields{
-			events = append(events, eventsRes.Events...)
+			"type": output.Type,
-		}
+		}).Debug("appservice OutputRoomEventConsumer ignoring event", string(msg.Data))
+		return true
 	}
 
 	// Send event to any relevant application services

build/gobind-pinecone/monolith.go

@@ -225,7 +225,7 @@ func (m *DendriteMonolith) Start() {
 		pk = sk.Public().(ed25519.PublicKey)
 	}
 
-	m.listener, err = net.Listen("tcp", ":65432")
+	m.listener, err = net.Listen("tcp", "localhost:65432")
 	if err != nil {
 		panic(err)
 	}

clientapi/auth/user_interactive.go

@@ -102,8 +102,7 @@ type userInteractiveFlow struct {
 // the user already has a valid access token, but we want to double-check
 // that it isn't stolen by re-authenticating them.
 type UserInteractive struct {
-	Completed []string
+	Flows []userInteractiveFlow
-	Flows     []userInteractiveFlow
 	// Map of login type to implementation
 	Types map[string]Type
 	// Map of session ID to completed login types, will need to be extended in future
@@ -117,11 +116,10 @@ func NewUserInteractive(
 	cfg *config.ClientAPI,
 ) *UserInteractive {
 	userInteractive := UserInteractive{
-		Completed: []string{},
+		Flows:    []userInteractiveFlow{},
-		Flows:     []userInteractiveFlow{},
+		Types:    make(map[string]Type),
-		Types:     make(map[string]Type),
+		Sessions: make(map[string][]string),
-		Sessions:  make(map[string][]string),
+		Params:   make(map[string]interface{}),
-		Params:    make(map[string]interface{}),
 	}
 
 	if !cfg.PasswordAuthenticationDisabled {
@@ -155,7 +153,6 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
 
 func (u *UserInteractive) AddCompletedStage(sessionID, authType string) {
 	// TODO: Handle multi-stage flows
-	u.Completed = append(u.Completed, authType)
 	delete(u.Sessions, sessionID)
 }
 
@@ -176,7 +173,7 @@ func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse {
 	return &util.JSONResponse{
 		Code: 401,
 		JSON: Challenge{
-			Completed: u.Completed,
+			Completed: u.Sessions[sessionID],
 			Flows:     u.Flows,
 			Session:   sessionID,
 			Params:    u.Params,

clientapi/auth/user_interactive_test.go

@@ -186,3 +186,38 @@ func TestUserInteractivePasswordBadLogin(t *testing.T) {
 		}
 	}
 }
+
+func TestUserInteractive_AddCompletedStage(t *testing.T) {
+	tests := []struct {
+		name      string
+		sessionID string
+	}{
+		{
+			name:      "first user",
+			sessionID: util.RandomString(8),
+		},
+		{
+			name:      "second user",
+			sessionID: util.RandomString(8),
+		},
+		{
+			name:      "third user",
+			sessionID: util.RandomString(8),
+		},
+	}
+	u := setup()
+	ctx := context.Background()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, resp := u.Verify(ctx, []byte("{}"), nil)
+			challenge, ok := resp.JSON.(Challenge)
+			if !ok {
+				t.Fatalf("expected a Challenge, got %T", resp.JSON)
+			}
+			if len(challenge.Completed) > 0 {
+				t.Fatalf("expected 0 completed stages, got %d", len(challenge.Completed))
+			}
+			u.AddCompletedStage(tt.sessionID, "")
+		})
+	}
+}

clientapi/jsonerror/jsonerror.go

@@ -154,6 +154,12 @@ func MissingParam(msg string) *MatrixError {
 	return &MatrixError{"M_MISSING_PARAM", msg}
 }
 
+// UnableToAuthoriseJoin is an error that is returned when a server can't
+// determine whether to allow a restricted join or not.
+func UnableToAuthoriseJoin(msg string) *MatrixError {
+	return &MatrixError{"M_UNABLE_TO_AUTHORISE_JOIN", msg}
+}
+
 // LeaveServerNoticeError is an error returned when trying to reject an invite
 // for a server notice room.
 func LeaveServerNoticeError() *MatrixError {

clientapi/routing/createroom.go

@@ -245,7 +245,9 @@ func createRoom(
 	case presetTrustedPrivateChat:
 		joinRuleContent.JoinRule = gomatrixserverlib.Invite
 		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
-		// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
+		for _, invitee := range r.Invite {
+			powerLevelContent.Users[invitee] = 100
+		}
 	case presetPublicChat:
 		joinRuleContent.JoinRule = gomatrixserverlib.Public
 		historyVisibilityContent.HistoryVisibility = historyVisibilityShared

clientapi/routing/sendevent.go

@@ -104,6 +104,13 @@ func SendEvent(
 		return *resErr
 	}
 
+	// If we're sending a membership update, make sure to strip the authorised
+	// via key if it is present, otherwise other servers won't be able to auth
+	// the event if the room is set to the "restricted" join rule.
+	if eventType == gomatrixserverlib.MRoomMember {
+		delete(r, "join_authorised_via_users_server")
+	}
+
 	evTime, err := httputil.ParseTSParam(req)
 	if err != nil {
 		return util.JSONResponse{

cmd/generate-keys/main.go

@@ -20,7 +20,7 @@ import (
 	"log"
 	"os"
 
-	"github.com/matrix-org/dendrite/internal/test"
+	"github.com/matrix-org/dendrite/test"
 )
 
 const usage = `Usage: %s

cmd/resolve-state/main.go

@@ -4,13 +4,14 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"os"
 	"strconv"
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -23,11 +24,17 @@ import (
 //   e.g. ./resolve-state --roomversion=5 1254 1235 1282
 
 var roomVersion = flag.String("roomversion", "5", "the room version to parse events as")
+var filterType = flag.String("filtertype", "", "the event types to filter on")
 
 func main() {
 	ctx := context.Background()
 	cfg := setup.ParseFlags(true)
-	args := os.Args[1:]
+	cfg.Logging = append(cfg.Logging[:0], config.LogrusHook{
+		Type:  "std",
+		Level: "error",
+	})
+	base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics)
+	args := flag.Args()
 
 	fmt.Println("Room version", *roomVersion)
 
@@ -45,7 +52,7 @@ func main() {
 		panic(err)
 	}
 
-	roomserverDB, err := storage.Open(nil, &cfg.RoomServer.Database, cache)
+	roomserverDB, err := storage.Open(base, &cfg.RoomServer.Database, cache)
 	if err != nil {
 		panic(err)
 	}
@@ -113,9 +120,18 @@ func main() {
 	}
 
 	fmt.Println("Resolved state contains", len(resolved), "events")
+	filteringEventType := *filterType
+	count := 0
 	for _, event := range resolved {
+		if filteringEventType != "" && event.Type() != filteringEventType {
+			continue
+		}
+		count++
 		fmt.Println()
 		fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey())
 		fmt.Printf("  %s\n", string(event.Content()))
 	}
+
+	fmt.Println()
+	fmt.Println("Returned", count, "state events after filtering")
 }

docs/coverage.md

@@ -0,0 +1,84 @@
+---
+title: Coverage
+parent: Development
+permalink: /development/coverage
+---
+
+To generate a test coverage report for Sytest, a small patch needs to be applied to the Sytest repository to compile and use the instrumented binary:
+```patch
+diff --git a/lib/SyTest/Homeserver/Dendrite.pm b/lib/SyTest/Homeserver/Dendrite.pm
+index 8f0e209c..ad057e52 100644
+--- a/lib/SyTest/Homeserver/Dendrite.pm
++++ b/lib/SyTest/Homeserver/Dendrite.pm
+@@ -337,7 +337,7 @@ sub _start_monolith
+ 
+    $output->diag( "Starting monolith server" );
+    my @command = (
+-      $self->{bindir} . '/dendrite-monolith-server',
++      $self->{bindir} . '/dendrite-monolith-server', '--test.coverprofile=' . $self->{hs_dir} . '/integrationcover.log', "DEVEL",
+       '--config', $self->{paths}{config},
+       '--http-bind-address', $self->{bind_host} . ':' . $self->unsecure_port,
+       '--https-bind-address', $self->{bind_host} . ':' . $self->secure_port,
+diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh
+index f009332b..7ea79869 100755
+--- a/scripts/dendrite_sytest.sh
++++ b/scripts/dendrite_sytest.sh
+@@ -34,7 +34,8 @@ export GOBIN=/tmp/bin
+ echo >&2 "--- Building dendrite from source"
+ cd /src
+ mkdir -p $GOBIN
+-go install -v ./cmd/dendrite-monolith-server
++# go install -v ./cmd/dendrite-monolith-server
++go test -c -cover -covermode=atomic -o $GOBIN/dendrite-monolith-server -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server
+ go install -v ./cmd/generate-keys
+ cd -
+ ```
+
+ Then run Sytest. This will generate a new file `integrationcover.log` in each server's directory e.g `server-0/integrationcover.log`. To parse it,
+ ensure your working directory is under the Dendrite repository then run:
+ ```bash
+ go tool cover -func=/path/to/server-0/integrationcover.log
+ ```
+ which will produce an output like:
+ ```
+ ...
+ github.com/matrix-org/util/json.go:83:											NewJSONRequestHandler				100.0%
+github.com/matrix-org/util/json.go:90:											Protect						57.1%
+github.com/matrix-org/util/json.go:110:											RequestWithLogging				100.0%
+github.com/matrix-org/util/json.go:132:											MakeJSONAPI					70.0%
+github.com/matrix-org/util/json.go:151:											respond						61.5%
+github.com/matrix-org/util/json.go:180:											WithCORSOptions					0.0%
+github.com/matrix-org/util/json.go:191:											SetCORSHeaders					100.0%
+github.com/matrix-org/util/json.go:202:											RandomString					100.0%
+github.com/matrix-org/util/json.go:210:											init						100.0%
+github.com/matrix-org/util/unique.go:13:										Unique						91.7%
+github.com/matrix-org/util/unique.go:48:										SortAndUnique					100.0%
+github.com/matrix-org/util/unique.go:55:										UniqueStrings					100.0%
+total:															(statements)					53.7%
+```
+The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in many different configurations,
+which will never be tested in a single test run (e.g sqlite or postgres, monolith or polylith). To get a more accurate value, additional processing is required
+to remove packages which will never be tested and extension MSCs:
+```bash
+# These commands are all similar but change which package paths are _removed_ from the output.
+
+# For Postgres (monolith)
+go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|sqlite|setup/mscs|api_trace' > coverage.txt
+
+# For Postgres (polylith)
+go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'sqlite|setup/mscs|api_trace' > coverage.txt
+
+# For SQLite (monolith)
+go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|postgres|setup/mscs|api_trace' > coverage.txt
+
+# For SQLite (polylith)
+go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'postgres|setup/mscs|api_trace' > coverage.txt
+```
+
+A total value can then be calculated using:
+```bash
+cat coverage.txt | awk -F '\t+' '{x = x + $3} END {print x/NR}'
+```
+
+
+We currently do not have a way to combine Sytest/Complement/Unit Tests into a single coverage report.

docs/installation/1_planning.md

@@ -9,21 +9,19 @@ permalink: /installation/planning
 
 ## Modes
 
-Dendrite can be run in one of two configurations:
+Dendrite consists of several components, each responsible for a different aspect of the Matrix protocol.
+Users can run Dendrite in one of two modes which dictate how these components are executed and communicate.
 
-* **Monolith mode**: All components run in the same process. In this mode,
+* **Monolith mode** runs all components in a single process. Components communicate through an internal NATS
-   it is possible to run an in-process NATS Server instead of running a standalone deployment.
+  server with generally low overhead. This mode dramatically simplifies deployment complexity and offers the
-   This will usually be the preferred model for low-to-mid volume deployments, providing the best
+  best balance between performance and resource usage for low-to-mid volume deployments.
-   balance between performance and resource usage.
 
-* **Polylith mode**: A cluster of individual components running in their own processes, dealing
+* **Polylith mode** runs all components in isolated processes. Components communicate through an external NATS
-  with different aspects of the Matrix protocol. Components communicate with each other using
+  server and HTTP APIs, which incur considerable overhead. While this mode allows for more granular control of
-  internal HTTP APIs and NATS Server. This will almost certainly be the preferred model for very
+  resources dedicated toward individual processes, given the additional communications overhead, it is only
-  large deployments but scalability comes with a cost. API calls are expensive and therefore a
+  necessary for very large deployments.
-  polylith deployment may end up using disproportionately more resources for a smaller number of
-  users compared to a monolith deployment.
 
-At present, we **recommend monolith mode deployments** in all cases.
+Given our current state of development, **we recommend monolith mode** for all deployments.
 
 ## Databases
 

federationapi/api/api.go

@@ -12,12 +12,16 @@ import (
 
 // FederationInternalAPI is used to query information from the federation sender.
 type FederationInternalAPI interface {
-	FederationClient
+	gomatrixserverlib.FederatedStateClient
+	KeyserverFederationAPI
 	gomatrixserverlib.KeyDatabase
 	ClientFederationAPI
 	RoomserverFederationAPI
 
 	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
+	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
+	MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
+	MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
 
 	// Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos.
 	PerformBroadcastEDU(
@@ -60,17 +64,43 @@ type RoomserverFederationAPI interface {
 	LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
-// FederationClient is a subset of gomatrixserverlib.FederationClient functions which the fedsender
+// KeyserverFederationAPI is a subset of gomatrixserverlib.FederationClient functions which the keyserver
 // implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
 // this interface are of type FederationClientError
-type FederationClient interface {
+type KeyserverFederationAPI interface {
-	gomatrixserverlib.FederatedStateClient
 	GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
 	ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
 	QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
+}
+
+// an interface for gmsl.FederationClient - contains functions called by federationapi only.
+type FederationClient interface {
+	gomatrixserverlib.KeyClient
+	SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error)
+
+	// Perform operations
+	LookupRoomAlias(ctx context.Context, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error)
+	Peek(ctx context.Context, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error)
+	MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error)
+	SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error)
+	MakeLeave(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error)
+	SendLeave(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error)
+	SendInviteV2(ctx context.Context, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error)
+
+	GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
+
+	GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
+	GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error)
+	ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error)
+	QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error)
+	Backfill(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
 	MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
 	MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
-	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
+
+	ExchangeThirdPartyInvite(ctx context.Context, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
+	LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error)
+	LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
+	LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
 // FederationClientError is returned from FederationClient methods in the event of a problem.

federationapi/consumers/keychange.go

@@ -39,7 +39,7 @@ type KeyChangeConsumer struct {
 	db         storage.Database
 	queues     *queue.OutgoingQueues
 	serverName gomatrixserverlib.ServerName
-	rsAPI      roomserverAPI.RoomserverInternalAPI
+	rsAPI      roomserverAPI.FederationRoomserverAPI
 	topic      string
 }
 
@@ -50,7 +50,7 @@ func NewKeyChangeConsumer(
 	js nats.JetStreamContext,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.FederationRoomserverAPI,
 ) *KeyChangeConsumer {
 	return &KeyChangeConsumer{
 		ctx:        process.Context(),
@@ -120,6 +120,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 		logger.WithError(err).Error("failed to calculate joined rooms for user")
 		return true
 	}
+
 	// send this key change to all servers who share rooms with this user.
 	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
 	if err != nil {

federationapi/consumers/roomserver.go

@@ -36,7 +36,7 @@ import (
 type OutputRoomEventConsumer struct {
 	ctx       context.Context
 	cfg       *config.FederationAPI
-	rsAPI     api.RoomserverInternalAPI
+	rsAPI     api.FederationRoomserverAPI
 	jetstream nats.JetStreamContext
 	durable   string
 	db        storage.Database
@@ -51,7 +51,7 @@ func NewOutputRoomEventConsumer(
 	js nats.JetStreamContext,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 ) *OutputRoomEventConsumer {
 	return &OutputRoomEventConsumer{
 		ctx:       process.Context(),
@@ -89,15 +89,7 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 	switch output.Type {
 	case api.OutputTypeNewRoomEvent:
 		ev := output.NewRoomEvent.Event
-
+		if err := s.processMessage(*output.NewRoomEvent, output.NewRoomEvent.RewritesState); err != nil {
-		if output.NewRoomEvent.RewritesState {
-			if err := s.db.PurgeRoomState(s.ctx, ev.RoomID()); err != nil {
-				log.WithError(err).Errorf("roomserver output log: purge room state failure")
-				return false
-			}
-		}
-
-		if err := s.processMessage(*output.NewRoomEvent); err != nil {
 			// panic rather than continue with an inconsistent database
 			log.WithFields(log.Fields{
 				"event_id":   ev.EventID(),
@@ -145,7 +137,7 @@ func (s *OutputRoomEventConsumer) processInboundPeek(orp api.OutputNewInboundPee
 
 // processMessage updates the list of currently joined hosts in the room
 // and then sends the event to the hosts that were joined before the event.
-func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error {
+func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rewritesState bool) error {
 	addsStateEvents, missingEventIDs := ore.NeededStateEventIDs()
 
 	// Ask the roomserver and add in the rest of the results into the set.
@@ -164,7 +156,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err
 		addsStateEvents = append(addsStateEvents, eventsRes.Events...)
 	}
 
-	addsJoinedHosts, err := joinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(addsStateEvents))
+	addsJoinedHosts, err := JoinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(addsStateEvents))
 	if err != nil {
 		return err
 	}
@@ -176,10 +168,9 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err
 	oldJoinedHosts, err := s.db.UpdateRoom(
 		s.ctx,
 		ore.Event.RoomID(),
-		ore.LastSentEventID,
-		ore.Event.EventID(),
 		addsJoinedHosts,
 		ore.RemovesStateEventIDs,
+		rewritesState, // if we're re-writing state, nuke all joined hosts before adding
 	)
 	if err != nil {
 		return err
@@ -238,7 +229,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 		return nil, err
 	}
 
-	combinedAddsJoinedHosts, err := joinedHostsFromEvents(combinedAddsEvents)
+	combinedAddsJoinedHosts, err := JoinedHostsFromEvents(combinedAddsEvents)
 	if err != nil {
 		return nil, err
 	}
@@ -284,10 +275,10 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 	return result, nil
 }
 
-// joinedHostsFromEvents turns a list of state events into a list of joined hosts.
+// JoinedHostsFromEvents turns a list of state events into a list of joined hosts.
 // This errors if one of the events was invalid.
 // It should be impossible for an invalid event to get this far in the pipeline.
-func joinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost, error) {
+func JoinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost, error) {
 	var joinedHosts []types.JoinedHost
 	for _, ev := range evs {
 		if ev.Type() != "m.room.member" || ev.StateKey() == nil {

federationapi/federationapi.go

@@ -93,8 +93,8 @@ func AddPublicRoutes(
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
 	base *base.BaseDendrite,
-	federation *gomatrixserverlib.FederationClient,
+	federation api.FederationClient,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.FederationRoomserverAPI,
 	caches *caching.Caches,
 	keyRing *gomatrixserverlib.KeyRing,
 	resetBlacklist bool,

federationapi/federationapi_keys_test.go

@@ -55,60 +55,62 @@ var servers = map[string]*server{
 func TestMain(m *testing.M) {
 	// Set up the server key API for each "server" that we
 	// will use in our tests.
-	for _, s := range servers {
+	os.Exit(func() int {
-		// Generate a new key.
+		for _, s := range servers {
-		_, testPriv, err := ed25519.GenerateKey(nil)
+			// Generate a new key.
-		if err != nil {
+			_, testPriv, err := ed25519.GenerateKey(nil)
-			panic("can't generate identity key: " + err.Error())
+			if err != nil {
+				panic("can't generate identity key: " + err.Error())
+			}
+
+			// Create a new cache but don't enable prometheus!
+			s.cache, err = caching.NewInMemoryLRUCache(false)
+			if err != nil {
+				panic("can't create cache: " + err.Error())
+			}
+
+			// Create a temporary directory for JetStream.
+			d, err := ioutil.TempDir("./", "jetstream*")
+			if err != nil {
+				panic(err)
+			}
+			defer os.RemoveAll(d)
+
+			// Draw up just enough Dendrite config for the server key
+			// API to work.
+			cfg := &config.Dendrite{}
+			cfg.Defaults(true)
+			cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
+			cfg.Global.PrivateKey = testPriv
+			cfg.Global.JetStream.InMemory = true
+			cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
+			cfg.Global.JetStream.StoragePath = config.Path(d)
+			cfg.Global.KeyID = serverKeyID
+			cfg.Global.KeyValidityPeriod = s.validity
+			cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
+			s.config = &cfg.FederationAPI
+
+			// Create a transport which redirects federation requests to
+			// the mock round tripper. Since we're not *really* listening for
+			// federation requests then this will return the key instead.
+			transport := &http.Transport{}
+			transport.RegisterProtocol("matrix", &MockRoundTripper{})
+
+			// Create the federation client.
+			s.fedclient = gomatrixserverlib.NewFederationClient(
+				s.config.Matrix.ServerName, serverKeyID, testPriv,
+				gomatrixserverlib.WithTransport(transport),
+			)
+
+			// Finally, build the server key APIs.
+			sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
+			s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
 		}
 
-		// Create a new cache but don't enable prometheus!
+		// Now that we have built our server key APIs, start the
-		s.cache, err = caching.NewInMemoryLRUCache(false)
+		// rest of the tests.
-		if err != nil {
+		return m.Run()
-			panic("can't create cache: " + err.Error())
+	}())
-		}
-
-		// Create a temporary directory for JetStream.
-		d, err := ioutil.TempDir("./", "jetstream*")
-		if err != nil {
-			panic(err)
-		}
-		defer os.RemoveAll(d)
-
-		// Draw up just enough Dendrite config for the server key
-		// API to work.
-		cfg := &config.Dendrite{}
-		cfg.Defaults(true)
-		cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
-		cfg.Global.PrivateKey = testPriv
-		cfg.Global.JetStream.InMemory = true
-		cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
-		cfg.Global.JetStream.StoragePath = config.Path(d)
-		cfg.Global.KeyID = serverKeyID
-		cfg.Global.KeyValidityPeriod = s.validity
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
-		s.config = &cfg.FederationAPI
-
-		// Create a transport which redirects federation requests to
-		// the mock round tripper. Since we're not *really* listening for
-		// federation requests then this will return the key instead.
-		transport := &http.Transport{}
-		transport.RegisterProtocol("matrix", &MockRoundTripper{})
-
-		// Create the federation client.
-		s.fedclient = gomatrixserverlib.NewFederationClient(
-			s.config.Matrix.ServerName, serverKeyID, testPriv,
-			gomatrixserverlib.WithTransport(transport),
-		)
-
-		// Finally, build the server key APIs.
-		sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
-		s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
-	}
-
-	// Now that we have built our server key APIs, start the
-	// rest of the tests.
-	os.Exit(m.Run())
 }
 
 type MockRoundTripper struct{}

federationapi/federationapi_test.go

@@ -3,18 +3,250 @@ package federationapi_test
 import (
 	"context"
 	"crypto/ed25519"
+	"encoding/json"
+	"fmt"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/internal"
-	"github.com/matrix-org/dendrite/internal/test"
+	keyapi "github.com/matrix-org/dendrite/keyserver/api"
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
 )
 
+type fedRoomserverAPI struct {
+	rsapi.FederationRoomserverAPI
+	inputRoomEvents   func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse)
+	queryRoomsForUser func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error
+}
+
+// PerformJoin will call this function
+func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
+	if f.inputRoomEvents == nil {
+		return
+	}
+	f.inputRoomEvents(ctx, req, res)
+}
+
+// keychange consumer calls this
+func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
+	if f.queryRoomsForUser == nil {
+		return nil
+	}
+	return f.queryRoomsForUser(ctx, req, res)
+}
+
+// TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate
+type fedClient struct {
+	api.FederationClient
+	allowJoins []*test.Room
+	keys       map[gomatrixserverlib.ServerName]struct {
+		key   ed25519.PrivateKey
+		keyID gomatrixserverlib.KeyID
+	}
+	t       *testing.T
+	sentTxn bool
+}
+
+func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) {
+	fmt.Println("GetServerKeys:", matrixServer)
+	var keys gomatrixserverlib.ServerKeys
+	var keyID gomatrixserverlib.KeyID
+	var pkey ed25519.PrivateKey
+	for srv, data := range f.keys {
+		if srv == matrixServer {
+			pkey = data.key
+			keyID = data.keyID
+			break
+		}
+	}
+	if pkey == nil {
+		return keys, nil
+	}
+
+	keys.ServerName = matrixServer
+	keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(time.Now().Add(10 * time.Hour))
+	publicKey := pkey.Public().(ed25519.PublicKey)
+	keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
+		keyID: {
+			Key: gomatrixserverlib.Base64Bytes(publicKey),
+		},
+	}
+	toSign, err := json.Marshal(keys.ServerKeyFields)
+	if err != nil {
+		return keys, err
+	}
+
+	keys.Raw, err = gomatrixserverlib.SignJSON(
+		string(matrixServer), keyID, pkey, toSign,
+	)
+	if err != nil {
+		return keys, err
+	}
+
+	return keys, nil
+}
+
+func (f *fedClient) MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) {
+	for _, r := range f.allowJoins {
+		if r.ID == roomID {
+			res.RoomVersion = r.Version
+			res.JoinEvent = gomatrixserverlib.EventBuilder{
+				Sender:     userID,
+				RoomID:     roomID,
+				Type:       "m.room.member",
+				StateKey:   &userID,
+				Content:    gomatrixserverlib.RawJSON([]byte(`{"membership":"join"}`)),
+				PrevEvents: r.ForwardExtremities(),
+			}
+			var needed gomatrixserverlib.StateNeeded
+			needed, err = gomatrixserverlib.StateNeededForEventBuilder(&res.JoinEvent)
+			if err != nil {
+				f.t.Errorf("StateNeededForEventBuilder: %v", err)
+				return
+			}
+			res.JoinEvent.AuthEvents = r.MustGetAuthEventRefsForEvent(f.t, needed)
+			return
+		}
+	}
+	return
+}
+func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
+	for _, r := range f.allowJoins {
+		if r.ID == event.RoomID() {
+			r.InsertEvent(f.t, event.Headered(r.Version))
+			f.t.Logf("Join event: %v", event.EventID())
+			res.StateEvents = gomatrixserverlib.NewEventJSONsFromHeaderedEvents(r.CurrentState())
+			res.AuthEvents = gomatrixserverlib.NewEventJSONsFromHeaderedEvents(r.Events())
+		}
+	}
+	return
+}
+
+func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
+	for _, edu := range t.EDUs {
+		if edu.Type == gomatrixserverlib.MDeviceListUpdate {
+			f.sentTxn = true
+		}
+	}
+	f.t.Logf("got /send")
+	return
+}
+
+// Regression test to make sure that /send_join is updating the destination hosts synchronously and
+// isn't relying on the roomserver.
+func TestFederationAPIJoinThenKeyUpdate(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		testFederationAPIJoinThenKeyUpdate(t, dbType)
+	})
+}
+
+func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
+	base, close := testrig.CreateBaseDendrite(t, dbType)
+	base.Cfg.FederationAPI.PreferDirectFetch = true
+	defer close()
+	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+
+	serverA := gomatrixserverlib.ServerName("server.a")
+	serverAKeyID := gomatrixserverlib.KeyID("ed25519:servera")
+	serverAPrivKey := test.PrivateKeyA
+	creator := test.NewUser(t, test.WithSigningServer(serverA, serverAKeyID, serverAPrivKey))
+
+	myServer := base.Cfg.Global.ServerName
+	myServerKeyID := base.Cfg.Global.KeyID
+	myServerPrivKey := base.Cfg.Global.PrivateKey
+	joiningUser := test.NewUser(t, test.WithSigningServer(myServer, myServerKeyID, myServerPrivKey))
+	fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID)
+	room := test.NewRoom(t, creator)
+
+	rsapi := &fedRoomserverAPI{
+		inputRoomEvents: func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
+			if req.Asynchronous {
+				t.Errorf("InputRoomEvents from PerformJoin MUST be synchronous")
+			}
+		},
+		queryRoomsForUser: func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
+			if req.UserID == joiningUser.ID && req.WantMembership == "join" {
+				res.RoomIDs = []string{room.ID}
+				return nil
+			}
+			return fmt.Errorf("unexpected queryRoomsForUser: %+v", *req)
+		},
+	}
+	fc := &fedClient{
+		allowJoins: []*test.Room{room},
+		t:          t,
+		keys: map[gomatrixserverlib.ServerName]struct {
+			key   ed25519.PrivateKey
+			keyID gomatrixserverlib.KeyID
+		}{
+			serverA: {
+				key:   serverAPrivKey,
+				keyID: serverAKeyID,
+			},
+			myServer: {
+				key:   myServerPrivKey,
+				keyID: myServerKeyID,
+			},
+		},
+	}
+	fsapi := federationapi.NewInternalAPI(base, fc, rsapi, base.Caches, nil, false)
+
+	var resp api.PerformJoinResponse
+	fsapi.PerformJoin(context.Background(), &api.PerformJoinRequest{
+		RoomID:      room.ID,
+		UserID:      joiningUser.ID,
+		ServerNames: []gomatrixserverlib.ServerName{serverA},
+	}, &resp)
+	if resp.JoinedVia != serverA {
+		t.Errorf("PerformJoin: joined via %v want %v", resp.JoinedVia, serverA)
+	}
+	if resp.LastError != nil {
+		t.Fatalf("PerformJoin: returned error: %+v", *resp.LastError)
+	}
+
+	// Inject a keyserver key change event and ensure we try to send it out. If we don't, then the
+	// federationapi is incorrectly waiting for an output room event to arrive to update the joined
+	// hosts table.
+	key := keyapi.DeviceMessage{
+		Type: keyapi.TypeDeviceKeyUpdate,
+		DeviceKeys: &keyapi.DeviceKeys{
+			UserID:      joiningUser.ID,
+			DeviceID:    "MY_DEVICE",
+			DisplayName: "BLARGLE",
+			KeyJSON:     []byte(`{}`),
+		},
+	}
+	b, err := json.Marshal(key)
+	if err != nil {
+		t.Fatalf("Failed to marshal device message: %s", err)
+	}
+
+	msg := &nats.Msg{
+		Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
+		Header:  nats.Header{},
+		Data:    b,
+	}
+	msg.Header.Set(jetstream.UserID, key.UserID)
+
+	testrig.MustPublishMsgs(t, jsctx, msg)
+	time.Sleep(500 * time.Millisecond)
+	if !fc.sentTxn {
+		t.Fatalf("did not send device list update")
+	}
+}
+
 // Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404.
 // Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated.
 func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
@@ -86,7 +318,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 		}
 		gerr, ok := err.(gomatrix.HTTPError)
 		if !ok {
-			t.Errorf("failed to cast response error as gomatrix.HTTPError")
+			t.Errorf("failed to cast response error as gomatrix.HTTPError: %s", err)
 			continue
 		}
 		t.Logf("Error: %+v", gerr)

federationapi/internal/api.go

@@ -25,8 +25,8 @@ type FederationInternalAPI struct {
 	db         storage.Database
 	cfg        *config.FederationAPI
 	statistics *statistics.Statistics
-	rsAPI      roomserverAPI.RoomserverInternalAPI
+	rsAPI      roomserverAPI.FederationRoomserverAPI
-	federation *gomatrixserverlib.FederationClient
+	federation api.FederationClient
 	keyRing    *gomatrixserverlib.KeyRing
 	queues     *queue.OutgoingQueues
 	joins      sync.Map // joins currently in progress
@@ -34,8 +34,8 @@ type FederationInternalAPI struct {
 
 func NewFederationInternalAPI(
 	db storage.Database, cfg *config.FederationAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.FederationRoomserverAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation api.FederationClient,
 	statistics *statistics.Statistics,
 	caches *caching.Caches,
 	queues *queue.OutgoingQueues,

federationapi/internal/perform.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/consumers"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrix"
@@ -165,7 +166,8 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	if content == nil {
 		content = map[string]interface{}{}
 	}
-	content["membership"] = "join"
+	_ = json.Unmarshal(respMakeJoin.JoinEvent.Content, &content)
+	content["membership"] = gomatrixserverlib.Join
 	if err = respMakeJoin.JoinEvent.SetContent(content); err != nil {
 		return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err)
 	}
@@ -208,10 +210,22 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	}
 	r.statistics.ForServer(serverName).Success()
 
-	authEvents := respSendJoin.AuthEvents.UntrustedEvents(respMakeJoin.RoomVersion)
+	// If the remote server returned an event in the "event" key of
+	// the send_join request then we should use that instead. It may
+	// contain signatures that we don't know about.
+	if len(respSendJoin.Event) > 0 {
+		var remoteEvent *gomatrixserverlib.Event
+		remoteEvent, err = respSendJoin.Event.UntrustedEvent(respMakeJoin.RoomVersion)
+		if err == nil && isWellFormedMembershipEvent(
+			remoteEvent, roomID, userID, r.cfg.Matrix.ServerName,
+		) {
+			event = remoteEvent
+		}
+	}
 
 	// Sanity-check the join response to ensure that it has a create
 	// event, that the room version is known, etc.
+	authEvents := respSendJoin.AuthEvents.UntrustedEvents(respMakeJoin.RoomVersion)
 	if err = sanityCheckAuthChain(authEvents); err != nil {
 		return fmt.Errorf("sanityCheckAuthChain: %w", err)
 	}
@@ -235,6 +249,21 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		return fmt.Errorf("respSendJoin.Check: %w", err)
 	}
 
+	// We need to immediately update our list of joined hosts for this room now as we are technically
+	// joined. We must do this synchronously: we cannot rely on the roomserver output events as they
+	// will happen asyncly. If we don't update this table, you can end up with bad failure modes like
+	// joining a room, waiting for 200 OK then changing device keys and have those keys not be sent
+	// to other servers (this was a cause of a flakey sytest "Local device key changes get to remote servers")
+	// The events are trusted now as we performed auth checks above.
+	joinedHosts, err := consumers.JoinedHostsFromEvents(respState.StateEvents.TrustedEvents(respMakeJoin.RoomVersion, false))
+	if err != nil {
+		return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err)
+	}
+	logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts")
+	if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil {
+		return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err)
+	}
+
 	// If we successfully performed a send_join above then the other
 	// server now thinks we're a part of the room. Send the newly
 	// returned state to the roomserver to update our local view.
@@ -254,6 +283,26 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	return nil
 }
 
+// isWellFormedMembershipEvent returns true if the event looks like a legitimate
+// membership event.
+func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string, origin gomatrixserverlib.ServerName) bool {
+	if membership, err := event.Membership(); err != nil {
+		return false
+	} else if membership != gomatrixserverlib.Join {
+		return false
+	}
+	if event.RoomID() != roomID {
+		return false
+	}
+	if event.Origin() != origin {
+		return false
+	}
+	if !event.StateKeyEquals(userID) {
+		return false
+	}
+	return true
+}
+
 // PerformOutboundPeekRequest implements api.FederationInternalAPI
 func (r *FederationInternalAPI) PerformOutboundPeek(
 	ctx context.Context,
@@ -650,7 +699,7 @@ func setDefaultRoomVersionFromJoinEvent(joinEvent gomatrixserverlib.EventBuilder
 
 // FederatedAuthProvider is an auth chain provider which fetches events from the server provided
 func federatedAuthProvider(
-	ctx context.Context, federation *gomatrixserverlib.FederationClient,
+	ctx context.Context, federation api.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier, server gomatrixserverlib.ServerName,
 ) gomatrixserverlib.AuthChainProvider {
 	// A list of events that we have retried, if they were not included in

federationapi/queue/destinationqueue.go

@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	fedapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
@@ -49,21 +50,21 @@ type destinationQueue struct {
 	db                 storage.Database
 	process            *process.ProcessContext
 	signing            *SigningInfo
-	rsAPI              api.RoomserverInternalAPI
+	rsAPI              api.FederationRoomserverAPI
-	client             *gomatrixserverlib.FederationClient // federation client
+	client             fedapi.FederationClient         // federation client
-	origin             gomatrixserverlib.ServerName        // origin of requests
+	origin             gomatrixserverlib.ServerName    // origin of requests
-	destination        gomatrixserverlib.ServerName        // destination of requests
+	destination        gomatrixserverlib.ServerName    // destination of requests
-	running            atomic.Bool                         // is the queue worker running?
+	running            atomic.Bool                     // is the queue worker running?
-	backingOff         atomic.Bool                         // true if we're backing off
+	backingOff         atomic.Bool                     // true if we're backing off
-	overflowed         atomic.Bool                         // the queues exceed maxPDUsInMemory/maxEDUsInMemory, so we should consult the database for more
+	overflowed         atomic.Bool                     // the queues exceed maxPDUsInMemory/maxEDUsInMemory, so we should consult the database for more
-	statistics         *statistics.ServerStatistics        // statistics about this remote server
+	statistics         *statistics.ServerStatistics    // statistics about this remote server
-	transactionIDMutex sync.Mutex                          // protects transactionID
+	transactionIDMutex sync.Mutex                      // protects transactionID
-	transactionID      gomatrixserverlib.TransactionID     // last transaction ID if retrying, or "" if last txn was successful
+	transactionID      gomatrixserverlib.TransactionID // last transaction ID if retrying, or "" if last txn was successful
-	notify             chan struct{}                       // interrupts idle wait pending PDUs/EDUs
+	notify             chan struct{}                   // interrupts idle wait pending PDUs/EDUs
-	pendingPDUs        []*queuedPDU                        // PDUs waiting to be sent
+	pendingPDUs        []*queuedPDU                    // PDUs waiting to be sent
-	pendingEDUs        []*queuedEDU                        // EDUs waiting to be sent
+	pendingEDUs        []*queuedEDU                    // EDUs waiting to be sent
-	pendingMutex       sync.RWMutex                        // protects pendingPDUs and pendingEDUs
+	pendingMutex       sync.RWMutex                    // protects pendingPDUs and pendingEDUs
-	interruptBackoff   chan bool                           // interrupts backoff
+	interruptBackoff   chan bool                       // interrupts backoff
 }
 
 // Send event adds the event to the pending queue for the destination.

federationapi/queue/queue.go

@@ -26,6 +26,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
 
+	fedapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
@@ -39,9 +40,9 @@ type OutgoingQueues struct {
 	db          storage.Database
 	process     *process.ProcessContext
 	disabled    bool
-	rsAPI       api.RoomserverInternalAPI
+	rsAPI       api.FederationRoomserverAPI
 	origin      gomatrixserverlib.ServerName
-	client      *gomatrixserverlib.FederationClient
+	client      fedapi.FederationClient
 	statistics  *statistics.Statistics
 	signing     *SigningInfo
 	queuesMutex sync.Mutex // protects the below
@@ -85,8 +86,8 @@ func NewOutgoingQueues(
 	process *process.ProcessContext,
 	disabled bool,
 	origin gomatrixserverlib.ServerName,
-	client *gomatrixserverlib.FederationClient,
+	client fedapi.FederationClient,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	statistics *statistics.Statistics,
 	signing *SigningInfo,
 ) *OutgoingQueues {

federationapi/routing/join.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"sort"
@@ -103,6 +104,16 @@ func MakeJoin(
 		}
 	}
 
+	// Check if the restricted join is allowed. If the room doesn't
+	// support restricted joins then this is effectively a no-op.
+	res, authorisedVia, err := checkRestrictedJoin(httpReq, rsAPI, verRes.RoomVersion, roomID, userID)
+	if err != nil {
+		util.GetLogger(httpReq.Context()).WithError(err).Error("checkRestrictedJoin failed")
+		return jsonerror.InternalServerError()
+	} else if res != nil {
+		return *res
+	}
+
 	// Try building an event for the server
 	builder := gomatrixserverlib.EventBuilder{
 		Sender:   userID,
@@ -110,8 +121,11 @@ func MakeJoin(
 		Type:     "m.room.member",
 		StateKey: &userID,
 	}
-	err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Join})
+	content := gomatrixserverlib.MemberContent{
-	if err != nil {
+		Membership:    gomatrixserverlib.Join,
+		AuthorisedVia: authorisedVia,
+	}
+	if err = builder.SetContent(content); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("builder.SetContent failed")
 		return jsonerror.InternalServerError()
 	}
@@ -161,6 +175,7 @@ func MakeJoin(
 // SendJoin implements the /send_join API
 // The make-join send-join dance makes much more sense as a single
 // flow so the cyclomatic complexity is high:
+// nolint:gocyclo
 func SendJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
@@ -314,6 +329,40 @@ func SendJoin(
 		}
 	}
 
+	// If the membership content contains a user ID for a server that is not
+	// ours then we should kick it back.
+	var memberContent gomatrixserverlib.MemberContent
+	if err := json.Unmarshal(event.Content(), &memberContent); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	}
+	if memberContent.AuthorisedVia != "" {
+		_, domain, err := gomatrixserverlib.SplitID('@', memberContent.AuthorisedVia)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON(fmt.Sprintf("The authorising username %q is invalid.", memberContent.AuthorisedVia)),
+			}
+		}
+		if domain != cfg.Matrix.ServerName {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON(fmt.Sprintf("The authorising username %q does not belong to this server.", memberContent.AuthorisedVia)),
+			}
+		}
+	}
+
+	// Sign the membership event. This is required for restricted joins to work
+	// in the case that the authorised via user is one of our own users. It also
+	// doesn't hurt to do it even if it isn't a restricted join.
+	signed := event.Sign(
+		string(cfg.Matrix.ServerName),
+		cfg.Matrix.KeyID,
+		cfg.Matrix.PrivateKey,
+	)
+
 	// Send the events to the room server.
 	// We are responsible for notifying other servers that the user has joined
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
@@ -323,7 +372,7 @@ func SendJoin(
 			InputRoomEvents: []api.InputRoomEvent{
 				{
 					Kind:          api.KindNew,
-					Event:         event.Headered(stateAndAuthChainResponse.RoomVersion),
+					Event:         signed.Headered(stateAndAuthChainResponse.RoomVersion),
 					SendAsServer:  string(cfg.Matrix.ServerName),
 					TransactionID: nil,
 				},
@@ -354,10 +403,77 @@ func SendJoin(
 			StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.StateEvents),
 			AuthEvents:  gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.AuthChainEvents),
 			Origin:      cfg.Matrix.ServerName,
+			Event:       signed.JSON(),
 		},
 	}
 }
 
+// checkRestrictedJoin finds out whether or not we can assist in processing
+// a restricted room join. If the room version does not support restricted
+// joins then this function returns with no side effects. This returns three
+// values:
+// * an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which
+//   should always be sent back to the client if one is specified
+// * a user ID of an authorising user, typically a user that has power to
+//   issue invites in the room, if one has been found
+// * an error if there was a problem finding out if this was allowable,
+//   like if the room version isn't known or a problem happened talking to
+//   the roomserver
+func checkRestrictedJoin(
+	httpReq *http.Request,
+	rsAPI api.FederationRoomserverAPI,
+	roomVersion gomatrixserverlib.RoomVersion,
+	roomID, userID string,
+) (*util.JSONResponse, string, error) {
+	if allowRestricted, err := roomVersion.MayAllowRestrictedJoinsInEventAuth(); err != nil {
+		return nil, "", err
+	} else if !allowRestricted {
+		return nil, "", nil
+	}
+	req := &api.QueryRestrictedJoinAllowedRequest{
+		RoomID: roomID,
+		UserID: userID,
+	}
+	res := &api.QueryRestrictedJoinAllowedResponse{}
+	if err := rsAPI.QueryRestrictedJoinAllowed(httpReq.Context(), req, res); err != nil {
+		return nil, "", err
+	}
+
+	switch {
+	case !res.Restricted:
+		// The join rules for the room don't restrict membership.
+		return nil, "", nil
+
+	case !res.Resident:
+		// The join rules restrict membership but our server isn't currently
+		// joined to all of the allowed rooms, so we can't actually decide
+		// whether or not to allow the user to join. This error code should
+		// tell the joining server to try joining via another resident server
+		// instead.
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnableToAuthoriseJoin("This server cannot authorise the join."),
+		}, "", nil
+
+	case !res.Allowed:
+		// The join rules restrict membership, our server is in the relevant
+		// rooms and the user wasn't joined to join any of the allowed rooms
+		// and therefore can't join this room.
+		return &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You are not joined to any matching rooms."),
+		}, "", nil
+
+	default:
+		// The join rules restrict membership, our server is in the relevant
+		// rooms and the user was allowed to join because they belong to one
+		// of the allowed rooms. We now need to pick one of our own local users
+		// from within the room to use as the authorising user ID, so that it
+		// can be referred to from within the membership content.
+		return nil, res.AuthorisedVia, nil
+	}
+}
+
 type eventsByDepth []*gomatrixserverlib.HeaderedEvent
 
 func (e eventsByDepth) Len() int {

federationapi/routing/query.go

@@ -30,7 +30,7 @@ import (
 // RoomAliasToID converts the queried alias into a room ID and returns it
 func RoomAliasToID(
 	httpReq *http.Request,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 	cfg *config.FederationAPI,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	senderAPI federationAPI.FederationInternalAPI,

federationapi/routing/routing.go

@@ -54,7 +54,7 @@ func Setup(
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	fsAPI *fedInternal.FederationInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 	userAPI userapi.FederationUserAPI,
 	keyAPI keyserverAPI.FederationKeyAPI,
 	mscCfg *config.MSCs,

federationapi/routing/send.go

@@ -85,7 +85,7 @@ func Send(
 	rsAPI api.FederationRoomserverAPI,
 	keyAPI keyapi.FederationKeyAPI,
 	keys gomatrixserverlib.JSONVerifier,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 	mu *internal.MutexByRoom,
 	servers federationAPI.ServersInRoomProvider,
 	producer *producers.SyncAPIProducer,

federationapi/routing/send_test.go

@@ -8,8 +8,8 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 

federationapi/routing/threepid.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -57,7 +58,7 @@ var (
 func CreateInvitesFrom3PIDInvites(
 	req *http.Request, rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 	userAPI userapi.FederationUserAPI,
 ) util.JSONResponse {
 	var body invites
@@ -107,7 +108,7 @@ func ExchangeThirdPartyInvite(
 	roomID string,
 	rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 ) util.JSONResponse {
 	var builder gomatrixserverlib.EventBuilder
 	if err := json.Unmarshal(request.Content(), &builder); err != nil {
@@ -165,7 +166,12 @@ func ExchangeThirdPartyInvite(
 
 	// Ask the requesting server to sign the newly created event so we know it
 	// acknowledged it
-	signedEvent, err := federation.SendInvite(httpReq.Context(), request.Origin(), event)
+	inviteReq, err := gomatrixserverlib.NewInviteV2Request(event.Headered(verRes.RoomVersion), nil)
+	if err != nil {
+		util.GetLogger(httpReq.Context()).WithError(err).Error("failed to make invite v2 request")
+		return jsonerror.InternalServerError()
+	}
+	signedEvent, err := federation.SendInviteV2(httpReq.Context(), request.Origin(), inviteReq)
 	if err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("federation.SendInvite failed")
 		return jsonerror.InternalServerError()
@@ -205,7 +211,7 @@ func ExchangeThirdPartyInvite(
 func createInviteFrom3PIDInvite(
 	ctx context.Context, rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
-	inv invite, federation *gomatrixserverlib.FederationClient,
+	inv invite, federation federationAPI.FederationClient,
 	userAPI userapi.FederationUserAPI,
 ) (*gomatrixserverlib.Event, error) {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
@@ -335,7 +341,7 @@ func buildMembershipEvent(
 // them responded with an error.
 func sendToRemoteServer(
 	ctx context.Context, inv invite,
-	federation *gomatrixserverlib.FederationClient, _ *config.FederationAPI,
+	federation federationAPI.FederationClient, _ *config.FederationAPI,
 	builder gomatrixserverlib.EventBuilder,
 ) (err error) {
 	remoteServers := make([]gomatrixserverlib.ServerName, 2)

federationapi/storage/interface.go

@@ -25,13 +25,12 @@ import (
 type Database interface {
 	gomatrixserverlib.KeyDatabase
 
-	UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error)
+	UpdateRoom(ctx context.Context, roomID string, addHosts []types.JoinedHost, removeHosts []string, purgeRoomFirst bool) (joinedHosts []types.JoinedHost, err error)
 
 	GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error)
 	GetAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error)
 	// GetJoinedHostsForRooms returns the complete set of servers in the rooms given.
 	GetJoinedHostsForRooms(ctx context.Context, roomIDs []string, excludeSelf bool) ([]gomatrixserverlib.ServerName, error)
-	PurgeRoomState(ctx context.Context, roomID string) error
 
 	StoreJSON(ctx context.Context, js string) (*shared.Receipt, error)
 

federationapi/storage/shared/storage.go

@@ -63,11 +63,21 @@ func (r *Receipt) String() string {
 // this isn't a duplicate message.
 func (d *Database) UpdateRoom(
 	ctx context.Context,
-	roomID, oldEventID, newEventID string,
+	roomID string,
 	addHosts []types.JoinedHost,
 	removeHosts []string,
+	purgeRoomFirst bool,
 ) (joinedHosts []types.JoinedHost, err error) {
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		if purgeRoomFirst {
+			// If the event is a create event then we'll delete all of the existing
+			// data for the room. The only reason that a create event would be replayed
+			// to us in this way is if we're about to receive the entire room state.
+			if err = d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
+				return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err)
+			}
+		}
+
 		joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID)
 		if err != nil {
 			return err
@@ -138,20 +148,6 @@ func (d *Database) StoreJSON(
 	}, nil
 }
 
-func (d *Database) PurgeRoomState(
-	ctx context.Context, roomID string,
-) error {
-	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		// If the event is a create event then we'll delete all of the existing
-		// data for the room. The only reason that a create event would be replayed
-		// to us in this way is if we're about to receive the entire room state.
-		if err := d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
-			return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err)
-		}
-		return nil
-	})
-}
-
 func (d *Database) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.FederationBlacklist.InsertBlacklist(context.TODO(), txn, serverName)

go.mod

@@ -1,22 +1,29 @@
 module github.com/matrix-org/dendrite
 
-replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9
+replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f
 
 replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e
 
 require (
-	github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0
+	github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98
 	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/DATA-DOG/go-sqlmock v1.5.0
+	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
+	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/codeclysm/extract v2.2.0+incompatible
-	github.com/docker/docker v20.10.14+incompatible
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.16+incompatible
 	github.com/docker/go-connections v0.4.0
-	github.com/ethereum/go-ethereum v1.10.17
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/ethereum/go-ethereum v1.10.18
+	github.com/frankban/quicktest v1.14.3 // indirect
 	github.com/getsentry/sentry-go v0.13.0
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gologme/log v1.3.0
-	github.com/google/go-cmp v0.5.7
+	github.com/google/go-cmp v0.5.8
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
@@ -28,94 +35,48 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
-	github.com/mattn/go-sqlite3 v1.14.10
+	github.com/mattn/go-sqlite3 v1.14.13
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/miekg/dns v1.1.49 // indirect
+	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb
 	github.com/nats-io/nats.go v1.14.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
-	github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31
+	github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
+	github.com/onsi/gomega v1.17.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pressly/goose v2.7.0+incompatible
-	github.com/prometheus/client_golang v1.12.1
+	github.com/prometheus/client_golang v1.12.2
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.7.1
 	github.com/tidwall/gjson v1.14.1
 	github.com/tidwall/sjson v1.2.4
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.3
 	go.uber.org/atomic v1.9.0
-	golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
-	golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
+	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
-	golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2
+	golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
-	golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3
+	golang.org/x/net v0.0.0-20220524220425-1d687d428aca
-	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0 // indirect
+	gotest.tools/v3 v3.0.3 // indirect
 	nhooyr.io/websocket v1.8.7
 )
 
-require (
+go 1.16
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
-	github.com/Microsoft/go-winio v0.5.1 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/btcsuite/btcd/btcec/v2 v2.1.2 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/cheekybits/genny v1.0.0 // indirect
-	github.com/containerd/containerd v1.6.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
-	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/frankban/quicktest v1.14.3 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
-	github.com/klauspost/compress v1.14.4 // indirect
-	github.com/lucas-clemente/quic-go v0.26.0 // indirect
-	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/miekg/dns v1.1.31 // indirect
-	github.com/minio/highwayhash v1.0.2 // indirect
-	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
-	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a // indirect
-	github.com/nats-io/nkeys v0.3.0 // indirect
-	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
-	github.com/onsi/gomega v1.15.0 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.2 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
-	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
-	golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
-	google.golang.org/grpc v1.43.0 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/macaroon.v2 v2.1.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-)
-
-go 1.17

internal/caching/cache_typing_test.go

@@ -20,7 +20,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/matrix-org/dendrite/internal/test"
+	"github.com/matrix-org/dendrite/test"
 )
 
 func TestEDUCache(t *testing.T) {

internal/sqlutil/trace.go

@@ -37,7 +37,7 @@ type traceInterceptor struct {
 	sqlmw.NullInterceptor
 }
 
-func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (driver.Rows, error) {
+func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (context.Context, driver.Rows, error) {
 	startedAt := time.Now()
 	rows, err := stmt.QueryContext(ctx, args)
 
@@ -45,7 +45,7 @@ func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.St
 
 	logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args)
 
-	return rows, err
+	return ctx, rows, err
 }
 
 func (in *traceInterceptor) StmtExecContext(ctx context.Context, stmt driver.StmtExecContext, query string, args []driver.NamedValue) (driver.Result, error) {

internal/test/client.go

@@ -1,158 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"crypto/tls"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// Request contains the information necessary to issue a request and test its result
-type Request struct {
-	Req              *http.Request
-	WantedBody       string
-	WantedStatusCode int
-	LastErr          *LastRequestErr
-}
-
-// LastRequestErr is a synchronised error wrapper
-// Useful for obtaining the last error from a set of requests
-type LastRequestErr struct {
-	sync.Mutex
-	Err error
-}
-
-// Set sets the error
-func (r *LastRequestErr) Set(err error) {
-	r.Lock()
-	defer r.Unlock()
-	r.Err = err
-}
-
-// Get gets the error
-func (r *LastRequestErr) Get() error {
-	r.Lock()
-	defer r.Unlock()
-	return r.Err
-}
-
-// CanonicalJSONInput canonicalises a slice of JSON strings
-// Useful for test input
-func CanonicalJSONInput(jsonData []string) []string {
-	for i := range jsonData {
-		jsonBytes, err := gomatrixserverlib.CanonicalJSON([]byte(jsonData[i]))
-		if err != nil && err != io.EOF {
-			panic(err)
-		}
-		jsonData[i] = string(jsonBytes)
-	}
-	return jsonData
-}
-
-// Do issues a request and checks the status code and body of the response
-func (r *Request) Do() (err error) {
-	client := &http.Client{
-		Timeout: 5 * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
-		},
-	}
-	res, err := client.Do(r.Req)
-	if err != nil {
-		return err
-	}
-	defer (func() { err = res.Body.Close() })()
-
-	if res.StatusCode != r.WantedStatusCode {
-		return fmt.Errorf("incorrect status code. Expected: %d  Got: %d", r.WantedStatusCode, res.StatusCode)
-	}
-
-	if r.WantedBody != "" {
-		resBytes, err := ioutil.ReadAll(res.Body)
-		if err != nil {
-			return err
-		}
-		jsonBytes, err := gomatrixserverlib.CanonicalJSON(resBytes)
-		if err != nil {
-			return err
-		}
-		if string(jsonBytes) != r.WantedBody {
-			return fmt.Errorf("returned wrong bytes. Expected:\n%s\n\nGot:\n%s", r.WantedBody, string(jsonBytes))
-		}
-	}
-
-	return nil
-}
-
-// DoUntilSuccess blocks and repeats the same request until the response returns the desired status code and body.
-// It then closes the given channel and returns.
-func (r *Request) DoUntilSuccess(done chan error) {
-	r.LastErr = &LastRequestErr{}
-	for {
-		if err := r.Do(); err != nil {
-			r.LastErr.Set(err)
-			time.Sleep(1 * time.Second) // don't tightloop
-			continue
-		}
-		close(done)
-		return
-	}
-}
-
-// Run repeatedly issues a request until success, error or a timeout is reached
-func (r *Request) Run(label string, timeout time.Duration, serverCmdChan chan error) {
-	fmt.Printf("==TESTING== %v (timeout: %v)\n", label, timeout)
-	done := make(chan error, 1)
-
-	// We need to wait for the server to:
-	// - have connected to the database
-	// - have created the tables
-	// - be listening on the given port
-	go r.DoUntilSuccess(done)
-
-	// wait for one of:
-	// - the test to pass (done channel is closed)
-	// - the server to exit with an error (error sent on serverCmdChan)
-	// - our test timeout to expire
-	// We don't need to clean up since the main() function handles that in the event we panic
-	select {
-	case <-time.After(timeout):
-		fmt.Printf("==TESTING== %v TIMEOUT\n", label)
-		if reqErr := r.LastErr.Get(); reqErr != nil {
-			fmt.Println("Last /sync request error:")
-			fmt.Println(reqErr)
-		}
-		panic(fmt.Sprintf("%v server timed out", label))
-	case err := <-serverCmdChan:
-		if err != nil {
-			fmt.Println("=============================================================================================")
-			fmt.Printf("%v server failed to run. If failing with 'pq: password authentication failed for user' try:", label)
-			fmt.Println("    export PGHOST=/var/run/postgresql")
-			fmt.Println("=============================================================================================")
-			panic(err)
-		}
-	case <-done:
-		fmt.Printf("==TESTING== %v PASSED\n", label)
-	}
-}

internal/test/kafka.go

@@ -1,76 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"io"
-	"os/exec"
-	"path/filepath"
-	"strings"
-)
-
-// KafkaExecutor executes kafka scripts.
-type KafkaExecutor struct {
-	// The location of Zookeeper. Typically this is `localhost:2181`.
-	ZookeeperURI string
-	// The directory where Kafka is installed to. Used to locate kafka scripts.
-	KafkaDirectory string
-	// The location of the Kafka logs. Typically this is `localhost:9092`.
-	KafkaURI string
-	// Where stdout and stderr should be written to. Typically this is `os.Stderr`.
-	OutputWriter io.Writer
-}
-
-// CreateTopic creates a new kafka topic. This is created with a single partition.
-func (e *KafkaExecutor) CreateTopic(topic string) error {
-	cmd := exec.Command(
-		filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
-		"--create",
-		"--zookeeper", e.ZookeeperURI,
-		"--replication-factor", "1",
-		"--partitions", "1",
-		"--topic", topic,
-	)
-	cmd.Stdout = e.OutputWriter
-	cmd.Stderr = e.OutputWriter
-	return cmd.Run()
-}
-
-// WriteToTopic writes data to a kafka topic.
-func (e *KafkaExecutor) WriteToTopic(topic string, data []string) error {
-	cmd := exec.Command(
-		filepath.Join(e.KafkaDirectory, "bin", "kafka-console-producer.sh"),
-		"--broker-list", e.KafkaURI,
-		"--topic", topic,
-	)
-	cmd.Stdout = e.OutputWriter
-	cmd.Stderr = e.OutputWriter
-	cmd.Stdin = strings.NewReader(strings.Join(data, "\n"))
-	return cmd.Run()
-}
-
-// DeleteTopic deletes a given kafka topic if it exists.
-func (e *KafkaExecutor) DeleteTopic(topic string) error {
-	cmd := exec.Command(
-		filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
-		"--delete",
-		"--if-exists",
-		"--zookeeper", e.ZookeeperURI,
-		"--topic", topic,
-	)
-	cmd.Stderr = e.OutputWriter
-	cmd.Stdout = e.OutputWriter
-	return cmd.Run()
-}

internal/test/server.go

@@ -1,152 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-// Defaulting allows assignment of string variables with a fallback default value
-// Useful for use with os.Getenv() for example
-func Defaulting(value, defaultValue string) string {
-	if value == "" {
-		value = defaultValue
-	}
-	return value
-}
-
-// CreateDatabase creates a new database, dropping it first if it exists
-func CreateDatabase(command string, args []string, database string) error {
-	cmd := exec.Command(command, args...)
-	cmd.Stdin = strings.NewReader(
-		fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;", database, database),
-	)
-	// Send stdout and stderr to our stderr so that we see error messages from
-	// the psql process
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
-}
-
-// CreateBackgroundCommand creates an executable command
-// The Cmd being executed is returned. A channel is also returned,
-// which will have any termination errors sent down it, followed immediately by the channel being closed.
-func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan error) {
-	cmd := exec.Command(command, args...)
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stderr
-
-	if err := cmd.Start(); err != nil {
-		panic("failed to start server: " + err.Error())
-	}
-	cmdChan := make(chan error, 1)
-	go func() {
-		cmdChan <- cmd.Wait()
-		close(cmdChan)
-	}()
-	return cmd, cmdChan
-}
-
-// InitDatabase creates the database and config file needed for the server to run
-func InitDatabase(postgresDatabase, postgresContainerName string, databases []string) {
-	if len(databases) > 0 {
-		var dbCmd string
-		var dbArgs []string
-		if postgresContainerName == "" {
-			dbCmd = "psql"
-			dbArgs = []string{postgresDatabase}
-		} else {
-			dbCmd = "docker"
-			dbArgs = []string{
-				"exec", "-i", postgresContainerName, "psql", "-U", "postgres", postgresDatabase,
-			}
-		}
-		for _, database := range databases {
-			if err := CreateDatabase(dbCmd, dbArgs, database); err != nil {
-				panic(err)
-			}
-		}
-	}
-}
-
-// StartProxy creates a reverse proxy
-func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
-	proxyArgs := []string{
-		"--bind-address", bindAddr,
-		"--sync-api-server-url", "http://" + string(cfg.SyncAPI.InternalAPI.Connect),
-		"--client-api-server-url", "http://" + string(cfg.ClientAPI.InternalAPI.Connect),
-		"--media-api-server-url", "http://" + string(cfg.MediaAPI.InternalAPI.Connect),
-		"--tls-cert", "server.crt",
-		"--tls-key", "server.key",
-	}
-	return CreateBackgroundCommand(
-		filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"),
-		proxyArgs,
-	)
-}
-
-// ListenAndServe will listen on a random high-numbered port and attach the given router.
-// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed.
-func ListenAndServe(t *testing.T, router http.Handler, useTLS bool) (apiURL string, cancel func()) {
-	listener, err := net.Listen("tcp", ":0")
-	if err != nil {
-		t.Fatalf("failed to listen: %s", err)
-	}
-	port := listener.Addr().(*net.TCPAddr).Port
-	srv := http.Server{}
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		srv.Handler = router
-		var err error
-		if useTLS {
-			certFile := filepath.Join(os.TempDir(), "dendrite.cert")
-			keyFile := filepath.Join(os.TempDir(), "dendrite.key")
-			err = NewTLSKey(keyFile, certFile)
-			if err != nil {
-				t.Logf("failed to generate tls key/cert: %s", err)
-				return
-			}
-			err = srv.ServeTLS(listener, certFile, keyFile)
-		} else {
-			err = srv.Serve(listener)
-		}
-		if err != nil && err != http.ErrServerClosed {
-			t.Logf("Listen failed: %s", err)
-		}
-	}()
-
-	secure := ""
-	if useTLS {
-		secure = "s"
-	}
-	return fmt.Sprintf("http%s://localhost:%d", secure, port), func() {
-		_ = srv.Shutdown(context.Background())
-		wg.Wait()
-	}
-}

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 4
+	VersionPatch = 6
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/internal/device_list_update.go

@@ -84,7 +84,7 @@ type DeviceListUpdater struct {
 	db          DeviceListUpdaterDatabase
 	api         DeviceListUpdaterAPI
 	producer    KeyChangeProducer
-	fedClient   fedsenderapi.FederationClient
+	fedClient   fedsenderapi.KeyserverFederationAPI
 	workerChans []chan gomatrixserverlib.ServerName
 
 	// When device lists are stale for a user, they get inserted into this map with a channel which `Update` will
@@ -127,7 +127,7 @@ type KeyChangeProducer interface {
 // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale.
 func NewDeviceListUpdater(
 	db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer,
-	fedClient fedsenderapi.FederationClient, numWorkers int,
+	fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int,
 ) *DeviceListUpdater {
 	return &DeviceListUpdater{
 		userIDToMutex:  make(map[string]*sync.Mutex),

keyserver/internal/internal.go

@@ -37,7 +37,7 @@ import (
 type KeyInternalAPI struct {
 	DB         storage.Database
 	ThisServer gomatrixserverlib.ServerName
-	FedClient  fedsenderapi.FederationClient
+	FedClient  fedsenderapi.KeyserverFederationAPI
 	UserAPI    userapi.KeyserverUserAPI
 	Producer   *producers.KeyChange
 	Updater    *DeviceListUpdater

keyserver/keyserver.go

@@ -37,7 +37,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) {
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.FederationClient,
+	base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.KeyserverFederationAPI,
 ) api.KeyInternalAPI {
 	js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 

roomserver/api/api.go

@@ -183,6 +183,8 @@ type FederationRoomserverAPI interface {
 	QueryMissingEvents(ctx context.Context, req *QueryMissingEventsRequest, res *QueryMissingEventsResponse) error
 	// Query whether a server is allowed to see an event
 	QueryServerAllowedToSeeEvent(ctx context.Context, req *QueryServerAllowedToSeeEventRequest, res *QueryServerAllowedToSeeEventResponse) error
+	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
+	QueryRestrictedJoinAllowed(ctx context.Context, req *QueryRestrictedJoinAllowedRequest, res *QueryRestrictedJoinAllowedResponse) error
 	PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error
 	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error
 	// Query a given amount (or less) of events prior to a given set of events.

roomserver/api/api_trace.go

@@ -354,6 +354,16 @@ func (t *RoomserverInternalAPITrace) QueryAuthChain(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) QueryRestrictedJoinAllowed(
+	ctx context.Context,
+	request *QueryRestrictedJoinAllowedRequest,
+	response *QueryRestrictedJoinAllowedResponse,
+) error {
+	err := t.Impl.QueryRestrictedJoinAllowed(ctx, request, response)
+	util.GetLogger(ctx).WithError(err).Infof("QueryRestrictedJoinAllowed req=%+v res=%+v", js(request), js(response))
+	return err
+}
+
 func js(thing interface{}) string {
 	b, err := json.Marshal(thing)
 	if err != nil {

roomserver/api/query.go

@@ -348,6 +348,26 @@ type QueryServerBannedFromRoomResponse struct {
 	Banned bool `json:"banned"`
 }
 
+type QueryRestrictedJoinAllowedRequest struct {
+	UserID string `json:"user_id"`
+	RoomID string `json:"room_id"`
+}
+
+type QueryRestrictedJoinAllowedResponse struct {
+	// True if the room membership is restricted by the join rule being set to "restricted"
+	Restricted bool `json:"restricted"`
+	// True if our local server is joined to all of the allowed rooms specified in the "allow"
+	// key of the join rule, false if we are missing from some of them and therefore can't
+	// reliably decide whether or not we can satisfy the join
+	Resident bool `json:"resident"`
+	// True if the restricted join is allowed because we found the membership in one of the
+	// allowed rooms from the join rule, false if not
+	Allowed bool `json:"allowed"`
+	// Contains the user ID of the selected user ID that has power to issue invites, this will
+	// get populated into the "join_authorised_via_users_server" content in the membership
+	AuthorisedVia string `json:"authorised_via,omitempty"`
+}
+
 // MarshalJSON stringifies the room ID and StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
 func (r *QueryBulkStateContentResponse) MarshalJSON() ([]byte, error) {
 	se := make(map[string]string)

roomserver/internal/input/input_events.go

@@ -535,8 +535,6 @@ func (r *Inputer) calculateAndSetState(
 	roomState := state.NewStateResolution(updater, roomInfo)
 
 	if input.HasState {
-		stateAtEvent.Overwrite = true
-
 		// We've been told what the state at the event is so we don't need to calculate it.
 		// Check that those state events are in the database and store the state.
 		var entries []types.StateEntry
@@ -549,8 +547,6 @@ func (r *Inputer) calculateAndSetState(
 			return fmt.Errorf("updater.AddState: %w", err)
 		}
 	} else {
-		stateAtEvent.Overwrite = false
-
 		// We haven't been told what the state at the event is so we need to calculate it from the prev_events
 		if stateAtEvent.BeforeStateSnapshotNID, err = roomState.CalculateAndStoreStateBeforeEvent(ctx, event, isRejected); err != nil {
 			return fmt.Errorf("roomState.CalculateAndStoreStateBeforeEvent: %w", err)

roomserver/internal/input/input_latest_events.go

@@ -27,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // updateLatestEvents updates the list of latest events for this room in the database and writes the
@@ -101,8 +102,8 @@ type latestEventsUpdater struct {
 	// The eventID of the event that was processed before this one.
 	lastEventIDSent string
 	// The latest events in the room after processing this event.
-	oldLatest []types.StateAtEventAndReference
+	oldLatest types.StateAtEventAndReferences
-	latest    []types.StateAtEventAndReference
+	latest    types.StateAtEventAndReferences
 	// The state entries removed from and added to the current state of the
 	// room as a result of processing this event. They are sorted lists.
 	removed []types.StateEntry
@@ -125,7 +126,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// state snapshot from somewhere else, e.g. a federated room join,
 	// then start with an empty set - none of the forward extremities
 	// that we knew about before matter anymore.
-	u.oldLatest = []types.StateAtEventAndReference{}
+	u.oldLatest = types.StateAtEventAndReferences{}
 	if !u.rewritesState {
 		u.oldStateNID = u.updater.CurrentStateSnapshotNID()
 		u.oldLatest = u.updater.LatestEvents()
@@ -233,12 +234,19 @@ func (u *latestEventsUpdater) latestState() error {
 		}
 	}
 
-	// Get a list of the current latest events. This may or may not
+	// Take the old set of extremities and the new set of extremities and
-	// include the new event from the input path, depending on whether
+	// mash them together into a list. This may or may not include the new event
-	// it is a forward extremity or not.
+	// from the input path, depending on whether it became a forward extremity
-	latestStateAtEvents := make([]types.StateAtEvent, len(u.latest))
+	// or not. We'll then run state resolution across all of them to determine
-	for i := range u.latest {
+	// the new current state of the room. Including the old extremities here
-		latestStateAtEvents[i] = u.latest[i].StateAtEvent
+	// ensures that new forward extremities with bad state snapshots (from
+	// possible malicious actors) can't completely corrupt the room state
+	// away from what it was before.
+	combinedExtremities := types.StateAtEventAndReferences(append(u.oldLatest, u.latest...))
+	combinedExtremities = combinedExtremities[:util.SortAndUnique(combinedExtremities)]
+	latestStateAtEvents := make([]types.StateAtEvent, len(combinedExtremities))
+	for i := range combinedExtremities {
+		latestStateAtEvents[i] = combinedExtremities[i].StateAtEvent
 	}
 
 	// Takes the NIDs of the latest events and creates a state snapshot
@@ -251,13 +259,6 @@ func (u *latestEventsUpdater) latestState() error {
 		return fmt.Errorf("roomState.CalculateAndStoreStateAfterEvents: %w", err)
 	}
 
-	// If we are overwriting the state then we should make sure that we
-	// don't send anything out over federation again, it will very likely
-	// be a repeat.
-	if u.stateAtEvent.Overwrite {
-		u.sendAsServer = ""
-	}
-
 	// Now that we have a new state snapshot based on the latest events,
 	// we can compare that new snapshot to the previous one and see what
 	// has changed. This gives us one list of removed state events and
@@ -270,6 +271,17 @@ func (u *latestEventsUpdater) latestState() error {
 		return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
 	}
 
+	if removed := len(u.removed) - len(u.added); removed > 0 {
+		logrus.WithFields(logrus.Fields{
+			"event_id":      u.event.EventID(),
+			"room_id":       u.event.RoomID(),
+			"old_state_nid": u.oldStateNID,
+			"new_state_nid": u.newStateNID,
+			"old_latest":    u.oldLatest.EventIDs(),
+			"new_latest":    u.latest.EventIDs(),
+		}).Errorf("Unexpected state deletion (removing %d events)", removed)
+	}
+
 	// Also work out the state before the event removes and the event
 	// adds.
 	u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = roomState.DifferenceBetweeenStateSnapshots(

roomserver/internal/input/input_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 )
@@ -22,7 +22,7 @@ var jc *nats.Conn
 
 func TestMain(m *testing.M) {
 	var b *base.BaseDendrite
-	b, js, jc = test.Base(nil)
+	b, js, jc = testrig.Base(nil)
 	code := m.Run()
 	b.ShutdownDendrite()
 	b.WaitForComponentsToFinish()

roomserver/internal/perform/perform_join.go

@@ -24,6 +24,7 @@ import (
 	"github.com/getsentry/sentry-go"
 	fsAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	rsAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
@@ -160,6 +161,7 @@ func (r *Joiner) performJoinRoomByAlias(
 }
 
 // TODO: Break this function up a bit
+// nolint:gocyclo
 func (r *Joiner) performJoinRoomByID(
 	ctx context.Context,
 	req *rsAPI.PerformJoinRequest,
@@ -210,6 +212,11 @@ func (r *Joiner) performJoinRoomByID(
 		req.Content = map[string]interface{}{}
 	}
 	req.Content["membership"] = gomatrixserverlib.Join
+	if authorisedVia, aerr := r.populateAuthorisedViaUserForRestrictedJoin(ctx, req); aerr != nil {
+		return "", "", aerr
+	} else if authorisedVia != "" {
+		req.Content["join_authorised_via_users_server"] = authorisedVia
+	}
 	if err = eb.SetContent(req.Content); err != nil {
 		return "", "", fmt.Errorf("eb.SetContent: %w", err)
 	}
@@ -350,6 +357,33 @@ func (r *Joiner) performFederatedJoinRoomByID(
 	return fedRes.JoinedVia, nil
 }
 
+func (r *Joiner) populateAuthorisedViaUserForRestrictedJoin(
+	ctx context.Context,
+	joinReq *rsAPI.PerformJoinRequest,
+) (string, error) {
+	req := &api.QueryRestrictedJoinAllowedRequest{
+		UserID: joinReq.UserID,
+		RoomID: joinReq.RoomIDOrAlias,
+	}
+	res := &api.QueryRestrictedJoinAllowedResponse{}
+	if err := r.Queryer.QueryRestrictedJoinAllowed(ctx, req, res); err != nil {
+		return "", fmt.Errorf("r.Queryer.QueryRestrictedJoinAllowed: %w", err)
+	}
+	if !res.Restricted {
+		return "", nil
+	}
+	if !res.Resident {
+		return "", nil
+	}
+	if !res.Allowed {
+		return "", &rsAPI.PerformError{
+			Code: rsAPI.PerformErrorNotAllowed,
+			Msg:  fmt.Sprintf("The join to room %s was not allowed.", joinReq.RoomIDOrAlias),
+		}
+	}
+	return res.AuthorisedVia, nil
+}
+
 func buildEvent(
 	ctx context.Context, db storage.Database, cfg *config.Global, builder *gomatrixserverlib.EventBuilder,
 ) (*gomatrixserverlib.HeaderedEvent, *rsAPI.QueryLatestEventsAndStateResponse, error) {

roomserver/internal/query/query.go

@@ -16,6 +16,7 @@ package query
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 
@@ -757,3 +758,128 @@ func (r *Queryer) QueryAuthChain(ctx context.Context, req *api.QueryAuthChainReq
 	res.AuthChain = hchain
 	return nil
 }
+
+// nolint:gocyclo
+func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse) error {
+	// Look up if we know anything about the room. If it doesn't exist
+	// or is a stub entry then we can't do anything.
+	roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID)
+	if err != nil {
+		return fmt.Errorf("r.DB.RoomInfo: %w", err)
+	}
+	if roomInfo == nil || roomInfo.IsStub {
+		return nil // fmt.Errorf("room %q doesn't exist or is stub room", req.RoomID)
+	}
+	// If the room version doesn't allow restricted joins then don't
+	// try to process any further.
+	allowRestrictedJoins, err := roomInfo.RoomVersion.MayAllowRestrictedJoinsInEventAuth()
+	if err != nil {
+		return fmt.Errorf("roomInfo.RoomVersion.AllowRestrictedJoinsInEventAuth: %w", err)
+	} else if !allowRestrictedJoins {
+		return nil
+	}
+	// Get the join rules to work out if the join rule is "restricted".
+	joinRulesEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomJoinRules, "")
+	if err != nil {
+		return fmt.Errorf("r.DB.GetStateEvent: %w", err)
+	}
+	var joinRules gomatrixserverlib.JoinRuleContent
+	if err = json.Unmarshal(joinRulesEvent.Content(), &joinRules); err != nil {
+		return fmt.Errorf("json.Unmarshal: %w", err)
+	}
+	// If the join rule isn't "restricted" then there's nothing more to do.
+	res.Restricted = joinRules.JoinRule == gomatrixserverlib.Restricted
+	if !res.Restricted {
+		return nil
+	}
+	// Start off by populating the "resident" flag in the response. If we
+	// come across any rooms in the request that are missing, we will unset
+	// the flag.
+	res.Resident = true
+	// If the user is already invited to the room then the join is allowed
+	// but we don't specify an authorised via user, since the event auth
+	// will allow the join anyway.
+	var pending bool
+	if pending, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil {
+		return fmt.Errorf("helpers.IsInvitePending: %w", err)
+	} else if pending {
+		res.Allowed = true
+		return nil
+	}
+	// We need to get the power levels content so that we can determine which
+	// users in the room are entitled to issue invites. We need to use one of
+	// these users as the authorising user.
+	powerLevelsEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomPowerLevels, "")
+	if err != nil {
+		return fmt.Errorf("r.DB.GetStateEvent: %w", err)
+	}
+	var powerLevels gomatrixserverlib.PowerLevelContent
+	if err = json.Unmarshal(powerLevelsEvent.Content(), &powerLevels); err != nil {
+		return fmt.Errorf("json.Unmarshal: %w", err)
+	}
+	// Step through the join rules and see if the user matches any of them.
+	for _, rule := range joinRules.Allow {
+		// We only understand "m.room_membership" rules at this point in
+		// time, so skip any rule that doesn't match those.
+		if rule.Type != gomatrixserverlib.MRoomMembership {
+			continue
+		}
+		// See if the room exists. If it doesn't exist or if it's a stub
+		// room entry then we can't check memberships.
+		targetRoomInfo, err := r.DB.RoomInfo(ctx, rule.RoomID)
+		if err != nil || targetRoomInfo == nil || targetRoomInfo.IsStub {
+			res.Resident = false
+			continue
+		}
+		// First of all work out if *we* are still in the room, otherwise
+		// it's possible that the memberships will be out of date.
+		isIn, err := r.DB.GetLocalServerInRoom(ctx, targetRoomInfo.RoomNID)
+		if err != nil || !isIn {
+			// If we aren't in the room, we can no longer tell if the room
+			// memberships are up-to-date.
+			res.Resident = false
+			continue
+		}
+		// At this point we're happy that we are in the room, so now let's
+		// see if the target user is in the room.
+		_, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
+		if err != nil {
+			continue
+		}
+		// If the user is not in the room then we will skip them.
+		if !isIn {
+			continue
+		}
+		// The user is in the room, so now we will need to authorise the
+		// join using the user ID of one of our own users in the room. Pick
+		// one.
+		joinNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, targetRoomInfo.RoomNID, true, true)
+		if err != nil || len(joinNIDs) == 0 {
+			// There should always be more than one join NID at this point
+			// because we are gated behind GetLocalServerInRoom, but y'know,
+			// sometimes strange things happen.
+			continue
+		}
+		// For each of the joined users, let's see if we can get a valid
+		// membership event.
+		for _, joinNID := range joinNIDs {
+			events, err := r.DB.Events(ctx, []types.EventNID{joinNID})
+			if err != nil || len(events) != 1 {
+				continue
+			}
+			event := events[0]
+			if event.Type() != gomatrixserverlib.MRoomMember || event.StateKey() == nil {
+				continue // shouldn't happen
+			}
+			// Only users that have the power to invite should be chosen.
+			if powerLevels.UserLevel(*event.StateKey()) < powerLevels.Invite {
+				continue
+			}
+			res.Resident = true
+			res.Allowed = true
+			res.AuthorisedVia = *event.StateKey()
+			return nil
+		}
+	}
+	return nil
+}

roomserver/internal/query/query_test.go

@@ -19,8 +19,8 @@ import (
 	"encoding/json"
 	"testing"
 
-	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 

roomserver/inthttp/client.go

@@ -61,6 +61,7 @@ const (
 	RoomserverQueryKnownUsersPath              = "/roomserver/queryKnownUsers"
 	RoomserverQueryServerBannedFromRoomPath    = "/roomserver/queryServerBannedFromRoom"
 	RoomserverQueryAuthChainPath               = "/roomserver/queryAuthChain"
+	RoomserverQueryRestrictedJoinAllowed       = "/roomserver/queryRestrictedJoinAllowed"
 )
 
 type httpRoomserverInternalAPI struct {
@@ -557,6 +558,16 @@ func (h *httpRoomserverInternalAPI) QueryServerBannedFromRoom(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
 }
 
+func (h *httpRoomserverInternalAPI) QueryRestrictedJoinAllowed(
+	ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRestrictedJoinAllowed")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverQueryRestrictedJoinAllowed
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
 func (h *httpRoomserverInternalAPI) PerformForget(ctx context.Context, req *api.PerformForgetRequest, res *api.PerformForgetResponse) error {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformForget")
 	defer span.Finish()

roomserver/inthttp/server.go

@@ -472,4 +472,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverQueryRestrictedJoinAllowed,
+		httputil.MakeInternalAPI("queryRestrictedJoinAllowed", func(req *http.Request) util.JSONResponse {
+			request := api.QueryRestrictedJoinAllowedRequest{}
+			response := api.QueryRestrictedJoinAllowedResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := r.QueryRestrictedJoinAllowed(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }

roomserver/storage/postgres/events_table.go

@@ -264,11 +264,11 @@ func (s *eventStatements) BulkSelectStateEventByNID(
 	ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID,
 	stateKeyTuples []types.StateKeyTuple,
 ) ([]types.StateEntry, error) {
-	tuples := stateKeyTupleSorter(stateKeyTuples)
+	tuples := types.StateKeyTupleSorter(stateKeyTuples)
 	sort.Sort(tuples)
-	eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays()
+	eventTypeNIDArray, eventStateKeyNIDArray := tuples.TypesAndStateKeysAsArrays()
 	stmt := sqlutil.TxStmt(txn, s.bulkSelectStateEventByNIDStmt)
-	rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs), eventTypeNIDArray, eventStateKeyNIDArray)
+	rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs), pq.Int64Array(eventTypeNIDArray), pq.Int64Array(eventStateKeyNIDArray))
 	if err != nil {
 		return nil, err
 	}

roomserver/storage/postgres/room_aliases_table.go

@@ -61,12 +61,12 @@ type roomAliasesStatements struct {
 	deleteRoomAliasStmt          *sql.Stmt
 }
 
-func createRoomAliasesTable(db *sql.DB) error {
+func CreateRoomAliasesTable(db *sql.DB) error {
 	_, err := db.Exec(roomAliasesSchema)
 	return err
 }
 
-func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
+func PrepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
 	s := &roomAliasesStatements{}
 
 	return s, sqlutil.StatementList{
@@ -108,8 +108,8 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID(
 	defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed")
 
 	var aliases []string
+	var alias string
 	for rows.Next() {
-		var alias string
 		if err = rows.Scan(&alias); err != nil {
 			return nil, err
 		}

roomserver/storage/postgres/rooms_table.go

@@ -95,12 +95,12 @@ type roomStatements struct {
 	bulkSelectRoomNIDsStmt             *sql.Stmt
 }
 
-func createRoomsTable(db *sql.DB) error {
+func CreateRoomsTable(db *sql.DB) error {
 	_, err := db.Exec(roomsSchema)
 	return err
 }
 
-func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
+func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 	s := &roomStatements{}
 
 	return s, sqlutil.StatementList{
@@ -117,7 +117,7 @@ func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 	}.Prepare(db)
 }
 
-func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error) {
+func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt)
 	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
@@ -125,8 +125,8 @@ func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]stri
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed")
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
@@ -231,9 +231,9 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomVersionsForRoomNIDsStmt: rows.close() failed")
 	result := make(map[types.RoomNID]gomatrixserverlib.RoomVersion)
+	var roomNID types.RoomNID
+	var roomVersion gomatrixserverlib.RoomVersion
 	for rows.Next() {
-		var roomNID types.RoomNID
-		var roomVersion gomatrixserverlib.RoomVersion
 		if err = rows.Scan(&roomNID, &roomVersion); err != nil {
 			return nil, err
 		}
@@ -254,8 +254,8 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomIDsStmt: rows.close() failed")
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
@@ -276,8 +276,8 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomNIDsStmt: rows.close() failed")
 	var roomNIDs []types.RoomNID
+	var roomNID types.RoomNID
 	for rows.Next() {
-		var roomNID types.RoomNID
 		if err = rows.Scan(&roomNID); err != nil {
 			return nil, err
 		}

roomserver/storage/postgres/state_block_table.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"sort"
 
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal"
@@ -71,12 +70,12 @@ type stateBlockStatements struct {
 	bulkSelectStateBlockEntriesStmt *sql.Stmt
 }
 
-func createStateBlockTable(db *sql.DB) error {
+func CreateStateBlockTable(db *sql.DB) error {
 	_, err := db.Exec(stateDataSchema)
 	return err
 }
 
-func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
+func PrepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
 	s := &stateBlockStatements{}
 
 	return s, sqlutil.StatementList{
@@ -90,9 +89,9 @@ func (s *stateBlockStatements) BulkInsertStateData(
 	entries types.StateEntries,
 ) (id types.StateBlockNID, err error) {
 	entries = entries[:util.SortAndUnique(entries)]
-	var nids types.EventNIDs
+	nids := make(types.EventNIDs, entries.Len())
-	for _, e := range entries {
+	for i := range entries {
-		nids = append(nids, e.EventNID)
+		nids[i] = entries[i].EventNID
 	}
 	stmt := sqlutil.TxStmt(txn, s.insertStateDataStmt)
 	err = stmt.QueryRowContext(
@@ -113,15 +112,15 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries(
 
 	results := make([][]types.EventNID, len(stateBlockNIDs))
 	i := 0
+	var stateBlockNID types.StateBlockNID
+	var result pq.Int64Array
 	for ; rows.Next(); i++ {
-		var stateBlockNID types.StateBlockNID
-		var result pq.Int64Array
 		if err = rows.Scan(&stateBlockNID, &result); err != nil {
 			return nil, err
 		}
-		r := []types.EventNID{}
+		r := make([]types.EventNID, len(result))
-		for _, e := range result {
+		for x := range result {
-			r = append(r, types.EventNID(e))
+			r[x] = types.EventNID(result[x])
 		}
 		results[i] = r
 	}
@@ -141,35 +140,3 @@ func stateBlockNIDsAsArray(stateBlockNIDs []types.StateBlockNID) pq.Int64Array {
 	}
 	return pq.Int64Array(nids)
 }
-
-type stateKeyTupleSorter []types.StateKeyTuple
-
-func (s stateKeyTupleSorter) Len() int           { return len(s) }
-func (s stateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) }
-func (s stateKeyTupleSorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-
-// Check whether a tuple is in the list. Assumes that the list is sorted.
-func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool {
-	i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) })
-	return i < len(s) && s[i] == value
-}
-
-// List the unique eventTypeNIDs and eventStateKeyNIDs.
-// Assumes that the list is sorted.
-func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs pq.Int64Array, eventStateKeyNIDs pq.Int64Array) {
-	eventTypeNIDs = make(pq.Int64Array, len(s))
-	eventStateKeyNIDs = make(pq.Int64Array, len(s))
-	for i := range s {
-		eventTypeNIDs[i] = int64(s[i].EventTypeNID)
-		eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID)
-	}
-	eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))]
-	eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))]
-	return
-}
-
-type int64Sorter []int64
-
-func (s int64Sorter) Len() int           { return len(s) }
-func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] }
-func (s int64Sorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

roomserver/storage/postgres/state_block_table_test.go

@@ -1,86 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"sort"
-	"testing"
-
-	"github.com/matrix-org/dendrite/roomserver/types"
-)
-
-func TestStateKeyTupleSorter(t *testing.T) {
-	input := stateKeyTupleSorter{
-		{EventTypeNID: 1, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 4},
-		{EventTypeNID: 2, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 1},
-	}
-	want := []types.StateKeyTuple{
-		{EventTypeNID: 1, EventStateKeyNID: 1},
-		{EventTypeNID: 1, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 4},
-		{EventTypeNID: 2, EventStateKeyNID: 2},
-	}
-	doNotWant := []types.StateKeyTuple{
-		{EventTypeNID: 0, EventStateKeyNID: 0},
-		{EventTypeNID: 1, EventStateKeyNID: 3},
-		{EventTypeNID: 2, EventStateKeyNID: 1},
-		{EventTypeNID: 3, EventStateKeyNID: 1},
-	}
-	wantTypeNIDs := []int64{1, 2}
-	wantStateKeyNIDs := []int64{1, 2, 4}
-
-	// Sort the input and check it's in the right order.
-	sort.Sort(input)
-	gotTypeNIDs, gotStateKeyNIDs := input.typesAndStateKeysAsArrays()
-
-	for i := range want {
-		if input[i] != want[i] {
-			t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i])
-		}
-
-		if !input.contains(want[i]) {
-			t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i])
-		}
-	}
-
-	for i := range doNotWant {
-		if input.contains(doNotWant[i]) {
-			t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i])
-		}
-	}
-
-	if len(wantTypeNIDs) != len(gotTypeNIDs) {
-		t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-	}
-
-	for i := range wantTypeNIDs {
-		if wantTypeNIDs[i] != gotTypeNIDs[i] {
-			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-		}
-	}
-
-	if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) {
-		t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs)
-	}
-
-	for i := range wantStateKeyNIDs {
-		if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] {
-			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-		}
-	}
-}

roomserver/storage/postgres/state_snapshot_table.go

@@ -77,12 +77,12 @@ type stateSnapshotStatements struct {
 	bulkSelectStateBlockNIDsStmt *sql.Stmt
 }
 
-func createStateSnapshotTable(db *sql.DB) error {
+func CreateStateSnapshotTable(db *sql.DB) error {
 	_, err := db.Exec(stateSnapshotSchema)
 	return err
 }
 
-func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
+func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
 	s := &stateSnapshotStatements{}
 
 	return s, sqlutil.StatementList{
@@ -95,12 +95,10 @@ func (s *stateSnapshotStatements) InsertState(
 	ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, nids types.StateBlockNIDs,
 ) (stateNID types.StateSnapshotNID, err error) {
 	nids = nids[:util.SortAndUnique(nids)]
-	var id int64
+	err = sqlutil.TxStmt(txn, s.insertStateStmt).QueryRowContext(ctx, nids.Hash(), int64(roomNID), stateBlockNIDsAsArray(nids)).Scan(&stateNID)
-	err = sqlutil.TxStmt(txn, s.insertStateStmt).QueryRowContext(ctx, nids.Hash(), int64(roomNID), stateBlockNIDsAsArray(nids)).Scan(&id)
 	if err != nil {
 		return 0, err
 	}
-	stateNID = types.StateSnapshotNID(id)
 	return
 }
 
@@ -119,9 +117,9 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 	defer rows.Close() // nolint: errcheck
 	results := make([]types.StateBlockNIDList, len(stateNIDs))
 	i := 0
+	var stateBlockNIDs pq.Int64Array
 	for ; rows.Next(); i++ {
 		result := &results[i]
-		var stateBlockNIDs pq.Int64Array
 		if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil {
 			return nil, err
 		}

roomserver/storage/postgres/storage.go

@@ -80,19 +80,19 @@ func (d *Database) create(db *sql.DB) error {
 	if err := CreateEventsTable(db); err != nil {
 		return err
 	}
-	if err := createRoomsTable(db); err != nil {
+	if err := CreateRoomsTable(db); err != nil {
 		return err
 	}
-	if err := createStateBlockTable(db); err != nil {
+	if err := CreateStateBlockTable(db); err != nil {
 		return err
 	}
-	if err := createStateSnapshotTable(db); err != nil {
+	if err := CreateStateSnapshotTable(db); err != nil {
 		return err
 	}
 	if err := CreatePrevEventsTable(db); err != nil {
 		return err
 	}
-	if err := createRoomAliasesTable(db); err != nil {
+	if err := CreateRoomAliasesTable(db); err != nil {
 		return err
 	}
 	if err := CreateInvitesTable(db); err != nil {
@@ -128,15 +128,15 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	rooms, err := prepareRoomsTable(db)
+	rooms, err := PrepareRoomsTable(db)
 	if err != nil {
 		return err
 	}
-	stateBlock, err := prepareStateBlockTable(db)
+	stateBlock, err := PrepareStateBlockTable(db)
 	if err != nil {
 		return err
 	}
-	stateSnapshot, err := prepareStateSnapshotTable(db)
+	stateSnapshot, err := PrepareStateSnapshotTable(db)
 	if err != nil {
 		return err
 	}
@@ -144,7 +144,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	roomAliases, err := prepareRoomAliasesTable(db)
+	roomAliases, err := PrepareRoomAliasesTable(db)
 	if err != nil {
 		return err
 	}

roomserver/storage/shared/storage.go

@@ -1216,7 +1216,7 @@ func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString strin
 
 // GetKnownRooms returns a list of all rooms we know about.
 func (d *Database) GetKnownRooms(ctx context.Context) ([]string, error) {
-	return d.RoomsTable.SelectRoomIDs(ctx, nil)
+	return d.RoomsTable.SelectRoomIDsWithEvents(ctx, nil)
 }
 
 // ForgetRoom sets a users room to forgotten

roomserver/storage/sqlite3/events_table.go

@@ -247,9 +247,9 @@ func (s *eventStatements) BulkSelectStateEventByNID(
 	ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID,
 	stateKeyTuples []types.StateKeyTuple,
 ) ([]types.StateEntry, error) {
-	tuples := stateKeyTupleSorter(stateKeyTuples)
+	tuples := types.StateKeyTupleSorter(stateKeyTuples)
 	sort.Sort(tuples)
-	eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays()
+	eventTypeNIDArray, eventStateKeyNIDArray := tuples.TypesAndStateKeysAsArrays()
 	params := make([]interface{}, 0, len(eventNIDs)+len(eventTypeNIDArray)+len(eventStateKeyNIDArray))
 	selectOrig := strings.Replace(bulkSelectStateEventByNIDSQL, "($1)", sqlutil.QueryVariadic(len(eventNIDs)), 1)
 	for _, v := range eventNIDs {

roomserver/storage/sqlite3/room_aliases_table.go

@@ -63,12 +63,12 @@ type roomAliasesStatements struct {
 	deleteRoomAliasStmt          *sql.Stmt
 }
 
-func createRoomAliasesTable(db *sql.DB) error {
+func CreateRoomAliasesTable(db *sql.DB) error {
 	_, err := db.Exec(roomAliasesSchema)
 	return err
 }
 
-func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
+func PrepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
 	s := &roomAliasesStatements{
 		db: db,
 	}
@@ -113,8 +113,8 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID(
 
 	defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed")
 
+	var alias string
 	for rows.Next() {
-		var alias string
 		if err = rows.Scan(&alias); err != nil {
 			return
 		}

roomserver/storage/sqlite3/rooms_table.go

@@ -86,12 +86,12 @@ type roomStatements struct {
 	selectRoomIDsStmt  *sql.Stmt
 }
 
-func createRoomsTable(db *sql.DB) error {
+func CreateRoomsTable(db *sql.DB) error {
 	_, err := db.Exec(roomsSchema)
 	return err
 }
 
-func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
+func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 	s := &roomStatements{
 		db: db,
 	}
@@ -108,7 +108,7 @@ func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 	}.Prepare(db)
 }
 
-func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error) {
+func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt)
 	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
@@ -116,8 +116,8 @@ func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]stri
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed")
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
@@ -241,9 +241,9 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomVersionsForRoomNIDsStmt: rows.close() failed")
 	result := make(map[types.RoomNID]gomatrixserverlib.RoomVersion)
+	var roomNID types.RoomNID
+	var roomVersion gomatrixserverlib.RoomVersion
 	for rows.Next() {
-		var roomNID types.RoomNID
-		var roomVersion gomatrixserverlib.RoomVersion
 		if err = rows.Scan(&roomNID, &roomVersion); err != nil {
 			return nil, err
 		}
@@ -270,8 +270,8 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomIDsStmt: rows.close() failed")
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
@@ -298,8 +298,8 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomNIDsStmt: rows.close() failed")
 	var roomNIDs []types.RoomNID
+	var roomNID types.RoomNID
 	for rows.Next() {
-		var roomNID types.RoomNID
 		if err = rows.Scan(&roomNID); err != nil {
 			return nil, err
 		}

roomserver/storage/sqlite3/state_block_table.go

@@ -20,7 +20,6 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"sort"
 	"strings"
 
 	"github.com/matrix-org/dendrite/internal"
@@ -64,12 +63,12 @@ type stateBlockStatements struct {
 	bulkSelectStateBlockEntriesStmt *sql.Stmt
 }
 
-func createStateBlockTable(db *sql.DB) error {
+func CreateStateBlockTable(db *sql.DB) error {
 	_, err := db.Exec(stateDataSchema)
 	return err
 }
 
-func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
+func PrepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
 	s := &stateBlockStatements{
 		db: db,
 	}
@@ -85,9 +84,9 @@ func (s *stateBlockStatements) BulkInsertStateData(
 	entries types.StateEntries,
 ) (id types.StateBlockNID, err error) {
 	entries = entries[:util.SortAndUnique(entries)]
-	nids := types.EventNIDs{} // zero slice to not store 'null' in the DB
+	nids := make(types.EventNIDs, entries.Len())
-	for _, e := range entries {
+	for i := range entries {
-		nids = append(nids, e.EventNID)
+		nids[i] = entries[i].EventNID
 	}
 	js, err := json.Marshal(nids)
 	if err != nil {
@@ -122,13 +121,13 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries(
 
 	results := make([][]types.EventNID, len(stateBlockNIDs))
 	i := 0
+	var stateBlockNID types.StateBlockNID
+	var result json.RawMessage
 	for ; rows.Next(); i++ {
-		var stateBlockNID types.StateBlockNID
-		var result json.RawMessage
 		if err = rows.Scan(&stateBlockNID, &result); err != nil {
 			return nil, err
 		}
-		r := []types.EventNID{}
+		var r []types.EventNID
 		if err = json.Unmarshal(result, &r); err != nil {
 			return nil, fmt.Errorf("json.Unmarshal: %w", err)
 		}
@@ -142,35 +141,3 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries(
 	}
 	return results, err
 }
-
-type stateKeyTupleSorter []types.StateKeyTuple
-
-func (s stateKeyTupleSorter) Len() int           { return len(s) }
-func (s stateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) }
-func (s stateKeyTupleSorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-
-// Check whether a tuple is in the list. Assumes that the list is sorted.
-func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool {
-	i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) })
-	return i < len(s) && s[i] == value
-}
-
-// List the unique eventTypeNIDs and eventStateKeyNIDs.
-// Assumes that the list is sorted.
-func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) {
-	eventTypeNIDs = make([]int64, len(s))
-	eventStateKeyNIDs = make([]int64, len(s))
-	for i := range s {
-		eventTypeNIDs[i] = int64(s[i].EventTypeNID)
-		eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID)
-	}
-	eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))]
-	eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))]
-	return
-}
-
-type int64Sorter []int64
-
-func (s int64Sorter) Len() int           { return len(s) }
-func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] }
-func (s int64Sorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

roomserver/storage/sqlite3/state_block_table_test.go

@@ -1,86 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"sort"
-	"testing"
-
-	"github.com/matrix-org/dendrite/roomserver/types"
-)
-
-func TestStateKeyTupleSorter(t *testing.T) {
-	input := stateKeyTupleSorter{
-		{EventTypeNID: 1, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 4},
-		{EventTypeNID: 2, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 1},
-	}
-	want := []types.StateKeyTuple{
-		{EventTypeNID: 1, EventStateKeyNID: 1},
-		{EventTypeNID: 1, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 4},
-		{EventTypeNID: 2, EventStateKeyNID: 2},
-	}
-	doNotWant := []types.StateKeyTuple{
-		{EventTypeNID: 0, EventStateKeyNID: 0},
-		{EventTypeNID: 1, EventStateKeyNID: 3},
-		{EventTypeNID: 2, EventStateKeyNID: 1},
-		{EventTypeNID: 3, EventStateKeyNID: 1},
-	}
-	wantTypeNIDs := []int64{1, 2}
-	wantStateKeyNIDs := []int64{1, 2, 4}
-
-	// Sort the input and check it's in the right order.
-	sort.Sort(input)
-	gotTypeNIDs, gotStateKeyNIDs := input.typesAndStateKeysAsArrays()
-
-	for i := range want {
-		if input[i] != want[i] {
-			t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i])
-		}
-
-		if !input.contains(want[i]) {
-			t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i])
-		}
-	}
-
-	for i := range doNotWant {
-		if input.contains(doNotWant[i]) {
-			t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i])
-		}
-	}
-
-	if len(wantTypeNIDs) != len(gotTypeNIDs) {
-		t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-	}
-
-	for i := range wantTypeNIDs {
-		if wantTypeNIDs[i] != gotTypeNIDs[i] {
-			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-		}
-	}
-
-	if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) {
-		t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs)
-	}
-
-	for i := range wantStateKeyNIDs {
-		if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] {
-			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-		}
-	}
-}

roomserver/storage/sqlite3/state_snapshot_table.go

@@ -68,12 +68,12 @@ type stateSnapshotStatements struct {
 	bulkSelectStateBlockNIDsStmt *sql.Stmt
 }
 
-func createStateSnapshotTable(db *sql.DB) error {
+func CreateStateSnapshotTable(db *sql.DB) error {
 	_, err := db.Exec(stateSnapshotSchema)
 	return err
 }
 
-func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
+func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
 	s := &stateSnapshotStatements{
 		db: db,
 	}
@@ -96,12 +96,10 @@ func (s *stateSnapshotStatements) InsertState(
 		return
 	}
 	insertStmt := sqlutil.TxStmt(txn, s.insertStateStmt)
-	var id int64
+	err = insertStmt.QueryRowContext(ctx, stateBlockNIDs.Hash(), int64(roomNID), string(stateBlockNIDsJSON)).Scan(&stateNID)
-	err = insertStmt.QueryRowContext(ctx, stateBlockNIDs.Hash(), int64(roomNID), string(stateBlockNIDsJSON)).Scan(&id)
 	if err != nil {
 		return 0, err
 	}
-	stateNID = types.StateSnapshotNID(id)
 	return
 }
 
@@ -127,9 +125,9 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockNIDs: rows.close() failed")
 	results := make([]types.StateBlockNIDList, len(stateNIDs))
 	i := 0
+	var stateBlockNIDsJSON string
 	for ; rows.Next(); i++ {
 		result := &results[i]
-		var stateBlockNIDsJSON string
 		if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil {
 			return nil, err
 		}

roomserver/storage/sqlite3/storage.go

@@ -89,19 +89,19 @@ func (d *Database) create(db *sql.DB) error {
 	if err := CreateEventsTable(db); err != nil {
 		return err
 	}
-	if err := createRoomsTable(db); err != nil {
+	if err := CreateRoomsTable(db); err != nil {
 		return err
 	}
-	if err := createStateBlockTable(db); err != nil {
+	if err := CreateStateBlockTable(db); err != nil {
 		return err
 	}
-	if err := createStateSnapshotTable(db); err != nil {
+	if err := CreateStateSnapshotTable(db); err != nil {
 		return err
 	}
 	if err := CreatePrevEventsTable(db); err != nil {
 		return err
 	}
-	if err := createRoomAliasesTable(db); err != nil {
+	if err := CreateRoomAliasesTable(db); err != nil {
 		return err
 	}
 	if err := CreateInvitesTable(db); err != nil {
@@ -137,15 +137,15 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	rooms, err := prepareRoomsTable(db)
+	rooms, err := PrepareRoomsTable(db)
 	if err != nil {
 		return err
 	}
-	stateBlock, err := prepareStateBlockTable(db)
+	stateBlock, err := PrepareStateBlockTable(db)
 	if err != nil {
 		return err
 	}
-	stateSnapshot, err := prepareStateSnapshotTable(db)
+	stateSnapshot, err := PrepareStateSnapshotTable(db)
 	if err != nil {
 		return err
 	}
@@ -153,7 +153,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	roomAliases, err := prepareRoomAliasesTable(db)
+	roomAliases, err := PrepareRoomAliasesTable(db)
 	if err != nil {
 		return err
 	}

roomserver/storage/tables/events_table_test.go

@@ -39,7 +39,7 @@ func mustCreateEventsTable(t *testing.T, dbType test.DBType) (tables.Events, fun
 }
 
 func Test_EventsTable(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	ctx := context.Background()
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
@@ -85,7 +85,6 @@ func Test_EventsTable(t *testing.T) {
 				assert.NoError(t, err)
 			}
 			stateAtEvent := types.StateAtEvent{
-				Overwrite:              false,
 				BeforeStateSnapshotNID: types.StateSnapshotNID(stateSnapshot),
 				IsRejected:             false,
 				StateEntry: types.StateEntry{

roomserver/storage/tables/interface.go

@@ -72,7 +72,7 @@ type Rooms interface {
 	UpdateLatestEventNIDs(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventNIDs []types.EventNID, lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID) error
 	SelectRoomVersionsForRoomNIDs(ctx context.Context, txn *sql.Tx, roomNID []types.RoomNID) (map[types.RoomNID]gomatrixserverlib.RoomVersion, error)
 	SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error)
-	SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error)
+	SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error)
 	BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error)
 	BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error)
 }

roomserver/storage/tables/previous_events_table_test.go

@@ -38,7 +38,7 @@ func mustCreatePreviousEventsTable(t *testing.T, dbType test.DBType) (tab tables
 
 func TestPreviousEventsTable(t *testing.T) {
 	ctx := context.Background()
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		tab, close := mustCreatePreviousEventsTable(t, dbType)

roomserver/storage/tables/published_table_test.go

@@ -38,7 +38,7 @@ func mustCreatePublishedTable(t *testing.T, dbType test.DBType) (tab tables.Publ
 
 func TestPublishedTable(t *testing.T) {
 	ctx := context.Background()
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		tab, close := mustCreatePublishedTable(t, dbType)

roomserver/storage/tables/room_aliases_table_test.go

@@ -0,0 +1,96 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateRoomAliasesTable(t *testing.T, dbType test.DBType) (tab tables.RoomAliases, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateRoomAliasesTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareRoomAliasesTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateRoomAliasesTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareRoomAliasesTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestRoomAliasesTable(t *testing.T) {
+	alice := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+	room2 := test.NewRoom(t, alice)
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateRoomAliasesTable(t, dbType)
+		defer close()
+		alias, alias2, alias3 := "#alias:localhost", "#alias2:localhost", "#alias3:localhost"
+		// insert aliases
+		err := tab.InsertRoomAlias(ctx, nil, alias, room.ID, alice.ID)
+		assert.NoError(t, err)
+
+		err = tab.InsertRoomAlias(ctx, nil, alias2, room.ID, alice.ID)
+		assert.NoError(t, err)
+
+		err = tab.InsertRoomAlias(ctx, nil, alias3, room2.ID, alice.ID)
+		assert.NoError(t, err)
+
+		// verify we can get the roomID for the alias
+		roomID, err := tab.SelectRoomIDFromAlias(ctx, nil, alias)
+		assert.NoError(t, err)
+		assert.Equal(t, room.ID, roomID)
+
+		// .. and the creator
+		creator, err := tab.SelectCreatorIDFromAlias(ctx, nil, alias)
+		assert.NoError(t, err)
+		assert.Equal(t, alice.ID, creator)
+
+		creator, err = tab.SelectCreatorIDFromAlias(ctx, nil, "#doesntexist:localhost")
+		assert.NoError(t, err)
+		assert.Equal(t, "", creator)
+
+		roomID, err = tab.SelectRoomIDFromAlias(ctx, nil, "#doesntexist:localhost")
+		assert.NoError(t, err)
+		assert.Equal(t, "", roomID)
+
+		// get all aliases for a room
+		aliases, err := tab.SelectAliasesFromRoomID(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, []string{alias, alias2}, aliases)
+
+		// delete an alias and verify it's deleted
+		err = tab.DeleteRoomAlias(ctx, nil, alias2)
+		assert.NoError(t, err)
+
+		aliases, err = tab.SelectAliasesFromRoomID(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, []string{alias}, aliases)
+
+		// deleting the same alias should be a no-op
+		err = tab.DeleteRoomAlias(ctx, nil, alias2)
+		assert.NoError(t, err)
+
+		// Delete non-existent alias should be a no-op
+		err = tab.DeleteRoomAlias(ctx, nil, "#doesntexist:localhost")
+		assert.NoError(t, err)
+	})
+}

roomserver/storage/tables/rooms_table_test.go

@@ -0,0 +1,128 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateRoomsTable(t *testing.T, dbType test.DBType) (tab tables.Rooms, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateRoomsTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareRoomsTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateRoomsTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareRoomsTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestRoomsTable(t *testing.T) {
+	alice := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateRoomsTable(t, dbType)
+		defer close()
+
+		wantRoomNID, err := tab.InsertRoomNID(ctx, nil, room.ID, room.Version)
+		assert.NoError(t, err)
+
+		// Create dummy room
+		_, err = tab.InsertRoomNID(ctx, nil, util.RandomString(16), room.Version)
+		assert.NoError(t, err)
+
+		gotRoomNID, err := tab.SelectRoomNID(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, wantRoomNID, gotRoomNID)
+
+		// Ensure non existent roomNID errors
+		roomNID, err := tab.SelectRoomNID(ctx, nil, "!doesnotexist:localhost")
+		assert.Error(t, err)
+		assert.Equal(t, types.RoomNID(0), roomNID)
+
+		roomInfo, err := tab.SelectRoomInfo(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, &types.RoomInfo{
+			RoomNID:          wantRoomNID,
+			RoomVersion:      room.Version,
+			StateSnapshotNID: 0,
+			IsStub:           true, // there are no latestEventNIDs
+		}, roomInfo)
+
+		roomInfo, err = tab.SelectRoomInfo(ctx, nil, "!doesnotexist:localhost")
+		assert.NoError(t, err)
+		assert.Nil(t, roomInfo)
+
+		// There are no rooms with latestEventNIDs yet
+		roomIDs, err := tab.SelectRoomIDsWithEvents(ctx, nil)
+		assert.NoError(t, err)
+		assert.Equal(t, 0, len(roomIDs))
+
+		roomVersions, err := tab.SelectRoomVersionsForRoomNIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337})
+		assert.NoError(t, err)
+		assert.Equal(t, roomVersions[wantRoomNID], room.Version)
+		// Room does not exist
+		_, ok := roomVersions[1337]
+		assert.False(t, ok)
+
+		roomIDs, err = tab.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337})
+		assert.NoError(t, err)
+		assert.Equal(t, []string{room.ID}, roomIDs)
+
+		roomNIDs, err := tab.BulkSelectRoomNIDs(ctx, nil, []string{room.ID, "!doesnotexist:localhost"})
+		assert.NoError(t, err)
+		assert.Equal(t, []types.RoomNID{wantRoomNID}, roomNIDs)
+
+		wantEventNIDs := []types.EventNID{1, 2, 3}
+		lastEventSentNID := types.EventNID(3)
+		stateSnapshotNID := types.StateSnapshotNID(1)
+		// make the room "usable"
+		err = tab.UpdateLatestEventNIDs(ctx, nil, wantRoomNID, wantEventNIDs, lastEventSentNID, stateSnapshotNID)
+		assert.NoError(t, err)
+
+		roomInfo, err = tab.SelectRoomInfo(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, &types.RoomInfo{
+			RoomNID:          wantRoomNID,
+			RoomVersion:      room.Version,
+			StateSnapshotNID: 1,
+			IsStub:           false,
+		}, roomInfo)
+
+		eventNIDs, snapshotNID, err := tab.SelectLatestEventNIDs(ctx, nil, wantRoomNID)
+		assert.NoError(t, err)
+		assert.Equal(t, wantEventNIDs, eventNIDs)
+		assert.Equal(t, types.StateSnapshotNID(1), snapshotNID)
+
+		// Again, doesn't exist
+		_, _, err = tab.SelectLatestEventNIDs(ctx, nil, 1337)
+		assert.Error(t, err)
+
+		eventNIDs, eventNID, snapshotNID, err := tab.SelectLatestEventsNIDsForUpdate(ctx, nil, wantRoomNID)
+		assert.NoError(t, err)
+		assert.Equal(t, wantEventNIDs, eventNIDs)
+		assert.Equal(t, types.EventNID(3), eventNID)
+		assert.Equal(t, types.StateSnapshotNID(1), snapshotNID)
+	})
+}

roomserver/storage/tables/state_block_table_test.go

@@ -0,0 +1,92 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateStateBlockTable(t *testing.T, dbType test.DBType) (tab tables.StateBlock, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateStateBlockTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareStateBlockTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateStateBlockTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareStateBlockTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestStateBlockTable(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateStateBlockTable(t, dbType)
+		defer close()
+
+		// generate some dummy data
+		var entries types.StateEntries
+		for i := 0; i < 100; i++ {
+			entry := types.StateEntry{
+				EventNID: types.EventNID(i),
+			}
+			entries = append(entries, entry)
+		}
+		stateBlockNID, err := tab.BulkInsertStateData(ctx, nil, entries)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateBlockNID(1), stateBlockNID)
+
+		// generate a different hash, to get a new StateBlockNID
+		var entries2 types.StateEntries
+		for i := 100; i < 300; i++ {
+			entry := types.StateEntry{
+				EventNID: types.EventNID(i),
+			}
+			entries2 = append(entries2, entry)
+		}
+		stateBlockNID, err = tab.BulkInsertStateData(ctx, nil, entries2)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateBlockNID(2), stateBlockNID)
+
+		eventNIDs, err := tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{1, 2})
+		assert.NoError(t, err)
+		assert.Equal(t, len(entries), len(eventNIDs[0]))
+		assert.Equal(t, len(entries2), len(eventNIDs[1]))
+
+		// try to get a StateBlockNID which does not exist
+		_, err = tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{5})
+		assert.Error(t, err)
+
+		// This should return an error, since we can only retrieve 1 StateBlock
+		_, err = tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{1, 5})
+		assert.Error(t, err)
+
+		for i := 0; i < 65555; i++ {
+			entry := types.StateEntry{
+				EventNID: types.EventNID(i),
+			}
+			entries2 = append(entries2, entry)
+		}
+		stateBlockNID, err = tab.BulkInsertStateData(ctx, nil, entries2)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateBlockNID(3), stateBlockNID)
+	})
+}

roomserver/storage/tables/state_snapshot_table_test.go

@@ -0,0 +1,86 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateStateSnapshotTable(t *testing.T, dbType test.DBType) (tab tables.StateSnapshot, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateStateSnapshotTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareStateSnapshotTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateStateSnapshotTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareStateSnapshotTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestStateSnapshotTable(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateStateSnapshotTable(t, dbType)
+		defer close()
+
+		// generate some dummy data
+		var stateBlockNIDs types.StateBlockNIDs
+		for i := 0; i < 100; i++ {
+			stateBlockNIDs = append(stateBlockNIDs, types.StateBlockNID(i))
+		}
+		stateNID, err := tab.InsertState(ctx, nil, 1, stateBlockNIDs)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateSnapshotNID(1), stateNID)
+
+		// verify ON CONFLICT; Note: this updates the sequence!
+		stateNID, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateSnapshotNID(1), stateNID)
+
+		// create a second snapshot
+		var stateBlockNIDs2 types.StateBlockNIDs
+		for i := 100; i < 150; i++ {
+			stateBlockNIDs2 = append(stateBlockNIDs2, types.StateBlockNID(i))
+		}
+
+		stateNID, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs2)
+		assert.NoError(t, err)
+		// StateSnapshotNID is now 3, since the DO UPDATE SET statement incremented the sequence
+		assert.Equal(t, types.StateSnapshotNID(3), stateNID)
+
+		nidLists, err := tab.BulkSelectStateBlockNIDs(ctx, nil, []types.StateSnapshotNID{1, 3})
+		assert.NoError(t, err)
+		assert.Equal(t, stateBlockNIDs, types.StateBlockNIDs(nidLists[0].StateBlockNIDs))
+		assert.Equal(t, stateBlockNIDs2, types.StateBlockNIDs(nidLists[1].StateBlockNIDs))
+
+		// check we get an error if the state snapshot does not exist
+		_, err = tab.BulkSelectStateBlockNIDs(ctx, nil, []types.StateSnapshotNID{2})
+		assert.Error(t, err)
+
+		// create a second snapshot
+		for i := 0; i < 65555; i++ {
+			stateBlockNIDs2 = append(stateBlockNIDs2, types.StateBlockNID(i))
+		}
+		_, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs2)
+		assert.NoError(t, err)
+	})
+}

roomserver/types/types.go

@@ -18,8 +18,10 @@ package types
 import (
 	"encoding/json"
 	"sort"
+	"strings"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
 	"golang.org/x/crypto/blake2b"
 )
 
@@ -96,6 +98,38 @@ func (a StateKeyTuple) LessThan(b StateKeyTuple) bool {
 	return a.EventStateKeyNID < b.EventStateKeyNID
 }
 
+type StateKeyTupleSorter []StateKeyTuple
+
+func (s StateKeyTupleSorter) Len() int           { return len(s) }
+func (s StateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) }
+func (s StateKeyTupleSorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+// Check whether a tuple is in the list. Assumes that the list is sorted.
+func (s StateKeyTupleSorter) contains(value StateKeyTuple) bool {
+	i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) })
+	return i < len(s) && s[i] == value
+}
+
+// List the unique eventTypeNIDs and eventStateKeyNIDs.
+// Assumes that the list is sorted.
+func (s StateKeyTupleSorter) TypesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) {
+	eventTypeNIDs = make([]int64, len(s))
+	eventStateKeyNIDs = make([]int64, len(s))
+	for i := range s {
+		eventTypeNIDs[i] = int64(s[i].EventTypeNID)
+		eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID)
+	}
+	eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))]
+	eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))]
+	return
+}
+
+type int64Sorter []int64
+
+func (s int64Sorter) Len() int           { return len(s) }
+func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] }
+func (s int64Sorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
 // A StateEntry is an entry in the room state of a matrix room.
 type StateEntry struct {
 	StateKeyTuple
@@ -139,10 +173,6 @@ func DeduplicateStateEntries(a []StateEntry) []StateEntry {
 
 // StateAtEvent is the state before and after a matrix event.
 type StateAtEvent struct {
-	// Should this state overwrite the latest events and memberships of the room?
-	// This might be necessary when rejoining a federated room after a period of
-	// absence, as our state and latest events will be out of date.
-	Overwrite bool
 	// The state before the event.
 	BeforeStateSnapshotNID StateSnapshotNID
 	// True if this StateEntry is rejected. State resolution should then treat this
@@ -166,6 +196,28 @@ type StateAtEventAndReference struct {
 	gomatrixserverlib.EventReference
 }
 
+type StateAtEventAndReferences []StateAtEventAndReference
+
+func (s StateAtEventAndReferences) Less(a, b int) bool {
+	return strings.Compare(s[a].EventID, s[b].EventID) < 0
+}
+
+func (s StateAtEventAndReferences) Len() int {
+	return len(s)
+}
+
+func (s StateAtEventAndReferences) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s StateAtEventAndReferences) EventIDs() string {
+	strs := make([]string, 0, len(s))
+	for _, r := range s {
+		strs = append(strs, r.EventID)
+	}
+	return "[" + strings.Join(strs, " ") + "]"
+}
+
 // An Event is a gomatrixserverlib.Event with the numeric event ID attached.
 // It is when performing bulk event lookup in the database.
 type Event struct {

roomserver/types/types_test.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"sort"
 	"testing"
 )
 
@@ -24,3 +25,66 @@ func TestDeduplicateStateEntries(t *testing.T) {
 		}
 	}
 }
+
+func TestStateKeyTupleSorter(t *testing.T) {
+	input := StateKeyTupleSorter{
+		{EventTypeNID: 1, EventStateKeyNID: 2},
+		{EventTypeNID: 1, EventStateKeyNID: 4},
+		{EventTypeNID: 2, EventStateKeyNID: 2},
+		{EventTypeNID: 1, EventStateKeyNID: 1},
+	}
+	want := []StateKeyTuple{
+		{EventTypeNID: 1, EventStateKeyNID: 1},
+		{EventTypeNID: 1, EventStateKeyNID: 2},
+		{EventTypeNID: 1, EventStateKeyNID: 4},
+		{EventTypeNID: 2, EventStateKeyNID: 2},
+	}
+	doNotWant := []StateKeyTuple{
+		{EventTypeNID: 0, EventStateKeyNID: 0},
+		{EventTypeNID: 1, EventStateKeyNID: 3},
+		{EventTypeNID: 2, EventStateKeyNID: 1},
+		{EventTypeNID: 3, EventStateKeyNID: 1},
+	}
+	wantTypeNIDs := []int64{1, 2}
+	wantStateKeyNIDs := []int64{1, 2, 4}
+
+	// Sort the input and check it's in the right order.
+	sort.Sort(input)
+	gotTypeNIDs, gotStateKeyNIDs := input.TypesAndStateKeysAsArrays()
+
+	for i := range want {
+		if input[i] != want[i] {
+			t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i])
+		}
+
+		if !input.contains(want[i]) {
+			t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i])
+		}
+	}
+
+	for i := range doNotWant {
+		if input.contains(doNotWant[i]) {
+			t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i])
+		}
+	}
+
+	if len(wantTypeNIDs) != len(gotTypeNIDs) {
+		t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
+	}
+
+	for i := range wantTypeNIDs {
+		if wantTypeNIDs[i] != gotTypeNIDs[i] {
+			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
+		}
+	}
+
+	if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) {
+		t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs)
+	}
+
+	for i := range wantStateKeyNIDs {
+		if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] {
+			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
+		}
+	}
+}

setup/config/config_appservice.go

@@ -50,11 +50,14 @@ func (c *AppServiceAPI) Defaults(generate bool) {
 }
 
 func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "app_service_api.internal_api.bind", string(c.InternalAPI.Connect))
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "app_service_api.internal_api.connect", string(c.InternalAPI.Connect))
 }
 
 // ApplicationServiceNamespace is the namespace that a specific application

setup/config/config_clientapi.go

@@ -75,19 +75,13 @@ func (c *ClientAPI) Defaults(generate bool) {
 }
 
 func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect))
-	if !isMonolith {
-		checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen))
-	}
-	if c.RecaptchaEnabled {
-		checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey))
-		checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey))
-		checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI))
-	}
 	c.TURN.Verify(configErrs)
 	c.RateLimiting.Verify(configErrs)
-
+	if c.RecaptchaEnabled {
+		checkNotEmpty(configErrs, "client_api.recaptcha_public_key", c.RecaptchaPublicKey)
+		checkNotEmpty(configErrs, "client_api.recaptcha_private_key", c.RecaptchaPrivateKey)
+		checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", c.RecaptchaSiteVerifyAPI)
+	}
 	// Ensure there is any spam counter measure when enabling registration
 	if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled {
 		if !c.RecaptchaEnabled {
@@ -101,6 +95,12 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 			)
 		}
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect))
+	checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen))
 }
 
 type TURN struct {

setup/config/config_federationapi.go

@@ -34,24 +34,24 @@ func (c *FederationAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7772"
 	c.InternalAPI.Connect = "http://localhost:7772"
 	c.ExternalAPI.Listen = "http://[::]:8072"
+	c.FederationMaxRetries = 16
+	c.DisableTLSValidation = false
 	c.Database.Defaults(10)
 	if generate {
 		c.Database.ConnectionString = "file:federationapi.db"
 	}
-
-	c.FederationMaxRetries = 16
-	c.DisableTLSValidation = false
 }
 
 func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
-	if !isMonolith {
-		checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
-	}
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
+	checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
 }
 
 // The config for setting a proxy to use for server->server requests

setup/config/config_jetstream.go

@@ -36,9 +36,10 @@ func (c *JetStream) Defaults(generate bool) {
 }
 
 func (c *JetStream) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	if isMonolith { // polylith required configs below
+		return
+	}
 	// If we are running in a polylith deployment then we need at least
 	// one NATS JetStream server to talk to.
-	if !isMonolith {
+	checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses)))
-		checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses)))
-	}
 }

setup/config/config_keyserver.go

@@ -18,9 +18,12 @@ func (c *KeyServer) Defaults(generate bool) {
 }
 
 func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "key_server.internal_api.bind", string(c.InternalAPI.Connect))
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "key_server.internal_api.connect", string(c.InternalAPI.Connect))
 }

setup/config/config_mediaapi.go

@@ -42,26 +42,19 @@ func (c *MediaAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7774"
 	c.InternalAPI.Connect = "http://localhost:7774"
 	c.ExternalAPI.Listen = "http://[::]:8074"
+	c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
+	c.MaxThumbnailGenerators = 10
 	c.Database.Defaults(5)
 	if generate {
 		c.Database.ConnectionString = "file:mediaapi.db"
 		c.BasePath = "./media_store"
 	}
-
-	c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
-	c.MaxThumbnailGenerators = 10
 }
 
 func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
-	if !isMonolith {
-		checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
-	}
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
 	}
-
 	checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))
 	checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes))
 	checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators))
@@ -70,4 +63,10 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 		checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width))
 		checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
+	checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
 }

setup/config/config_roomserver.go

@@ -18,9 +18,12 @@ func (c *RoomServer) Defaults(generate bool) {
 }
 
 func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "room_server.internal_ap.bind", string(c.InternalAPI.Connect))
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "room_server.internal_ap.connect", string(c.InternalAPI.Connect))
 }

setup/config/config_syncapi.go

@@ -22,12 +22,13 @@ func (c *SyncAPI) Defaults(generate bool) {
 }
 
 func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "sync_api.internal_api.bind", string(c.InternalAPI.Connect))
-	if !isMonolith {
-		checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
-	}
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect))
+	checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
 }

setup/config/config_userapi.go

@@ -26,19 +26,22 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
 func (c *UserAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7781"
 	c.InternalAPI.Connect = "http://localhost:7781"
+	c.BCryptCost = bcrypt.DefaultCost
+	c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
 	c.AccountDatabase.Defaults(10)
 	if generate {
 		c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
 	}
-	c.BCryptCost = bcrypt.DefaultCost
-	c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
 }
 
 func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
-	checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
 	}
-	checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
 }

syncapi/consumers/presence.go

@@ -138,9 +138,12 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
 	presence := msg.Header.Get("presence")
 	timestamp := msg.Header.Get("last_active_ts")
 	fromSync, _ := strconv.ParseBool(msg.Header.Get("from_sync"))
-
 	logrus.Debugf("syncAPI received presence event: %+v", msg.Header)
 
+	if fromSync { // do not process local presence changes; we already did this synchronously.
+		return true
+	}
+
 	ts, err := strconv.Atoi(timestamp)
 	if err != nil {
 		return true
@@ -151,15 +154,19 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
 		newMsg := msg.Header.Get("status_msg")
 		statusMsg = &newMsg
 	}
-	// OK is already checked, so no need to do it again
+	// already checked, so no need to check error
 	p, _ := types.PresenceFromString(presence)
-	pos, err := s.db.UpdatePresence(ctx, userID, p, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync)
-	if err != nil {
-		return true
-	}
-
-	s.stream.Advance(pos)
-	s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID)
 
+	s.EmitPresence(ctx, userID, p, statusMsg, ts, fromSync)
 	return true
 }
+
+func (s *PresenceConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) {
+	pos, err := s.db.UpdatePresence(ctx, userID, presence, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync)
+	if err != nil {
+		logrus.WithError(err).WithField("user", userID).WithField("presence", presence).Warn("failed to updated presence for user")
+		return
+	}
+	s.stream.Advance(pos)
+	s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID)
+}

syncapi/storage/storage_test.go

@@ -47,7 +47,7 @@ func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserver
 
 func TestWriteEvents(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		alice := test.NewUser()
+		alice := test.NewUser(t)
 		r := test.NewRoom(t, alice)
 		db, close := MustCreateDatabase(t, dbType)
 		defer close()
@@ -60,7 +60,7 @@ func TestRecentEventsPDU(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		db, close := MustCreateDatabase(t, dbType)
 		defer close()
-		alice := test.NewUser()
+		alice := test.NewUser(t)
 		// dummy room to make sure SQL queries are filtering on room ID
 		MustWriteEvents(t, db, test.NewRoom(t, alice).Events())
 
@@ -163,7 +163,7 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		db, close := MustCreateDatabase(t, dbType)
 		defer close()
-		alice := test.NewUser()
+		alice := test.NewUser(t)
 		r := test.NewRoom(t, alice)
 		for i := 0; i < 10; i++ {
 			r.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("hi %d", i)})

syncapi/storage/tables/output_room_events_test.go

@@ -45,7 +45,7 @@ func newOutputRoomEventsTable(t *testing.T, dbType test.DBType) (tables.Events,
 
 func TestOutputRoomEventsTable(t *testing.T) {
 	ctx := context.Background()
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		tab, db, close := newOutputRoomEventsTable(t, dbType)

syncapi/storage/tables/topology_test.go

@@ -40,7 +40,7 @@ func newTopologyTable(t *testing.T, dbType test.DBType) (tables.Topology, *sql.D
 
 func TestTopologyTable(t *testing.T) {
 	ctx := context.Background()
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		tab, db, close := newTopologyTable(t, dbType)

syncapi/sync/requestpool.go

@@ -53,19 +53,24 @@ type RequestPool struct {
 	streams  *streams.Streams
 	Notifier *notifier.Notifier
 	producer PresencePublisher
+	consumer PresenceConsumer
 }
 
 type PresencePublisher interface {
 	SendPresence(userID string, presence types.Presence, statusMsg *string) error
 }
 
+type PresenceConsumer interface {
+	EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool)
+}
+
 // NewRequestPool makes a new RequestPool
 func NewRequestPool(
 	db storage.Database, cfg *config.SyncAPI,
 	userAPI userapi.SyncUserAPI, keyAPI keyapi.SyncKeyAPI,
 	rsAPI roomserverAPI.SyncRoomserverAPI,
 	streams *streams.Streams, notifier *notifier.Notifier,
-	producer PresencePublisher, enableMetrics bool,
+	producer PresencePublisher, consumer PresenceConsumer, enableMetrics bool,
 ) *RequestPool {
 	if enableMetrics {
 		prometheus.MustRegister(
@@ -83,6 +88,7 @@ func NewRequestPool(
 		streams:  streams,
 		Notifier: notifier,
 		producer: producer,
+		consumer: consumer,
 	}
 	go rp.cleanLastSeen()
 	go rp.cleanPresence(db, time.Minute*5)
@@ -160,6 +166,13 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
 		logrus.WithError(err).Error("Unable to publish presence message from sync")
 		return
 	}
+
+	// now synchronously update our view of the world. It's critical we do this before calculating
+	// the /sync response else we may not return presence: online immediately.
+	rp.consumer.EmitPresence(
+		context.Background(), userID, presenceID, newPresence.ClientFields.StatusMsg,
+		int(gomatrixserverlib.AsTimestamp(time.Now())), true,
+	)
 }
 
 func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) {
@@ -238,122 +251,151 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 	waitingSyncRequests.Inc()
 	defer waitingSyncRequests.Dec()
 
-	currentPos := rp.Notifier.CurrentPosition()
+	// loop until we get some data
+	for {
+		startTime := time.Now()
+		currentPos := rp.Notifier.CurrentPosition()
 
-	if !rp.shouldReturnImmediately(syncReq, currentPos) {
+		// if the since token matches the current positions, wait via the notifier
-		timer := time.NewTimer(syncReq.Timeout) // case of timeout=0 is handled above
+		if !rp.shouldReturnImmediately(syncReq, currentPos) {
-		defer timer.Stop()
+			timer := time.NewTimer(syncReq.Timeout) // case of timeout=0 is handled above
+			defer timer.Stop()
 
-		userStreamListener := rp.Notifier.GetListener(*syncReq)
+			userStreamListener := rp.Notifier.GetListener(*syncReq)
-		defer userStreamListener.Close()
+			defer userStreamListener.Close()
 
-		giveup := func() util.JSONResponse {
+			giveup := func() util.JSONResponse {
-			syncReq.Log.Debugln("Responding to sync since client gave up or timeout was reached")
+				syncReq.Log.Debugln("Responding to sync since client gave up or timeout was reached")
-			syncReq.Response.NextBatch = syncReq.Since
+				syncReq.Response.NextBatch = syncReq.Since
-			// We should always try to include OTKs in sync responses, otherwise clients might upload keys
+				// We should always try to include OTKs in sync responses, otherwise clients might upload keys
-			// even if that's not required. See also:
+				// even if that's not required. See also:
-			// https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281
+				// https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281
-			err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
+				// Only try to get OTKs if the context isn't already done.
-			if err != nil {
+				if syncReq.Context.Err() == nil {
-				syncReq.Log.WithError(err).Error("failed to get OTK counts")
+					err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
+					if err != nil && err != context.Canceled {
+						syncReq.Log.WithError(err).Warn("failed to get OTK counts")
+					}
+				}
+				return util.JSONResponse{
+					Code: http.StatusOK,
+					JSON: syncReq.Response,
+				}
 			}
-			return util.JSONResponse{
+
-				Code: http.StatusOK,
+			select {
-				JSON: syncReq.Response,
+			case <-syncReq.Context.Done(): // Caller gave up
+				return giveup()
+
+			case <-timer.C: // Timeout reached
+				return giveup()
+
+			case <-userStreamListener.GetNotifyChannel(syncReq.Since):
+				syncReq.Log.Debugln("Responding to sync after wake-up")
+				currentPos.ApplyUpdates(userStreamListener.GetSyncPosition())
+			}
+		} else {
+			syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately")
+		}
+
+		if syncReq.Since.IsEmpty() {
+			// Complete sync
+			syncReq.Response.NextBatch = types.StreamingToken{
+				PDUPosition: rp.streams.PDUStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				TypingPosition: rp.streams.TypingStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				InvitePosition: rp.streams.InviteStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+			}
+		} else {
+			// Incremental sync
+			syncReq.Response.NextBatch = types.StreamingToken{
+				PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.PDUPosition, currentPos.PDUPosition,
+				),
+				TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.TypingPosition, currentPos.TypingPosition,
+				),
+				ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
+				),
+				InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.InvitePosition, currentPos.InvitePosition,
+				),
+				SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
+				),
+				AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
+				),
+				NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
+				),
+				DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
+				),
+				PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.PresencePosition, currentPos.PresencePosition,
+				),
+			}
+			// it's possible for there to be no updates for this user even though since < current pos,
+			// e.g busy servers with a quiet user. In this scenario, we don't want to return a no-op
+			// response immediately, so let's try this again but pretend they bumped their since token.
+			// If the incremental sync was processed very quickly then we expect the next loop to block
+			// with a notifier, but if things are slow it's entirely possible that currentPos is no
+			// longer the current position so we will hit this code path again. We need to do this and
+			// not return a no-op response because:
+			// - It's an inefficient use of bandwidth.
+			// - Some sytests which test 'waking up' sync rely on some sync requests to block, which
+			//   they weren't always doing, resulting in flakey tests.
+			if !syncReq.Response.HasUpdates() {
+				syncReq.Since = currentPos
+				// do not loop again if the ?timeout= is 0 as that means "return immediately"
+				if syncReq.Timeout > 0 {
+					syncReq.Timeout = syncReq.Timeout - time.Since(startTime)
+					if syncReq.Timeout < 0 {
+						syncReq.Timeout = 0
+					}
+					continue
+				}
 			}
 		}
 
-		select {
+		return util.JSONResponse{
-		case <-syncReq.Context.Done(): // Caller gave up
+			Code: http.StatusOK,
-			return giveup()
+			JSON: syncReq.Response,
-
-		case <-timer.C: // Timeout reached
-			return giveup()
-
-		case <-userStreamListener.GetNotifyChannel(syncReq.Since):
-			syncReq.Log.Debugln("Responding to sync after wake-up")
-			currentPos.ApplyUpdates(userStreamListener.GetSyncPosition())
 		}
-	} else {
-		syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately")
-	}
-
-	if syncReq.Since.IsEmpty() {
-		// Complete sync
-		syncReq.Response.NextBatch = types.StreamingToken{
-			PDUPosition: rp.streams.PDUStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			TypingPosition: rp.streams.TypingStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			InvitePosition: rp.streams.InviteStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-		}
-	} else {
-		// Incremental sync
-		syncReq.Response.NextBatch = types.StreamingToken{
-			PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.PDUPosition, currentPos.PDUPosition,
-			),
-			TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.TypingPosition, currentPos.TypingPosition,
-			),
-			ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
-			),
-			InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.InvitePosition, currentPos.InvitePosition,
-			),
-			SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
-			),
-			AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
-			),
-			NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
-			),
-			DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
-			),
-			PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.PresencePosition, currentPos.PresencePosition,
-			),
-		}
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: syncReq.Response,
 	}
 }
 

syncapi/sync/requestpool_test.go

@@ -38,6 +38,12 @@ func (d dummyDB) MaxStreamPositionForPresence(ctx context.Context) (types.Stream
 	return 0, nil
 }
 
+type dummyConsumer struct{}
+
+func (d dummyConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) {
+
+}
+
 func TestRequestPool_updatePresence(t *testing.T) {
 	type args struct {
 		presence string
@@ -45,6 +51,7 @@ func TestRequestPool_updatePresence(t *testing.T) {
 		sleep    time.Duration
 	}
 	publisher := &dummyPublisher{}
+	consumer := &dummyConsumer{}
 	syncMap := sync.Map{}
 
 	tests := []struct {
@@ -101,6 +108,7 @@ func TestRequestPool_updatePresence(t *testing.T) {
 	rp := &RequestPool{
 		presence: &syncMap,
 		producer: publisher,
+		consumer: consumer,
 		cfg: &config.SyncAPI{
 			Matrix: &config.Global{
 				JetStream: config.JetStream{

syncapi/syncapi.go

@@ -64,8 +64,17 @@ func AddPublicRoutes(
 		Topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
 		JetStream: js,
 	}
+	presenceConsumer := consumers.NewPresenceConsumer(
+		base.ProcessContext, cfg, js, natsClient, syncDB,
+		notifier, streams.PresenceStreamProvider,
+		userAPI,
+	)
 
-	requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer, base.EnableMetrics)
+	requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer, presenceConsumer, base.EnableMetrics)
+
+	if err = presenceConsumer.Start(); err != nil {
+		logrus.WithError(err).Panicf("failed to start presence consumer")
+	}
 
 	userAPIStreamEventProducer := &producers.UserAPIStreamEventProducer{
 		JetStream: js,
@@ -131,15 +140,6 @@ func AddPublicRoutes(
 		logrus.WithError(err).Panicf("failed to start receipts consumer")
 	}
 
-	presenceConsumer := consumers.NewPresenceConsumer(
-		base.ProcessContext, cfg, js, natsClient, syncDB,
-		notifier, streams.PresenceStreamProvider,
-		userAPI,
-	)
-	if err = presenceConsumer.Start(); err != nil {
-		logrus.WithError(err).Panicf("failed to start presence consumer")
-	}
-
 	routing.Setup(
 		base.PublicClientAPIMux, requestPool, syncDB, userAPI,
 		rsAPI, cfg, base.Caches,

syncapi/syncapi_test.go

@@ -15,9 +15,11 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
+	"github.com/tidwall/gjson"
 )
 
 type syncRoomserverAPI struct {
@@ -86,7 +88,7 @@ func TestSyncAPIAccessTokens(t *testing.T) {
 }
 
 func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
-	user := test.NewUser()
+	user := test.NewUser(t)
 	room := test.NewRoom(t, user)
 	alice := userapi.Device{
 		ID:          "ALICEID",
@@ -96,14 +98,14 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
 		AccountType: userapi.AccountTypeUser,
 	}
 
-	base, close := test.CreateBaseDendrite(t, dbType)
+	base, close := testrig.CreateBaseDendrite(t, dbType)
 	defer close()
 
 	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
 	msgs := toNATSMsgs(t, base, room.Events())
 	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
-	test.MustPublishMsgs(t, jsctx, msgs...)
+	testrig.MustPublishMsgs(t, jsctx, msgs...)
 
 	testCases := []struct {
 		name            string
@@ -173,7 +175,7 @@ func TestSyncAPICreateRoomSyncEarly(t *testing.T) {
 }
 
 func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
-	user := test.NewUser()
+	user := test.NewUser(t)
 	room := test.NewRoom(t, user)
 	alice := userapi.Device{
 		ID:          "ALICEID",
@@ -183,7 +185,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
 		AccountType: userapi.AccountTypeUser,
 	}
 
-	base, close := test.CreateBaseDendrite(t, dbType)
+	base, close := testrig.CreateBaseDendrite(t, dbType)
 	defer close()
 
 	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
@@ -198,7 +200,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
 	sinceTokens := make([]string, len(msgs))
 	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
 	for i, msg := range msgs {
-		test.MustPublishMsgs(t, jsctx, msg)
+		testrig.MustPublishMsgs(t, jsctx, msg)
 		time.Sleep(100 * time.Millisecond)
 		w := httptest.NewRecorder()
 		base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
@@ -255,6 +257,60 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
 	}
 }
 
+// Test that if we hit /sync we get back presence: online, regardless of whether messages get delivered
+// via NATS. Regression test for a flakey test "User sees their own presence in a sync"
+func TestSyncAPIUpdatePresenceImmediately(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		testSyncAPIUpdatePresenceImmediately(t, dbType)
+	})
+}
+
+func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) {
+	user := test.NewUser(t)
+	alice := userapi.Device{
+		ID:          "ALICEID",
+		UserID:      user.ID,
+		AccessToken: "ALICE_BEARER_TOKEN",
+		DisplayName: "Alice",
+		AccountType: userapi.AccountTypeUser,
+	}
+
+	base, close := testrig.CreateBaseDendrite(t, dbType)
+	base.Cfg.Global.Presence.EnableOutbound = true
+	base.Cfg.Global.Presence.EnableInbound = true
+	defer close()
+
+	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, &syncKeyAPI{})
+	w := httptest.NewRecorder()
+	base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+		"access_token": alice.AccessToken,
+		"timeout":      "0",
+		"set_presence": "online",
+	})))
+	if w.Code != 200 {
+		t.Fatalf("got HTTP %d want %d", w.Code, 200)
+	}
+	var res types.Response
+	if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
+		t.Errorf("failed to decode response body: %s", err)
+	}
+	if len(res.Presence.Events) != 1 {
+		t.Fatalf("expected 1 presence events, got: %+v", res.Presence.Events)
+	}
+	if res.Presence.Events[0].Sender != alice.UserID {
+		t.Errorf("sender: got %v want %v", res.Presence.Events[0].Sender, alice.UserID)
+	}
+	if res.Presence.Events[0].Type != "m.presence" {
+		t.Errorf("type: got %v want %v", res.Presence.Events[0].Type, "m.presence")
+	}
+	if gjson.ParseBytes(res.Presence.Events[0].Content).Get("presence").Str != "online" {
+		t.Errorf("content: not online,  got %v", res.Presence.Events[0].Content)
+	}
+
+}
+
 func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverlib.HeaderedEvent) []*nats.Msg {
 	result := make([]*nats.Msg, len(input))
 	for i, ev := range input {
@@ -262,7 +318,7 @@ func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverli
 		if ev.StateKey() != nil {
 			addsStateIDs = append(addsStateIDs, ev.EventID())
 		}
-		result[i] = test.NewOutputEventMsg(t, base, ev.RoomID(), api.OutputEvent{
+		result[i] = testrig.NewOutputEventMsg(t, base, ev.RoomID(), api.OutputEvent{
 			Type: rsapi.OutputTypeNewRoomEvent,
 			NewRoomEvent: &rsapi.OutputNewRoomEvent{
 				Event:             ev,

syncapi/types/types.go

@@ -350,6 +350,19 @@ type Response struct {
 	DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"`
 }
 
+func (r *Response) HasUpdates() bool {
+	// purposefully exclude DeviceListsOTKCount as we always include them
+	return (len(r.AccountData.Events) > 0 ||
+		len(r.Presence.Events) > 0 ||
+		len(r.Rooms.Invite) > 0 ||
+		len(r.Rooms.Join) > 0 ||
+		len(r.Rooms.Leave) > 0 ||
+		len(r.Rooms.Peek) > 0 ||
+		len(r.ToDevice.Events) > 0 ||
+		len(r.DeviceLists.Changed) > 0 ||
+		len(r.DeviceLists.Left) > 0)
+}
+
 // NewResponse creates an empty response with initialised maps.
 func NewResponse() *Response {
 	res := Response{}

test/db.go

@@ -44,8 +44,9 @@ func fatalError(t *testing.T, format string, args ...interface{}) {
 }
 
 func createLocalDB(t *testing.T, dbName string) {
-	if !Quiet {
+	if _, err := exec.LookPath("createdb"); err != nil {
-		t.Log("Note: tests require a postgres install accessible to the current user")
+		fatalError(t, "Note: tests require a postgres install accessible to the current user")
+		return
 	}
 	createDB := exec.Command("createdb", dbName)
 	if !Quiet {
@@ -63,6 +64,9 @@ func createRemoteDB(t *testing.T, dbName, user, connStr string) {
 	if err != nil {
 		fatalError(t, "failed to open postgres conn with connstr=%s : %s", connStr, err)
 	}
+	if err = db.Ping(); err != nil {
+		fatalError(t, "failed to open postgres conn with connstr=%s : %s", connStr, err)
+	}
 	_, err = db.Exec(fmt.Sprintf(`CREATE DATABASE %s;`, dbName))
 	if err != nil {
 		pqErr, ok := err.(*pq.Error)

test/event.go

@@ -52,6 +52,24 @@ func WithUnsigned(unsigned interface{}) eventModifier {
 	}
 }
 
+func WithKeyID(keyID gomatrixserverlib.KeyID) eventModifier {
+	return func(e *eventMods) {
+		e.keyID = keyID
+	}
+}
+
+func WithPrivateKey(pkey ed25519.PrivateKey) eventModifier {
+	return func(e *eventMods) {
+		e.privKey = pkey
+	}
+}
+
+func WithOrigin(origin gomatrixserverlib.ServerName) eventModifier {
+	return func(e *eventMods) {
+		e.origin = origin
+	}
+}
+
 // Reverse a list of events
 func Reversed(in []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
 	out := make([]*gomatrixserverlib.HeaderedEvent, len(in))

test/http.go

@@ -2,10 +2,15 @@ package test
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
+	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
+	"path/filepath"
+	"sync"
 	"testing"
 )
 
@@ -43,3 +48,45 @@ func NewRequest(t *testing.T, method, path string, opts ...HTTPRequestOpt) *http
 	}
 	return req
 }
+
+// ListenAndServe will listen on a random high-numbered port and attach the given router.
+// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed.
+func ListenAndServe(t *testing.T, router http.Handler, withTLS bool) (apiURL string, cancel func()) {
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatalf("failed to listen: %s", err)
+	}
+	port := listener.Addr().(*net.TCPAddr).Port
+	srv := http.Server{}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		srv.Handler = router
+		var err error
+		if withTLS {
+			certFile := filepath.Join(t.TempDir(), "dendrite.cert")
+			keyFile := filepath.Join(t.TempDir(), "dendrite.key")
+			err = NewTLSKey(keyFile, certFile)
+			if err != nil {
+				t.Errorf("failed to make TLS key: %s", err)
+				return
+			}
+			err = srv.ServeTLS(listener, certFile, keyFile)
+		} else {
+			err = srv.Serve(listener)
+		}
+		if err != nil && err != http.ErrServerClosed {
+			t.Logf("Listen failed: %s", err)
+		}
+	}()
+	s := ""
+	if withTLS {
+		s = "s"
+	}
+	return fmt.Sprintf("http%s://localhost:%d", s, port), func() {
+		_ = srv.Shutdown(context.Background())
+		wg.Wait()
+	}
+}