Merge branch 'main' into implement-push-notifications

CHANGES.md

@@ -1,5 +1,39 @@
 # Changelog
 
+## Dendrite 0.6.4 (2022-02-21)
+
+### Features
+
+* All Client-Server API endpoints are now available under the `/v3` namespace
+* The `/whoami` response format now matches the latest Matrix spec version
+* Support added for the `/context` endpoint, which should help clients to render quote-replies correctly
+* Accounts now have an optional account type field, allowing admin accounts to be created
+* Server notices are now supported
+* Refactored the user API storage to deduplicate a significant amount of code, as well as merging both user API databases into a single database
+  * The account database is now used for all user API storage and the device database is now obsolete
+  * For some installations that have separate account and device databases, this may result in access tokens being revoked and client sessions being logged out — users may need to log in again
+  * The above can be avoided by moving the `device_devices` table into the account database manually
+* Guest registration can now be separately disabled with the new `client_api.guests_disabled` configuration option
+* Outbound connections now obey proxy settings from the environment, deprecating the `federation_api.proxy_outbound` configuration options
+
+### Fixes
+
+* The roomserver input API will now strictly consume only one database transaction per room, which should prevent situations where the roomserver can deadlock waiting for database connections to become available
+* Room joins will now fall back to federation if the local room state is insufficient to create a membership event
+* Create events are now correctly filtered from federation `/send` transactions
+* Excessive logging when federation is disabled should now be fixed
+* Dendrite will no longer panic if trying to retire an invite event that has not been seen yet
+* The device list updater will now wait for longer after a connection issue, rather than flooding the logs with errors
+* The device list updater will no longer produce unnecessary output events for federated key updates with no changes, which should help to reduce CPU usage
+* Local device name changes will now generate key change events correctly
+* The sync API will now try to share device list update notifications even if all state key NIDs cannot be fetched
+* An off-by-one error in the sync stream token handling which could result in a crash has been fixed
+* State events will no longer be re-sent unnecessary by the roomserver to other components if they have already been sent, which should help to reduce the NATS message sizes on the roomserver output topic in some cases
+* The roomserver input API now uses the process context and should handle graceful shutdowns better
+* Guest registration is now correctly disabled when the `client_api.registration_disabled` configuration option is set
+* One-time encryption keys are now cleaned up correctly when a device is logged out or removed
+* Invalid state snapshots in the state storage refactoring migration are now reset rather than causing a panic at startup
+
 ## Dendrite 0.6.3 (2022-02-10)
 
 ### Features

clientapi/routing/state.go

@@ -235,7 +235,7 @@ func OnIncomingStateTypeRequest(
 		}
 		// If the user has never been in the room then stop at this point.
 		// We won't tell the user about a room they have never joined.
-		if !membershipRes.HasBeenInRoom {
+		if !membershipRes.HasBeenInRoom || membershipRes.Membership == gomatrixserverlib.Ban {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),

cmd/dendrite-demo-yggdrasil/main.go

@@ -21,8 +21,11 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
@@ -42,8 +45,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/gomatrixserverlib"
-
 	"github.com/sirupsen/logrus"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -63,20 +64,23 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	/*
+
-		ygg.SetMulticastEnabled(true)
+	// iterate through the cli args and check if the config flag was set
-		if instancePeer != nil && *instancePeer != "" {
+	configFlagSet := false
-			if err = ygg.SetStaticPeer(*instancePeer); err != nil {
+	for _, arg := range os.Args {
-				logrus.WithError(err).Error("Failed to set static peer")
+		if arg == "--config" || arg == "-config" {
+			configFlagSet = true
+			break
 		}
 	}
-	*/
 
 	cfg := &config.Dendrite{}
+
+	// use custom config if config flag is set
+	if configFlagSet {
+		cfg = setup.ParseFlags(true)
+	} else {
 		cfg.Defaults(true)
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Global.PrivateKey = ygg.PrivateKey()
-	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
 		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
@@ -90,6 +94,12 @@ func main() {
 		if err = cfg.Derive(); err != nil {
 			panic(err)
 		}
+	}
+
+	// always override ServerName, PrivateKey and KeyID
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
+	cfg.Global.PrivateKey = ygg.PrivateKey()
+	cfg.Global.KeyID = signing.KeyID
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck

federationapi/internal/perform.go

@@ -554,7 +554,6 @@ func (r *FederationInternalAPI) PerformInvite(
 	if err != nil {
 		return fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err)
 	}
-	logrus.Infof("GOT INVITE RESPONSE %s", string(inviteRes.Event))
 
 	inviteEvent, err := inviteRes.Event.UntrustedEvent(request.RoomVersion)
 	if err != nil {

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 6
-	VersionPatch = 3
+	VersionPatch = 4
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/api/api.go

@@ -15,6 +15,7 @@
 package api
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"strings"
@@ -73,6 +74,26 @@ type DeviceMessage struct {
 	DeviceChangeID int64
 }
 
+// DeviceKeysEqual returns true if the device keys updates contain the
+// same display name and key JSON. This will return false if either of
+// the updates is not a device keys update, or if the user ID/device ID
+// differ between the two.
+func (m1 *DeviceMessage) DeviceKeysEqual(m2 *DeviceMessage) bool {
+	if m1.DeviceKeys == nil || m2.DeviceKeys == nil {
+		return false
+	}
+	if m1.UserID != m2.UserID || m1.DeviceID != m2.DeviceID {
+		return false
+	}
+	if m1.DisplayName != m2.DisplayName {
+		return false // different display names
+	}
+	if len(m1.KeyJSON) == 0 || len(m2.KeyJSON) == 0 {
+		return false // either is empty
+	}
+	return bytes.Equal(m1.KeyJSON, m2.KeyJSON)
+}
+
 // DeviceKeys represents a set of device keys for a single device
 // https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-keys-upload
 type DeviceKeys struct {

keyserver/internal/device_list_update.go

@@ -224,7 +224,7 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib.
 	}).Info("DeviceListUpdater.Update")
 
 	// if we haven't missed anything update the database and notify users
-	if exists {
+	if exists || event.Deleted {
 		k := event.Keys
 		if event.Deleted {
 			k = nil
@@ -241,14 +241,33 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib.
 				StreamID: event.StreamID,
 			},
 		}
+
+		// DeviceKeysJSON will side-effect modify this, so it needs
+		// to be a copy, not sharing any pointers with the above.
+		deviceKeysCopy := *keys[0].DeviceKeys
+		deviceKeysCopy.KeyJSON = nil
+		existingKeys := []api.DeviceMessage{
+			{
+				Type:       keys[0].Type,
+				DeviceKeys: &deviceKeysCopy,
+				StreamID:   keys[0].StreamID,
+			},
+		}
+
+		// fetch what keys we had already and only emit changes
+		if err = u.db.DeviceKeysJSON(ctx, existingKeys); err != nil {
+			// non-fatal, log and continue
+			util.GetLogger(ctx).WithError(err).WithField("user_id", event.UserID).Errorf(
+				"failed to query device keys json for calculating diffs",
+			)
+		}
+
 		err = u.db.StoreRemoteDeviceKeys(ctx, keys, nil)
 		if err != nil {
 			return false, fmt.Errorf("failed to store remote device keys for %s (%s): %w", event.UserID, event.DeviceID, err)
 		}
-		// ALWAYS emit key changes when we've been poked over federation even if there's no change
+
-		// just in case this poke is important for something.
+		if err = emitDeviceKeyChanges(u.producer, existingKeys, keys, false); err != nil {
-		err = u.producer.ProduceKeyChanges(keys)
-		if err != nil {
 			return false, fmt.Errorf("failed to produce device key changes for %s (%s): %w", event.UserID, event.DeviceID, err)
 		}
 		return false, nil
@@ -454,7 +473,7 @@ func (u *DeviceListUpdater) updateDeviceList(res *gomatrixserverlib.RespUserDevi
 	if err != nil {
 		return fmt.Errorf("failed to mark device list as fresh: %w", err)
 	}
-	err = emitDeviceKeyChanges(u.producer, existingKeys, keys)
+	err = emitDeviceKeyChanges(u.producer, existingKeys, keys, false)
 	if err != nil {
 		return fmt.Errorf("failed to emit key changes for fresh device list: %w", err)
 	}

keyserver/internal/internal.go

@@ -648,7 +648,7 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per
 		}
 		return
 	}
-	err = emitDeviceKeyChanges(a.Producer, existingKeys, keysToStore)
+	err = emitDeviceKeyChanges(a.Producer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates)
 	if err != nil {
 		util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err)
 	}
@@ -710,7 +710,11 @@ func (a *KeyInternalAPI) uploadOneTimeKeys(ctx context.Context, req *api.Perform
 
 }
 
-func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.DeviceMessage) error {
+func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.DeviceMessage, onlyUpdateDisplayName bool) error {
+	// if we only want to update the display names, we can skip the checks below
+	if onlyUpdateDisplayName {
+		return producer.ProduceKeyChanges(new)
+	}
 	// find keys in new that are not in existing
 	var keysAdded []api.DeviceMessage
 	for _, newKey := range new {
@@ -718,7 +722,7 @@ func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.Device
 		for _, existingKey := range existing {
 			// Do not treat the absence of keys as equal, or else we will not emit key changes
 			// when users delete devices which never had a key to begin with as both KeyJSONs are nil.
-			if bytes.Equal(existingKey.KeyJSON, newKey.KeyJSON) && len(existingKey.KeyJSON) > 0 {
+			if existingKey.DeviceKeysEqual(&newKey) {
 				exists = true
 				break
 			}

roomserver/internal/perform/perform_join.go

@@ -29,6 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
 	"github.com/matrix-org/dendrite/roomserver/internal/query"
 	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
@@ -367,8 +368,16 @@ func buildEvent(
 		StateToFetch: eventsNeeded.Tuples(),
 	}, &queryRes)
 	if err != nil {
+		switch err.(type) {
+		case types.MissingStateError:
+			// We know something about the room but the state seems to be
+			// insufficient to actually build a new event, so in effect we
+			// had might as well treat the room as if it doesn't exist.
+			return nil, nil, eventutil.ErrRoomNoExists
+		default:
 			return nil, nil, fmt.Errorf("QueryLatestEventsAndState: %w", err)
 		}
+	}
 
 	ev, err := eventutil.BuildEvent(ctx, builder, cfg, time.Now(), &eventsNeeded, &queryRes)
 	if err != nil {

roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go

@@ -256,23 +256,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
-		var debugEventID, debugRoomID string
+		var res sql.Result
-		var debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth int64
+		var c int64
-		err = tx.QueryRow(
+		res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid)
-			`SELECT event_id, event_type_nid, event_state_key_nid, roomserver_events.state_snapshot_nid, depth, room_id FROM roomserver_events
+		if err != nil && err != sql.ErrNoRows {
-				JOIN roomserver_rooms ON roomserver_rooms.room_nid = roomserver_events.room_nid WHERE roomserver_events.state_snapshot_nid < $1 AND roomserver_events.state_snapshot_nid != 0`, maxsnapshotid,
+			return fmt.Errorf("failed to reset invalid state snapshots: %w", err)
-		).Scan(&debugEventID, &debugEventTypeNID, &debugStateKeyNID, &debugSnapNID, &debugDepth, &debugRoomID)
+		}
-		if err != nil {
+		if c, err = res.RowsAffected(); err != nil {
-			logrus.Errorf("cannot extract debug info: %v", err)
+			return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err)
-		} else {
+		} else if c != count {
-			logrus.Errorf(
+			return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c)
-				"Affected row: event_id=%v room_id=%v type=%v state_key=%v snapshot=%v depth=%v",
-				debugEventID, debugRoomID, debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth,
-			)
-			logrus.Errorf("To fix this manually, run this query first then retry the migration: "+
-				"UPDATE roomserver_events SET state_snapshot_nid=0 WHERE event_id='%v'", debugEventID)
 		}
-		return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
 	}
 	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)

roomserver/storage/postgres/state_snapshot_table.go

@@ -134,7 +134,7 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 		return nil, err
 	}
 	if i != len(stateNIDs) {
-		return nil, fmt.Errorf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))
+		return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs)))
 	}
 	return results, nil
 }

roomserver/storage/shared/storage.go

@@ -13,6 +13,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
 )
 
@@ -1101,7 +1102,7 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs []string)
 		return nil, err
 	}
 	if len(nidToUserID) != len(userNIDToCount) {
-		return nil, fmt.Errorf("found %d users but only have state key nids for %d of them", len(userNIDToCount), len(nidToUserID))
+		logrus.Warnf("SelectJoinedUsersSetForRooms found %d users but BulkSelectEventStateKey only returned state key NIDs for %d of them", len(userNIDToCount), len(nidToUserID))
 	}
 	result := make(map[string]int, len(userNIDToCount))
 	for nid, count := range userNIDToCount {

roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go

@@ -179,7 +179,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
-		return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
+		var res sql.Result
+		var c int64
+		res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID)
+		if err != nil && err != sql.ErrNoRows {
+			return fmt.Errorf("failed to reset invalid state snapshots: %w", err)
+		}
+		if c, err = res.RowsAffected(); err != nil {
+			return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err)
+		} else if c != count {
+			return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c)
+		}
 	}
 	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)

roomserver/storage/sqlite3/state_snapshot_table.go

@@ -137,7 +137,7 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 		}
 	}
 	if i != len(stateNIDs) {
-		return nil, fmt.Errorf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))
+		return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs)))
 	}
 	return results, nil
 }

roomserver/types/types.go

@@ -213,6 +213,12 @@ type MissingEventError string
 
 func (e MissingEventError) Error() string { return string(e) }
 
+// A MissingStateError is an error that happened because the roomserver was
+// missing requested state snapshots from its databases.
+type MissingStateError string
+
+func (e MissingStateError) Error() string { return string(e) }
+
 // A RejectedError is returned when an event is stored as rejected. The error
 // contains the reason why.
 type RejectedError string

syncapi/routing/context.go

@@ -0,0 +1,191 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"database/sql"
+	"encoding/json"
+	"net/http"
+	"strconv"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	roomserver "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+)
+
+type ContextRespsonse struct {
+	End          string                          `json:"end"`
+	Event        gomatrixserverlib.ClientEvent   `json:"event"`
+	EventsAfter  []gomatrixserverlib.ClientEvent `json:"events_after,omitempty"`
+	EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before,omitempty"`
+	Start        string                          `json:"start"`
+	State        []gomatrixserverlib.ClientEvent `json:"state"`
+}
+
+func Context(
+	req *http.Request, device *userapi.Device,
+	rsAPI roomserver.RoomserverInternalAPI,
+	syncDB storage.Database,
+	roomID, eventID string,
+) util.JSONResponse {
+	filter, err := parseContextParams(req)
+	if err != nil {
+		errMsg := ""
+		switch err.(type) {
+		case *json.InvalidUnmarshalError:
+			errMsg = "unable to parse filter"
+		case *strconv.NumError:
+			errMsg = "unable to parse limit"
+		}
+		return util.JSONResponse{
+			Code:    http.StatusBadRequest,
+			JSON:    jsonerror.InvalidParam(errMsg),
+			Headers: nil,
+		}
+	}
+	filter.Rooms = append(filter.Rooms, roomID)
+
+	ctx := req.Context()
+	membershipRes := roomserver.QueryMembershipForUserResponse{}
+	membershipReq := roomserver.QueryMembershipForUserRequest{UserID: device.UserID, RoomID: roomID}
+	if err = rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes); err != nil {
+		logrus.WithError(err).Error("unable to fo membership")
+		return jsonerror.InternalServerError()
+	}
+
+	stateFilter := gomatrixserverlib.StateFilter{
+		Limit:                   100,
+		NotSenders:              filter.NotSenders,
+		NotTypes:                filter.NotTypes,
+		Senders:                 filter.Senders,
+		Types:                   filter.Types,
+		LazyLoadMembers:         filter.LazyLoadMembers,
+		IncludeRedundantMembers: filter.IncludeRedundantMembers,
+		NotRooms:                filter.NotRooms,
+		Rooms:                   filter.Rooms,
+		ContainsURL:             filter.ContainsURL,
+	}
+
+	// TODO: Get the actual state at the last event returned by SelectContextAfterEvent
+	state, _ := syncDB.CurrentState(ctx, roomID, &stateFilter, nil)
+	// verify the user is allowed to see the context for this room/event
+	for _, x := range state {
+		var hisVis string
+		hisVis, err = x.HistoryVisibility()
+		if err != nil {
+			continue
+		}
+		allowed := hisVis == gomatrixserverlib.WorldReadable || membershipRes.Membership == gomatrixserverlib.Join
+		if !allowed {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden("User is not allowed to query context"),
+			}
+		}
+	}
+
+	id, requestedEvent, err := syncDB.SelectContextEvent(ctx, roomID, eventID)
+	if err != nil {
+		logrus.WithError(err).WithField("eventID", eventID).Error("unable to find requested event")
+		return jsonerror.InternalServerError()
+	}
+
+	eventsBefore, err := syncDB.SelectContextBeforeEvent(ctx, id, roomID, filter)
+	if err != nil && err != sql.ErrNoRows {
+		logrus.WithError(err).Error("unable to fetch before events")
+		return jsonerror.InternalServerError()
+	}
+
+	_, eventsAfter, err := syncDB.SelectContextAfterEvent(ctx, id, roomID, filter)
+	if err != nil && err != sql.ErrNoRows {
+		logrus.WithError(err).Error("unable to fetch after events")
+		return jsonerror.InternalServerError()
+	}
+
+	eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatAll)
+	eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatAll)
+	newState := applyLazyLoadMembers(filter, eventsAfterClient, eventsBeforeClient, state)
+
+	response := ContextRespsonse{
+		Event:        gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll),
+		EventsAfter:  eventsAfterClient,
+		EventsBefore: eventsBeforeClient,
+		State:        gomatrixserverlib.HeaderedToClientEvents(newState, gomatrixserverlib.FormatAll),
+	}
+
+	if len(response.State) > filter.Limit {
+		response.State = response.State[len(response.State)-filter.Limit:]
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: response,
+	}
+}
+
+func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent, state []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
+	if filter == nil || !filter.LazyLoadMembers {
+		return state
+	}
+	allEvents := append(eventsBefore, eventsAfter...)
+	x := make(map[string]bool)
+	// get members who actually send an event
+	for _, e := range allEvents {
+		x[e.Sender] = true
+	}
+
+	newState := []*gomatrixserverlib.HeaderedEvent{}
+	for _, event := range state {
+		if event.Type() != gomatrixserverlib.MRoomMember {
+			newState = append(newState, event)
+		} else {
+			// did the user send an event?
+			if x[event.Sender()] {
+				newState = append(newState, event)
+			}
+		}
+	}
+	return newState
+}
+
+func parseContextParams(req *http.Request) (*gomatrixserverlib.RoomEventFilter, error) {
+	// Default room filter
+	filter := &gomatrixserverlib.RoomEventFilter{Limit: 10}
+
+	l := req.URL.Query().Get("limit")
+	f := req.URL.Query().Get("filter")
+	if l != "" {
+		limit, err := strconv.Atoi(l)
+		if err != nil {
+			return nil, err
+		}
+		// NOTSPEC: feels like a good idea to have an upper bound limit
+		if limit > 100 {
+			limit = 100
+		}
+		filter.Limit = limit
+	}
+	if f != "" {
+		if err := json.Unmarshal([]byte(f), &filter); err != nil {
+			return nil, err
+		}
+	}
+
+	return filter, nil
+}

syncapi/routing/context_test.go

@@ -0,0 +1,68 @@
+package routing
+
+import (
+	"net/http"
+	"reflect"
+	"testing"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func Test_parseContextParams(t *testing.T) {
+
+	noParamsReq, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI", nil)
+	limit2Req, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI&limit=2", nil)
+	limit10000Req, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI&limit=10000", nil)
+	invalidLimitReq, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI&limit=100as", nil)
+	lazyLoadReq, _ := http.NewRequest("GET", "https://localhost:8800//_matrix/client/r0/rooms/!kvEtX3rFamfwKHO3:localhost:8800/context/%24GjmkRbajRHy8_cxcSbUU4qF_njV8yHeLphI2azTrPaI?limit=2&filter=%7B+%22lazy_load_members%22+%3A+true+%7D&access_token=t1Njzm74w3G40CJ5xrlf1V2haXom0z0Iq1qyyVWhbVo", nil)
+	invalidFilterReq, _ := http.NewRequest("GET", "https://localhost:8800//_matrix/client/r0/rooms/!kvEtX3rFamfwKHO3:localhost:8800/context/%24GjmkRbajRHy8_cxcSbUU4qF_njV8yHeLphI2azTrPaI?limit=2&filter=%7B+%22lazy_load_members%22+%3A+true&access_token=t1Njzm74w3G40CJ5xrlf1V2haXom0z0Iq1qyyVWhbVo", nil)
+	tests := []struct {
+		name       string
+		req        *http.Request
+		wantFilter *gomatrixserverlib.RoomEventFilter
+		wantErr    bool
+	}{
+		{
+			name:       "no params set",
+			req:        noParamsReq,
+			wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 10},
+		},
+		{
+			name:       "limit 2 param set",
+			req:        limit2Req,
+			wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 2},
+		},
+		{
+			name:       "limit 10000 param set",
+			req:        limit10000Req,
+			wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 100},
+		},
+		{
+			name:       "filter lazy_load_members param set",
+			req:        lazyLoadReq,
+			wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 2, LazyLoadMembers: true},
+		},
+		{
+			name:    "invalid limit req",
+			req:     invalidLimitReq,
+			wantErr: true,
+		},
+		{
+			name:    "invalid filter req",
+			req:     invalidFilterReq,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotFilter, err := parseContextParams(tt.req)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseContextParams() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotFilter, tt.wantFilter) {
+				t.Errorf("parseContextParams() gotFilter = %v, want %v", gotFilter, tt.wantFilter)
+			}
+		})
+	}
+}

syncapi/routing/routing.go

@@ -77,4 +77,19 @@ func Setup(
 	v3mux.Handle("/keys/changes", httputil.MakeAuthAPI("keys_changes", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		return srp.OnIncomingKeyChangeRequest(req, device)
 	})).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomId}/context/{eventId}",
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+
+			return Context(
+				req, device,
+				rsAPI, syncDB,
+				vars["roomId"], vars["eventId"],
+			)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
 }

syncapi/storage/interface.go

@@ -145,4 +145,8 @@ type Database interface {
 
 	// GetUserUnreadNotificationCounts returns statistics per room a user is interested in.
 	GetUserUnreadNotificationCounts(ctx context.Context, userID string, from, to types.StreamPosition) (map[string]*eventutil.NotificationData, error)
+
+	SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error)
+	SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error)
+	SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error)
 }

syncapi/storage/postgres/output_room_events_table.go

@@ -130,6 +130,25 @@ const selectStateInRangeSQL = "" +
 const deleteEventsForRoomSQL = "" +
 	"DELETE FROM syncapi_output_room_events WHERE room_id = $1"
 
+const selectContextEventSQL = "" +
+	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2"
+
+const selectContextBeforeEventSQL = "" +
+	"SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" +
+	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
+	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
+	" AND ( $6::text[] IS NULL OR     type LIKE ANY($6)  )" +
+	" AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" +
+	" ORDER BY id DESC LIMIT $3"
+
+const selectContextAfterEventSQL = "" +
+	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" +
+	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
+	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
+	" AND ( $6::text[] IS NULL OR     type LIKE ANY($6)  )" +
+	" AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" +
+	" ORDER BY id ASC LIMIT $3"
+
 type outputRoomEventsStatements struct {
 	insertEventStmt               *sql.Stmt
 	selectEventsStmt              *sql.Stmt
@@ -140,6 +159,9 @@ type outputRoomEventsStatements struct {
 	selectStateInRangeStmt        *sql.Stmt
 	updateEventJSONStmt           *sql.Stmt
 	deleteEventsForRoomStmt       *sql.Stmt
+	selectContextEventStmt        *sql.Stmt
+	selectContextBeforeEventStmt  *sql.Stmt
+	selectContextAfterEventStmt   *sql.Stmt
 }
 
 func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
@@ -148,34 +170,20 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
 	if err != nil {
 		return nil, err
 	}
-	if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
+	return s, sqlutil.StatementList{
-		return nil, err
+		{&s.insertEventStmt, insertEventSQL},
-	}
+		{&s.selectEventsStmt, selectEventsSQL},
-	if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil {
+		{&s.selectMaxEventIDStmt, selectMaxEventIDSQL},
-		return nil, err
+		{&s.selectRecentEventsStmt, selectRecentEventsSQL},
-	}
+		{&s.selectRecentEventsForSyncStmt, selectRecentEventsForSyncSQL},
-	if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil {
+		{&s.selectEarlyEventsStmt, selectEarlyEventsSQL},
-		return nil, err
+		{&s.selectStateInRangeStmt, selectStateInRangeSQL},
-	}
+		{&s.updateEventJSONStmt, updateEventJSONSQL},
-	if s.selectRecentEventsStmt, err = db.Prepare(selectRecentEventsSQL); err != nil {
+		{&s.deleteEventsForRoomStmt, deleteEventsForRoomSQL},
-		return nil, err
+		{&s.selectContextEventStmt, selectContextEventSQL},
-	}
+		{&s.selectContextBeforeEventStmt, selectContextBeforeEventSQL},
-	if s.selectRecentEventsForSyncStmt, err = db.Prepare(selectRecentEventsForSyncSQL); err != nil {
+		{&s.selectContextAfterEventStmt, selectContextAfterEventSQL},
-		return nil, err
+	}.Prepare(db)
-	}
-	if s.selectEarlyEventsStmt, err = db.Prepare(selectEarlyEventsSQL); err != nil {
-		return nil, err
-	}
-	if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil {
-		return nil, err
-	}
-	if s.updateEventJSONStmt, err = db.Prepare(updateEventJSONSQL); err != nil {
-		return nil, err
-	}
-	if s.deleteEventsForRoomStmt, err = db.Prepare(deleteEventsForRoomSQL); err != nil {
-		return nil, err
-	}
-	return s, nil
 }
 
 func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
@@ -436,6 +444,84 @@ func (s *outputRoomEventsStatements) DeleteEventsForRoom(
 	return err
 }
 
+func (s *outputRoomEventsStatements) SelectContextEvent(ctx context.Context, txn *sql.Tx, roomID, eventID string) (id int, evt gomatrixserverlib.HeaderedEvent, err error) {
+	row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID)
+
+	var eventAsString string
+	if err = row.Scan(&id, &eventAsString); err != nil {
+		return 0, evt, err
+	}
+
+	if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil {
+		return 0, evt, err
+	}
+	return id, evt, nil
+}
+
+func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
+	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
+) (evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	rows, err := sqlutil.TxStmt(txn, s.selectContextBeforeEventStmt).QueryContext(
+		ctx, roomID, id, filter.Limit,
+		pq.StringArray(filter.Senders),
+		pq.StringArray(filter.NotSenders),
+		pq.StringArray(filterConvertTypeWildcardToSQL(filter.Types)),
+		pq.StringArray(filterConvertTypeWildcardToSQL(filter.NotTypes)),
+	)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			eventBytes []byte
+			evt        *gomatrixserverlib.HeaderedEvent
+		)
+		if err = rows.Scan(&eventBytes); err != nil {
+			return evts, err
+		}
+		if err = json.Unmarshal(eventBytes, &evt); err != nil {
+			return evts, err
+		}
+		evts = append(evts, evt)
+	}
+
+	return evts, rows.Err()
+}
+
+func (s *outputRoomEventsStatements) SelectContextAfterEvent(
+	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
+) (lastID int, evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	rows, err := sqlutil.TxStmt(txn, s.selectContextAfterEventStmt).QueryContext(
+		ctx, roomID, id, filter.Limit,
+		pq.StringArray(filter.Senders),
+		pq.StringArray(filter.NotSenders),
+		pq.StringArray(filterConvertTypeWildcardToSQL(filter.Types)),
+		pq.StringArray(filterConvertTypeWildcardToSQL(filter.NotTypes)),
+	)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			eventBytes []byte
+			evt        *gomatrixserverlib.HeaderedEvent
+		)
+		if err = rows.Scan(&lastID, &eventBytes); err != nil {
+			return 0, evts, err
+		}
+		if err = json.Unmarshal(eventBytes, &evt); err != nil {
+			return 0, evts, err
+		}
+		evts = append(evts, evt)
+	}
+
+	return lastID, evts, rows.Err()
+}
+
 func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 	var result []types.StreamEvent
 	for rows.Next() {

syncapi/storage/shared/syncserver.go

@@ -976,3 +976,14 @@ func (d *Database) UpsertRoomUnreadNotificationCounts(ctx context.Context, userI
 func (d *Database) GetUserUnreadNotificationCounts(ctx context.Context, userID string, from, to types.StreamPosition) (map[string]*eventutil.NotificationData, error) {
 	return d.NotificationData.SelectUserUnreadCounts(ctx, userID, from, to)
 }
+
+func (s *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) {
+	return s.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID)
+}
+
+func (s *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	return s.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter)
+}
+func (s *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) {
+	return s.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter)
+}

syncapi/storage/sqlite3/current_room_state_table.go

@@ -68,6 +68,7 @@ const selectRoomIDsWithMembershipSQL = "" +
 
 const selectCurrentStateSQL = "" +
 	"SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1"
+
 // WHEN, ORDER BY and LIMIT will be added by prepareWithFilter
 
 const selectJoinedUsersSQL = "" +

syncapi/storage/sqlite3/output_room_events_table.go

@@ -90,6 +90,17 @@ const selectStateInRangeSQL = "" +
 const deleteEventsForRoomSQL = "" +
 	"DELETE FROM syncapi_output_room_events WHERE room_id = $1"
 
+const selectContextEventSQL = "" +
+	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2"
+
+const selectContextBeforeEventSQL = "" +
+	"SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2"
+// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
+
+const selectContextAfterEventSQL = "" +
+	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2"
+// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
+
 type outputRoomEventsStatements struct {
 	db                           *sql.DB
 	streamIDStatements           *streamIDStatements
@@ -98,6 +109,9 @@ type outputRoomEventsStatements struct {
 	selectMaxEventIDStmt         *sql.Stmt
 	updateEventJSONStmt          *sql.Stmt
 	deleteEventsForRoomStmt      *sql.Stmt
+	selectContextEventStmt       *sql.Stmt
+	selectContextBeforeEventStmt *sql.Stmt
+	selectContextAfterEventStmt  *sql.Stmt
 }
 
 func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Events, error) {
@@ -109,22 +123,16 @@ func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Even
 	if err != nil {
 		return nil, err
 	}
-	if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
+	return s, sqlutil.StatementList{
-		return nil, err
+		{&s.insertEventStmt, insertEventSQL},
-	}
+		{&s.selectEventsStmt, selectEventsSQL},
-	if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil {
+		{&s.selectMaxEventIDStmt, selectMaxEventIDSQL},
-		return nil, err
+		{&s.updateEventJSONStmt, updateEventJSONSQL},
-	}
+		{&s.deleteEventsForRoomStmt, deleteEventsForRoomSQL},
-	if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil {
+		{&s.selectContextEventStmt, selectContextEventSQL},
-		return nil, err
+		{&s.selectContextBeforeEventStmt, selectContextBeforeEventSQL},
-	}
+		{&s.selectContextAfterEventStmt, selectContextAfterEventSQL},
-	if s.updateEventJSONStmt, err = db.Prepare(updateEventJSONSQL); err != nil {
+	}.Prepare(db)
-		return nil, err
-	}
-	if s.deleteEventsForRoomStmt, err = db.Prepare(deleteEventsForRoomSQL); err != nil {
-		return nil, err
-	}
-	return s, nil
 }
 
 func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
@@ -462,6 +470,91 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 	}
 	return result, nil
 }
+func (s *outputRoomEventsStatements) SelectContextEvent(
+	ctx context.Context, txn *sql.Tx, roomID, eventID string,
+) (id int, evt gomatrixserverlib.HeaderedEvent, err error) {
+	row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID)
+	var eventAsString string
+	if err = row.Scan(&id, &eventAsString); err != nil {
+		return 0, evt, err
+	}
+
+	if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil {
+		return 0, evt, err
+	}
+	return id, evt, nil
+}
+
+func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
+	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
+) (evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	stmt, params, err := prepareWithFilters(
+		s.db, txn, selectContextBeforeEventSQL,
+		[]interface{}{
+			roomID, id,
+		},
+		filter.Senders, filter.NotSenders,
+		filter.Types, filter.NotTypes,
+		nil, filter.Limit, FilterOrderDesc,
+	)
+
+	rows, err := stmt.QueryContext(ctx, params...)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			eventBytes []byte
+			evt        *gomatrixserverlib.HeaderedEvent
+		)
+		if err = rows.Scan(&eventBytes); err != nil {
+			return evts, err
+		}
+		if err = json.Unmarshal(eventBytes, &evt); err != nil {
+			return evts, err
+		}
+		evts = append(evts, evt)
+	}
+
+	return evts, rows.Err()
+}
+
+func (s *outputRoomEventsStatements) SelectContextAfterEvent(
+	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
+) (lastID int, evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	stmt, params, err := prepareWithFilters(
+		s.db, txn, selectContextAfterEventSQL,
+		[]interface{}{
+			roomID, id,
+		},
+		filter.Senders, filter.NotSenders,
+		filter.Types, filter.NotTypes,
+		nil, filter.Limit, FilterOrderAsc,
+	)
+
+	rows, err := stmt.QueryContext(ctx, params...)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			eventBytes []byte
+			evt        *gomatrixserverlib.HeaderedEvent
+		)
+		if err = rows.Scan(&lastID, &eventBytes); err != nil {
+			return 0, evts, err
+		}
+		if err = json.Unmarshal(eventBytes, &evt); err != nil {
+			return 0, evts, err
+		}
+		evts = append(evts, evt)
+	}
+	return lastID, evts, rows.Err()
+}
 
 func unmarshalStateIDs(addIDsJSON, delIDsJSON string) (addIDs []string, delIDs []string, err error) {
 	if len(addIDsJSON) > 0 {

syncapi/storage/tables/interface.go

@@ -64,6 +64,10 @@ type Events interface {
 	UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error
 	// DeleteEventsForRoom removes all event information for a room. This should only be done when removing the room entirely.
 	DeleteEventsForRoom(ctx context.Context, txn *sql.Tx, roomID string) (err error)
+
+	SelectContextEvent(ctx context.Context, txn *sql.Tx, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error)
+	SelectContextBeforeEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error)
+	SelectContextAfterEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error)
 }
 
 // Topology keeps track of the depths and stream positions for all events.

sytest-blacklist

@@ -24,11 +24,8 @@ Local device key changes get to remote servers with correct prev_id
 
 # Flakey
 Local device key changes appear in /keys/changes
-Device list doesn't change if remote server is down
 
 # we don't support groups
 Remove group category
 Remove group role
 
-# See https://github.com/matrix-org/sytest/pull/1142
-Device list doesn't change if remote server is down

sytest-whitelist

@@ -648,3 +648,11 @@ If a device list update goes missing, the server resyncs on the next one
 uploading self-signing key notifies over federation
 uploading signed devices gets propagated over federation
 Device list doesn't change if remote server is down
+/context/ on joined room works
+/context/ on non world readable room does not work
+/context/ returns correct number of events
+/context/ with lazy_load_members filter works
+Can query remote device keys using POST after notification
+Device deletion propagates over federation
+Get left notifs in sync and /keys/changes when other user leaves
+Remote banned user is kicked and may not rejoin until unbanned