From a02dd7721d8555391597444d185d402f94b626ae Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 21 Feb 2022 15:25:54 +0000
Subject: [PATCH 01/12] Reset invalid state snapshots for events during state
 storage refactor migration (#2209)

This should help with #2204. We can't do this for rooms, only events.
---
 .../2021041615092700_state_blocks_refactor.go | 26 +++++++------------
 .../2021041615092700_state_blocks_refactor.go | 12 ++++++++-
 2 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go
index 06740dc8b..06442a4c3 100644
--- a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go
+++ b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go
@@ -256,23 +256,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
-		var debugEventID, debugRoomID string
-		var debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth int64
-		err = tx.QueryRow(
-			`SELECT event_id, event_type_nid, event_state_key_nid, roomserver_events.state_snapshot_nid, depth, room_id FROM roomserver_events
-				JOIN roomserver_rooms ON roomserver_rooms.room_nid = roomserver_events.room_nid WHERE roomserver_events.state_snapshot_nid < $1 AND roomserver_events.state_snapshot_nid != 0`, maxsnapshotid,
-		).Scan(&debugEventID, &debugEventTypeNID, &debugStateKeyNID, &debugSnapNID, &debugDepth, &debugRoomID)
-		if err != nil {
-			logrus.Errorf("cannot extract debug info: %v", err)
-		} else {
-			logrus.Errorf(
-				"Affected row: event_id=%v room_id=%v type=%v state_key=%v snapshot=%v depth=%v",
-				debugEventID, debugRoomID, debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth,
-			)
-			logrus.Errorf("To fix this manually, run this query first then retry the migration: "+
-				"UPDATE roomserver_events SET state_snapshot_nid=0 WHERE event_id='%v'", debugEventID)
+		var res sql.Result
+		var c int64
+		res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid)
+		if err != nil && err != sql.ErrNoRows {
+			return fmt.Errorf("failed to reset invalid state snapshots: %w", err)
+		}
+		if c, err = res.RowsAffected(); err != nil {
+			return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err)
+		} else if c != count {
+			return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c)
 		}
-		return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
 	}
 	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)
diff --git a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go
index 8d0331748..8f5ab8fc5 100644
--- a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go
+++ b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go
@@ -179,7 +179,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
-		return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
+		var res sql.Result
+		var c int64
+		res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID)
+		if err != nil && err != sql.ErrNoRows {
+			return fmt.Errorf("failed to reset invalid state snapshots: %w", err)
+		}
+		if c, err = res.RowsAffected(); err != nil {
+			return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err)
+		} else if c != count {
+			return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c)
+		}
 	}
 	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)

From 280e9b19a195e3ce19f0fa5bc0e94bb09e397a23 Mon Sep 17 00:00:00 2001
From: Benjamin Nater <me@bn4t.me>
Date: Mon, 21 Feb 2022 16:36:03 +0100
Subject: [PATCH 02/12] Don't ignore config flag for yggdrasil demo if set
 (#2070)

* Don't ignore config flag for yggdrasil demo if set

Signed-off-by: bn4t <me@bn4t.me>

* run goimports

Signed-off-by: bn4t <me@bn4t.me>

* always override ServerName, PrivateKey and KeyID

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 cmd/dendrite-demo-yggdrasil/main.go | 58 +++++++++++++++++------------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 49e096bd1..d16f0e9e5 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -21,8 +21,11 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
@@ -42,8 +45,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/gomatrixserverlib"
-
 	"github.com/sirupsen/logrus"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -63,33 +64,42 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	/*
-		ygg.SetMulticastEnabled(true)
-		if instancePeer != nil && *instancePeer != "" {
-			if err = ygg.SetStaticPeer(*instancePeer); err != nil {
-				logrus.WithError(err).Error("Failed to set static peer")
-			}
+
+	// iterate through the cli args and check if the config flag was set
+	configFlagSet := false
+	for _, arg := range os.Args {
+		if arg == "--config" || arg == "-config" {
+			configFlagSet = true
+			break
 		}
-	*/
+	}
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+
+	// use custom config if config flag is set
+	if configFlagSet {
+		cfg = setup.ParseFlags(true)
+	} else {
+		cfg.Defaults(true)
+		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
+		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
+		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
+		cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+		cfg.MSCs.MSCs = []string{"msc2836"}
+		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
+		if err = cfg.Derive(); err != nil {
+			panic(err)
+		}
+	}
+
+	// always override ServerName, PrivateKey and KeyID
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
 	cfg.Global.PrivateKey = ygg.PrivateKey()
-	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
-	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.MSCs.MSCs = []string{"msc2836"}
-	cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
-	if err = cfg.Derive(); err != nil {
-		panic(err)
-	}
+	cfg.Global.KeyID = signing.KeyID
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck

From cf525d1f619cc65df244c20ec0f220ace22ae2bd Mon Sep 17 00:00:00 2001
From: S7evinK <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 21 Feb 2022 17:12:22 +0100
Subject: [PATCH 03/12] Implement `/context` (#2207)

* Add QueryEventsAfter

* Add /context

* Make all tests pass on sqlite

* Add queries to get the events for /context requests

* Move /context to the syncapi

* Revert "Add QueryEventsAfter"

This reverts commit 440a771d10632622e8c65d35fe90f0804bc98862.

* Simplify getting the required events

* Apply RoomEventFilter when getting events

* Add passing tests

* Remove logging

* Remove unused SQL statements
Update comments & add TODO
---
 syncapi/routing/context.go                    | 190 ++++++++++++++++++
 syncapi/routing/context_test.go               |  68 +++++++
 syncapi/routing/routing.go                    |  15 ++
 syncapi/storage/interface.go                  |   4 +
 .../postgres/output_room_events_table.go      | 142 ++++++++++---
 syncapi/storage/shared/syncserver.go          |  11 +
 .../sqlite3/current_room_state_table.go       |   3 +-
 .../sqlite3/output_room_events_table.go       | 147 +++++++++++---
 syncapi/storage/tables/interface.go           |   4 +
 sytest-blacklist                              |   3 -
 sytest-whitelist                              |   6 +-
 11 files changed, 533 insertions(+), 60 deletions(-)
 create mode 100644 syncapi/routing/context.go
 create mode 100644 syncapi/routing/context_test.go

diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
new file mode 100644
index 000000000..709f62916
--- /dev/null
+++ b/syncapi/routing/context.go
@@ -0,0 +1,190 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"database/sql"
+	"encoding/json"
+	"net/http"
+	"strconv"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	roomserver "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+)
+
+type ContextRespsonse struct {
+	End          string                          `json:"end"`
+	Event        gomatrixserverlib.ClientEvent   `json:"event"`
+	EventsAfter  []gomatrixserverlib.ClientEvent `json:"events_after,omitempty"`
+	EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before,omitempty"`
+	Start        string                          `json:"start"`
+	State        []gomatrixserverlib.ClientEvent `json:"state"`
+}
+
+func Context(
+	req *http.Request, device *userapi.Device,
+	rsAPI roomserver.RoomserverInternalAPI,
+	syncDB storage.Database,
+	roomID, eventID string,
+) util.JSONResponse {
+	filter, err := parseContextParams(req)
+	if err != nil {
+		errMsg := ""
+		switch err.(type) {
+		case *json.InvalidUnmarshalError:
+			errMsg = "unable to parse filter"
+		case *strconv.NumError:
+			errMsg = "unable to parse limit"
+		}
+		return util.JSONResponse{
+			Code:    http.StatusBadRequest,
+			JSON:    jsonerror.InvalidParam(errMsg),
+			Headers: nil,
+		}
+	}
+	filter.Rooms = append(filter.Rooms, roomID)
+
+	ctx := req.Context()
+	membershipRes := roomserver.QueryMembershipForUserResponse{}
+	membershipReq := roomserver.QueryMembershipForUserRequest{UserID: device.UserID, RoomID: roomID}
+	if err := rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes); err != nil {
+		logrus.WithError(err).Error("unable to fo membership")
+		return jsonerror.InternalServerError()
+	}
+
+	stateFilter := gomatrixserverlib.StateFilter{
+		Limit:                   100,
+		NotSenders:              filter.NotSenders,
+		NotTypes:                filter.NotTypes,
+		Senders:                 filter.Senders,
+		Types:                   filter.Types,
+		LazyLoadMembers:         filter.LazyLoadMembers,
+		IncludeRedundantMembers: filter.IncludeRedundantMembers,
+		NotRooms:                filter.NotRooms,
+		Rooms:                   filter.Rooms,
+		ContainsURL:             filter.ContainsURL,
+	}
+
+	// TODO: Get the actual state at the last event returned by SelectContextAfterEvent
+	state, _ := syncDB.CurrentState(ctx, roomID, &stateFilter, nil)
+	// verify the user is allowed to see the context for this room/event
+	for _, x := range state {
+		hisVis, err := x.HistoryVisibility()
+		if err != nil {
+			continue
+		}
+		allowed := hisVis != "world_readable" && membershipRes.Membership == "join"
+		if !allowed {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden("User is not allowed to query context"),
+			}
+		}
+	}
+
+	id, requestedEvent, err := syncDB.SelectContextEvent(ctx, roomID, eventID)
+	if err != nil {
+		logrus.WithError(err).WithField("eventID", eventID).Error("unable to find requested event")
+		return jsonerror.InternalServerError()
+	}
+
+	eventsBefore, err := syncDB.SelectContextBeforeEvent(ctx, id, roomID, filter)
+	if err != nil && err != sql.ErrNoRows {
+		logrus.WithError(err).Error("unable to fetch before events")
+		return jsonerror.InternalServerError()
+	}
+
+	_, eventsAfter, err := syncDB.SelectContextAfterEvent(ctx, id, roomID, filter)
+	if err != nil && err != sql.ErrNoRows {
+		logrus.WithError(err).Error("unable to fetch after events")
+		return jsonerror.InternalServerError()
+	}
+
+	eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatAll)
+	eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatAll)
+	newState := applyLazyLoadMembers(filter, eventsAfterClient, eventsBeforeClient, state)
+
+	response := ContextRespsonse{
+		Event:        gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll),
+		EventsAfter:  eventsAfterClient,
+		EventsBefore: eventsBeforeClient,
+		State:        gomatrixserverlib.HeaderedToClientEvents(newState, gomatrixserverlib.FormatAll),
+	}
+
+	if len(response.State) > filter.Limit {
+		response.State = response.State[len(response.State)-filter.Limit:]
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: response,
+	}
+}
+
+func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent, state []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
+	if filter == nil || !filter.LazyLoadMembers {
+		return state
+	}
+	allEvents := append(eventsBefore, eventsAfter...)
+	x := make(map[string]bool)
+	// get members who actually send an event
+	for _, e := range allEvents {
+		x[e.Sender] = true
+	}
+
+	newState := []*gomatrixserverlib.HeaderedEvent{}
+	for _, event := range state {
+		if event.Type() != gomatrixserverlib.MRoomMember {
+			newState = append(newState, event)
+		} else {
+			// did the user send an event?
+			if x[event.Sender()] {
+				newState = append(newState, event)
+			}
+		}
+	}
+	return newState
+}
+
+func parseContextParams(req *http.Request) (*gomatrixserverlib.RoomEventFilter, error) {
+	// Default room filter
+	filter := &gomatrixserverlib.RoomEventFilter{Limit: 10}
+
+	l := req.URL.Query().Get("limit")
+	f := req.URL.Query().Get("filter")
+	if l != "" {
+		limit, err := strconv.Atoi(l)
+		if err != nil {
+			return nil, err
+		}
+		// NOTSPEC: feels like a good idea to have an upper bound limit
+		if limit > 100 {
+			limit = 100
+		}
+		filter.Limit = limit
+	}
+	if f != "" {
+		if err := json.Unmarshal([]byte(f), &filter); err != nil {
+			return nil, err
+		}
+	}
+
+	return filter, nil
+}
diff --git a/syncapi/routing/context_test.go b/syncapi/routing/context_test.go
new file mode 100644
index 000000000..1b430d83a
--- /dev/null
+++ b/syncapi/routing/context_test.go
@@ -0,0 +1,68 @@
+package routing
+
+import (
+	"net/http"
+	"reflect"
+	"testing"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func Test_parseContextParams(t *testing.T) {
+
+	noParamsReq, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI", nil)
+	limit2Req, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI&limit=2", nil)
+	limit10000Req, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI&limit=10000", nil)
+	invalidLimitReq, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI&limit=100as", nil)
+	lazyLoadReq, _ := http.NewRequest("GET", "https://localhost:8800//_matrix/client/r0/rooms/!kvEtX3rFamfwKHO3:localhost:8800/context/%24GjmkRbajRHy8_cxcSbUU4qF_njV8yHeLphI2azTrPaI?limit=2&filter=%7B+%22lazy_load_members%22+%3A+true+%7D&access_token=t1Njzm74w3G40CJ5xrlf1V2haXom0z0Iq1qyyVWhbVo", nil)
+	invalidFilterReq, _ := http.NewRequest("GET", "https://localhost:8800//_matrix/client/r0/rooms/!kvEtX3rFamfwKHO3:localhost:8800/context/%24GjmkRbajRHy8_cxcSbUU4qF_njV8yHeLphI2azTrPaI?limit=2&filter=%7B+%22lazy_load_members%22+%3A+true&access_token=t1Njzm74w3G40CJ5xrlf1V2haXom0z0Iq1qyyVWhbVo", nil)
+	tests := []struct {
+		name       string
+		req        *http.Request
+		wantFilter *gomatrixserverlib.RoomEventFilter
+		wantErr    bool
+	}{
+		{
+			name:       "no params set",
+			req:        noParamsReq,
+			wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 10},
+		},
+		{
+			name:       "limit 2 param set",
+			req:        limit2Req,
+			wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 2},
+		},
+		{
+			name:       "limit 10000 param set",
+			req:        limit10000Req,
+			wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 100},
+		},
+		{
+			name:       "filter lazy_load_members param set",
+			req:        lazyLoadReq,
+			wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 2, LazyLoadMembers: true},
+		},
+		{
+			name:    "invalid limit req",
+			req:     invalidLimitReq,
+			wantErr: true,
+		},
+		{
+			name:    "invalid filter req",
+			req:     invalidFilterReq,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotFilter, err := parseContextParams(tt.req)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseContextParams() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotFilter, tt.wantFilter) {
+				t.Errorf("parseContextParams() gotFilter = %v, want %v", gotFilter, tt.wantFilter)
+			}
+		})
+	}
+}
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index 005a33555..be366ba10 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -77,4 +77,19 @@ func Setup(
 	v3mux.Handle("/keys/changes", httputil.MakeAuthAPI("keys_changes", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		return srp.OnIncomingKeyChangeRequest(req, device)
 	})).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomId}/context/{eventId}",
+		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+
+			return Context(
+				req, device,
+				rsAPI, syncDB,
+				vars["roomId"], vars["eventId"],
+			)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
 }
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index b464ad9cd..126bc8658 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -137,4 +137,8 @@ type Database interface {
 	StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error)
 	// GetRoomReceipts gets all receipts for a given roomID
 	GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]eduAPI.OutputReceiptEvent, error)
+
+	SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error)
+	SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error)
+	SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error)
 }
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index 44de02c92..d4cc4f3fb 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -130,6 +130,25 @@ const selectStateInRangeSQL = "" +
 const deleteEventsForRoomSQL = "" +
 	"DELETE FROM syncapi_output_room_events WHERE room_id = $1"
 
+const selectContextEventSQL = "" +
+	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2"
+
+const selectContextBeforeEventSQL = "" +
+	"SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" +
+	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
+	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
+	" AND ( $6::text[] IS NULL OR     type LIKE ANY($6)  )" +
+	" AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" +
+	" ORDER BY id DESC LIMIT $3"
+
+const selectContextAfterEventSQL = "" +
+	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" +
+	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
+	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
+	" AND ( $6::text[] IS NULL OR     type LIKE ANY($6)  )" +
+	" AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" +
+	" ORDER BY id ASC LIMIT $3"
+
 type outputRoomEventsStatements struct {
 	insertEventStmt               *sql.Stmt
 	selectEventsStmt              *sql.Stmt
@@ -140,6 +159,9 @@ type outputRoomEventsStatements struct {
 	selectStateInRangeStmt        *sql.Stmt
 	updateEventJSONStmt           *sql.Stmt
 	deleteEventsForRoomStmt       *sql.Stmt
+	selectContextEventStmt        *sql.Stmt
+	selectContextBeforeEventStmt  *sql.Stmt
+	selectContextAfterEventStmt   *sql.Stmt
 }
 
 func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
@@ -148,34 +170,20 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
 	if err != nil {
 		return nil, err
 	}
-	if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
-		return nil, err
-	}
-	if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil {
-		return nil, err
-	}
-	if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil {
-		return nil, err
-	}
-	if s.selectRecentEventsStmt, err = db.Prepare(selectRecentEventsSQL); err != nil {
-		return nil, err
-	}
-	if s.selectRecentEventsForSyncStmt, err = db.Prepare(selectRecentEventsForSyncSQL); err != nil {
-		return nil, err
-	}
-	if s.selectEarlyEventsStmt, err = db.Prepare(selectEarlyEventsSQL); err != nil {
-		return nil, err
-	}
-	if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil {
-		return nil, err
-	}
-	if s.updateEventJSONStmt, err = db.Prepare(updateEventJSONSQL); err != nil {
-		return nil, err
-	}
-	if s.deleteEventsForRoomStmt, err = db.Prepare(deleteEventsForRoomSQL); err != nil {
-		return nil, err
-	}
-	return s, nil
+	return s, sqlutil.StatementList{
+		{&s.insertEventStmt, insertEventSQL},
+		{&s.selectEventsStmt, selectEventsSQL},
+		{&s.selectMaxEventIDStmt, selectMaxEventIDSQL},
+		{&s.selectRecentEventsStmt, selectRecentEventsSQL},
+		{&s.selectRecentEventsForSyncStmt, selectRecentEventsForSyncSQL},
+		{&s.selectEarlyEventsStmt, selectEarlyEventsSQL},
+		{&s.selectStateInRangeStmt, selectStateInRangeSQL},
+		{&s.updateEventJSONStmt, updateEventJSONSQL},
+		{&s.deleteEventsForRoomStmt, deleteEventsForRoomSQL},
+		{&s.selectContextEventStmt, selectContextEventSQL},
+		{&s.selectContextBeforeEventStmt, selectContextBeforeEventSQL},
+		{&s.selectContextAfterEventStmt, selectContextAfterEventSQL},
+	}.Prepare(db)
 }
 
 func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
@@ -436,6 +444,84 @@ func (s *outputRoomEventsStatements) DeleteEventsForRoom(
 	return err
 }
 
+func (s *outputRoomEventsStatements) SelectContextEvent(ctx context.Context, txn *sql.Tx, roomID, eventID string) (id int, evt gomatrixserverlib.HeaderedEvent, err error) {
+	row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID)
+
+	var eventAsString string
+	if err = row.Scan(&id, &eventAsString); err != nil {
+		return 0, evt, err
+	}
+
+	if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil {
+		return 0, evt, err
+	}
+	return id, evt, nil
+}
+
+func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
+	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
+) (evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	rows, err := sqlutil.TxStmt(txn, s.selectContextBeforeEventStmt).QueryContext(
+		ctx, roomID, id, filter.Limit,
+		pq.StringArray(filter.Senders),
+		pq.StringArray(filter.NotSenders),
+		pq.StringArray(filterConvertTypeWildcardToSQL(filter.Types)),
+		pq.StringArray(filterConvertTypeWildcardToSQL(filter.NotTypes)),
+	)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			eventBytes []byte
+			evt        *gomatrixserverlib.HeaderedEvent
+		)
+		if err = rows.Scan(&eventBytes); err != nil {
+			return evts, err
+		}
+		if err = json.Unmarshal(eventBytes, &evt); err != nil {
+			return evts, err
+		}
+		evts = append(evts, evt)
+	}
+
+	return evts, rows.Err()
+}
+
+func (s *outputRoomEventsStatements) SelectContextAfterEvent(
+	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
+) (lastID int, evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	rows, err := sqlutil.TxStmt(txn, s.selectContextAfterEventStmt).QueryContext(
+		ctx, roomID, id, filter.Limit,
+		pq.StringArray(filter.Senders),
+		pq.StringArray(filter.NotSenders),
+		pq.StringArray(filterConvertTypeWildcardToSQL(filter.Types)),
+		pq.StringArray(filterConvertTypeWildcardToSQL(filter.NotTypes)),
+	)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			eventBytes []byte
+			evt        *gomatrixserverlib.HeaderedEvent
+		)
+		if err = rows.Scan(&lastID, &eventBytes); err != nil {
+			return 0, evts, err
+		}
+		if err = json.Unmarshal(eventBytes, &evt); err != nil {
+			return 0, evts, err
+		}
+		evts = append(evts, evt)
+	}
+
+	return lastID, evts, rows.Err()
+}
+
 func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 	var result []types.StreamEvent
 	for rows.Next() {
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index e6c681832..819851b33 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -955,3 +955,14 @@ func (d *Database) GetRoomReceipts(ctx context.Context, roomIDs []string, stream
 	_, receipts, err := d.Receipts.SelectRoomReceiptsAfter(ctx, roomIDs, streamPos)
 	return receipts, err
 }
+
+func (s *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) {
+	return s.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID)
+}
+
+func (s *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	return s.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter)
+}
+func (s *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) {
+	return s.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter)
+}
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index 4fbbf45cf..c91ca6923 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -68,7 +68,8 @@ const selectRoomIDsWithMembershipSQL = "" +
 
 const selectCurrentStateSQL = "" +
 	"SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1"
-	// WHEN, ORDER BY and LIMIT will be added by prepareWithFilter
+
+// WHEN, ORDER BY and LIMIT will be added by prepareWithFilter
 
 const selectJoinedUsersSQL = "" +
 	"SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'"
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index afdbe55ce..581ee6928 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -62,17 +62,17 @@ const selectEventsSQL = "" +
 const selectRecentEventsSQL = "" +
 	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3"
-	// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
+// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
 
 const selectRecentEventsForSyncSQL = "" +
 	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE"
-	// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
+// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
 
 const selectEarlyEventsSQL = "" +
 	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3"
-	// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
+// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
 
 const selectMaxEventIDSQL = "" +
 	"SELECT MAX(id) FROM syncapi_output_room_events"
@@ -85,19 +85,33 @@ const selectStateInRangeSQL = "" +
 	" FROM syncapi_output_room_events" +
 	" WHERE (id > $1 AND id <= $2)" +
 	" AND ((add_state_ids IS NOT NULL AND add_state_ids != '') OR (remove_state_ids IS NOT NULL AND remove_state_ids != ''))"
-	// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
+// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
 
 const deleteEventsForRoomSQL = "" +
 	"DELETE FROM syncapi_output_room_events WHERE room_id = $1"
 
+const selectContextEventSQL = "" +
+	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2"
+
+const selectContextBeforeEventSQL = "" +
+	"SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2"
+// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
+
+const selectContextAfterEventSQL = "" +
+	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2"
+// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
+
 type outputRoomEventsStatements struct {
-	db                      *sql.DB
-	streamIDStatements      *streamIDStatements
-	insertEventStmt         *sql.Stmt
-	selectEventsStmt        *sql.Stmt
-	selectMaxEventIDStmt    *sql.Stmt
-	updateEventJSONStmt     *sql.Stmt
-	deleteEventsForRoomStmt *sql.Stmt
+	db                           *sql.DB
+	streamIDStatements           *streamIDStatements
+	insertEventStmt              *sql.Stmt
+	selectEventsStmt             *sql.Stmt
+	selectMaxEventIDStmt         *sql.Stmt
+	updateEventJSONStmt          *sql.Stmt
+	deleteEventsForRoomStmt      *sql.Stmt
+	selectContextEventStmt       *sql.Stmt
+	selectContextBeforeEventStmt *sql.Stmt
+	selectContextAfterEventStmt  *sql.Stmt
 }
 
 func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Events, error) {
@@ -109,22 +123,16 @@ func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Even
 	if err != nil {
 		return nil, err
 	}
-	if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
-		return nil, err
-	}
-	if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil {
-		return nil, err
-	}
-	if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil {
-		return nil, err
-	}
-	if s.updateEventJSONStmt, err = db.Prepare(updateEventJSONSQL); err != nil {
-		return nil, err
-	}
-	if s.deleteEventsForRoomStmt, err = db.Prepare(deleteEventsForRoomSQL); err != nil {
-		return nil, err
-	}
-	return s, nil
+	return s, sqlutil.StatementList{
+		{&s.insertEventStmt, insertEventSQL},
+		{&s.selectEventsStmt, selectEventsSQL},
+		{&s.selectMaxEventIDStmt, selectMaxEventIDSQL},
+		{&s.updateEventJSONStmt, updateEventJSONSQL},
+		{&s.deleteEventsForRoomStmt, deleteEventsForRoomSQL},
+		{&s.selectContextEventStmt, selectContextEventSQL},
+		{&s.selectContextBeforeEventStmt, selectContextBeforeEventSQL},
+		{&s.selectContextAfterEventStmt, selectContextAfterEventSQL},
+	}.Prepare(db)
 }
 
 func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
@@ -462,6 +470,91 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 	}
 	return result, nil
 }
+func (s *outputRoomEventsStatements) SelectContextEvent(
+	ctx context.Context, txn *sql.Tx, roomID, eventID string,
+) (id int, evt gomatrixserverlib.HeaderedEvent, err error) {
+	row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID)
+	var eventAsString string
+	if err = row.Scan(&id, &eventAsString); err != nil {
+		return 0, evt, err
+	}
+
+	if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil {
+		return 0, evt, err
+	}
+	return id, evt, nil
+}
+
+func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
+	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
+) (evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	stmt, params, err := prepareWithFilters(
+		s.db, txn, selectContextBeforeEventSQL,
+		[]interface{}{
+			roomID, id,
+		},
+		filter.Senders, filter.NotSenders,
+		filter.Types, filter.NotTypes,
+		nil, filter.Limit, FilterOrderDesc,
+	)
+
+	rows, err := stmt.QueryContext(ctx, params...)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			eventBytes []byte
+			evt        *gomatrixserverlib.HeaderedEvent
+		)
+		if err = rows.Scan(&eventBytes); err != nil {
+			return evts, err
+		}
+		if err = json.Unmarshal(eventBytes, &evt); err != nil {
+			return evts, err
+		}
+		evts = append(evts, evt)
+	}
+
+	return evts, rows.Err()
+}
+
+func (s *outputRoomEventsStatements) SelectContextAfterEvent(
+	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
+) (lastID int, evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	stmt, params, err := prepareWithFilters(
+		s.db, txn, selectContextAfterEventSQL,
+		[]interface{}{
+			roomID, id,
+		},
+		filter.Senders, filter.NotSenders,
+		filter.Types, filter.NotTypes,
+		nil, filter.Limit, FilterOrderAsc,
+	)
+
+	rows, err := stmt.QueryContext(ctx, params...)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			eventBytes []byte
+			evt        *gomatrixserverlib.HeaderedEvent
+		)
+		if err = rows.Scan(&lastID, &eventBytes); err != nil {
+			return 0, evts, err
+		}
+		if err = json.Unmarshal(eventBytes, &evt); err != nil {
+			return 0, evts, err
+		}
+		evts = append(evts, evt)
+	}
+	return lastID, evts, rows.Err()
+}
 
 func unmarshalStateIDs(addIDsJSON, delIDsJSON string) (addIDs []string, delIDs []string, err error) {
 	if len(addIDsJSON) > 0 {
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 028872716..1d807ee6b 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -63,6 +63,10 @@ type Events interface {
 	UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error
 	// DeleteEventsForRoom removes all event information for a room. This should only be done when removing the room entirely.
 	DeleteEventsForRoom(ctx context.Context, txn *sql.Tx, roomID string) (err error)
+
+	SelectContextEvent(ctx context.Context, txn *sql.Tx, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error)
+	SelectContextBeforeEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error)
+	SelectContextAfterEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error)
 }
 
 // Topology keeps track of the depths and stream positions for all events.
diff --git a/sytest-blacklist b/sytest-blacklist
index 3e08f0cb4..16abce8da 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -24,11 +24,8 @@ Local device key changes get to remote servers with correct prev_id
 
 # Flakey
 Local device key changes appear in /keys/changes
-Device list doesn't change if remote server is down
 
 # we don't support groups
 Remove group category
 Remove group role
 
-# See https://github.com/matrix-org/sytest/pull/1142
-Device list doesn't change if remote server is down
diff --git a/sytest-whitelist b/sytest-whitelist
index d739313ac..187a0f475 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -592,4 +592,8 @@ Forward extremities remain so even after the next events are populated as outlie
 If a device list update goes missing, the server resyncs on the next one
 uploading self-signing key notifies over federation
 uploading signed devices gets propagated over federation
-Device list doesn't change if remote server is down
\ No newline at end of file
+Device list doesn't change if remote server is down
+/context/ on joined room works
+/context/ on non world readable room does not work
+/context/ returns correct number of events
+/context/ with lazy_load_members filter works
\ No newline at end of file

From aa6bbf484a7963a754d47ba04869c06b72d4ee49 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 21 Feb 2022 16:22:29 +0000
Subject: [PATCH 04/12] Return `ErrRoomNoExists` if insufficient state is
 available for a `buildEvent` to succeed when joining a room (#2210)

This may help cases like #2206, since it should prompt us to try a federated join again instead.
---
 roomserver/internal/perform/perform_join.go         | 11 ++++++++++-
 roomserver/storage/postgres/state_snapshot_table.go |  2 +-
 roomserver/storage/sqlite3/state_snapshot_table.go  |  2 +-
 roomserver/types/types.go                           |  6 ++++++
 4 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index 9d2a66d4c..a40f66d21 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -29,6 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
 	"github.com/matrix-org/dendrite/roomserver/internal/query"
 	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
@@ -367,7 +368,15 @@ func buildEvent(
 		StateToFetch: eventsNeeded.Tuples(),
 	}, &queryRes)
 	if err != nil {
-		return nil, nil, fmt.Errorf("QueryLatestEventsAndState: %w", err)
+		switch err.(type) {
+		case types.MissingStateError:
+			// We know something about the room but the state seems to be
+			// insufficient to actually build a new event, so in effect we
+			// had might as well treat the room as if it doesn't exist.
+			return nil, nil, eventutil.ErrRoomNoExists
+		default:
+			return nil, nil, fmt.Errorf("QueryLatestEventsAndState: %w", err)
+		}
 	}
 
 	ev, err := eventutil.BuildEvent(ctx, builder, cfg, time.Now(), &eventsNeeded, &queryRes)
diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go
index ce9f24636..8ed886030 100644
--- a/roomserver/storage/postgres/state_snapshot_table.go
+++ b/roomserver/storage/postgres/state_snapshot_table.go
@@ -134,7 +134,7 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 		return nil, err
 	}
 	if i != len(stateNIDs) {
-		return nil, fmt.Errorf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))
+		return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs)))
 	}
 	return results, nil
 }
diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go
index 3c4bde3f5..01df31e90 100644
--- a/roomserver/storage/sqlite3/state_snapshot_table.go
+++ b/roomserver/storage/sqlite3/state_snapshot_table.go
@@ -137,7 +137,7 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 		}
 	}
 	if i != len(stateNIDs) {
-		return nil, fmt.Errorf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))
+		return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs)))
 	}
 	return results, nil
 }
diff --git a/roomserver/types/types.go b/roomserver/types/types.go
index 5d52ccfcd..65fbee04e 100644
--- a/roomserver/types/types.go
+++ b/roomserver/types/types.go
@@ -213,6 +213,12 @@ type MissingEventError string
 
 func (e MissingEventError) Error() string { return string(e) }
 
+// A MissingStateError is an error that happened because the roomserver was
+// missing requested state snapshots from its databases.
+type MissingStateError string
+
+func (e MissingStateError) Error() string { return string(e) }
+
 // A RejectedError is returned when an event is stored as rejected. The error
 // contains the reason why.
 type RejectedError string

From bbe7d37928194d411d99079055cdcec9dbdd4959 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 21 Feb 2022 16:38:53 +0000
Subject: [PATCH 05/12] Fix logic error on context history visibility (#2211)

---
 syncapi/routing/context.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 709f62916..ef7efa2b0 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -64,7 +64,7 @@ func Context(
 	ctx := req.Context()
 	membershipRes := roomserver.QueryMembershipForUserResponse{}
 	membershipReq := roomserver.QueryMembershipForUserRequest{UserID: device.UserID, RoomID: roomID}
-	if err := rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes); err != nil {
+	if err = rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes); err != nil {
 		logrus.WithError(err).Error("unable to fo membership")
 		return jsonerror.InternalServerError()
 	}
@@ -86,11 +86,12 @@ func Context(
 	state, _ := syncDB.CurrentState(ctx, roomID, &stateFilter, nil)
 	// verify the user is allowed to see the context for this room/event
 	for _, x := range state {
-		hisVis, err := x.HistoryVisibility()
+		var hisVis string
+		hisVis, err = x.HistoryVisibility()
 		if err != nil {
 			continue
 		}
-		allowed := hisVis != "world_readable" && membershipRes.Membership == "join"
+		allowed := hisVis == gomatrixserverlib.WorldReadable || membershipRes.Membership == gomatrixserverlib.Join
 		if !allowed {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,

From 600fbae31f1b0b6919bb9045958b932071de6737 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 22 Feb 2022 13:35:06 +0000
Subject: [PATCH 06/12] Only emit key change notifications from federation when
 changes are made (#2217)

* Only emit key changes when poked over federation

* Remove logging

* Fix unit test possibly
---
 keyserver/internal/device_list_update.go | 27 ++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index c5a5d40c7..b208f0ce5 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -241,14 +241,33 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib.
 				StreamID: event.StreamID,
 			},
 		}
+
+		// DeviceKeysJSON will side-effect modify this, so it needs
+		// to be a copy, not sharing any pointers with the above.
+		deviceKeysCopy := *keys[0].DeviceKeys
+		deviceKeysCopy.KeyJSON = nil
+		existingKeys := []api.DeviceMessage{
+			{
+				Type:       keys[0].Type,
+				DeviceKeys: &deviceKeysCopy,
+				StreamID:   keys[0].StreamID,
+			},
+		}
+
+		// fetch what keys we had already and only emit changes
+		if err = u.db.DeviceKeysJSON(ctx, existingKeys); err != nil {
+			// non-fatal, log and continue
+			util.GetLogger(ctx).WithError(err).WithField("user_id", event.UserID).Errorf(
+				"failed to query device keys json for calculating diffs",
+			)
+		}
+
 		err = u.db.StoreRemoteDeviceKeys(ctx, keys, nil)
 		if err != nil {
 			return false, fmt.Errorf("failed to store remote device keys for %s (%s): %w", event.UserID, event.DeviceID, err)
 		}
-		// ALWAYS emit key changes when we've been poked over federation even if there's no change
-		// just in case this poke is important for something.
-		err = u.producer.ProduceKeyChanges(keys)
-		if err != nil {
+
+		if err = emitDeviceKeyChanges(u.producer, existingKeys, keys); err != nil {
 			return false, fmt.Errorf("failed to produce device key changes for %s (%s): %w", event.UserID, event.DeviceID, err)
 		}
 		return false, nil

From e9545dc12fc699b237f51c1aa67dbbef898b27ce Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 22 Feb 2022 13:40:08 +0000
Subject: [PATCH 07/12] Remove error when state keys are missing for user NIDs
 (#2213)

* Remove error when state keys are missing for user NIDs

There is still an actual bug here somewhere in the membership updater, but this check does more harm than good, since it means that the key consumers don't actually distribute updates to *anyone*. It's better just to deal with this silently for now.

To find these broken rows:

```
SELECT * FROM roomserver_membership AS m WHERE NOT EXISTS (
	SELECT event_state_key_nid FROM roomserver_event_state_keys AS s
	WHERE m.sender_nid = s.event_state_key_nid
);
```

* Logging
---
 roomserver/storage/shared/storage.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index b255cfb3f..e270e121c 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -13,6 +13,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
 )
 
@@ -1101,7 +1102,7 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs []string)
 		return nil, err
 	}
 	if len(nidToUserID) != len(userNIDToCount) {
-		return nil, fmt.Errorf("found %d users but only have state key nids for %d of them", len(userNIDToCount), len(nidToUserID))
+		logrus.Warnf("SelectJoinedUsersSetForRooms found %d users but BulkSelectEventStateKey only returned state key NIDs for %d of them", len(userNIDToCount), len(nidToUserID))
 	}
 	result := make(map[string]int, len(userNIDToCount))
 	for nid, count := range userNIDToCount {

From 34116178e8ef75569a55655e3001a51f77dfb789 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 22 Feb 2022 13:47:14 +0000
Subject: [PATCH 08/12] Remove logging line in `PerformInvite`

---
 federationapi/internal/perform.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index c51ecf146..b888b3654 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -554,7 +554,6 @@ func (r *FederationInternalAPI) PerformInvite(
 	if err != nil {
 		return fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err)
 	}
-	logrus.Infof("GOT INVITE RESPONSE %s", string(inviteRes.Event))
 
 	inviteEvent, err := inviteRes.Event.UntrustedEvent(request.RoomVersion)
 	if err != nil {

From c7811e9d71041527128f4a1782e2b92453f0f57c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 22 Feb 2022 15:43:17 +0000
Subject: [PATCH 09/12] Add `DeviceKeysEqual` (#2219)

* Add `DeviceKeysEqual`

* Update check order

* Fix check

* Tweak conditions again

* One more time

* Single return value
---
 keyserver/api/api.go           | 21 +++++++++++++++++++++
 keyserver/internal/internal.go |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index 3933961c1..54eb04f8a 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -15,6 +15,7 @@
 package api
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"strings"
@@ -73,6 +74,26 @@ type DeviceMessage struct {
 	DeviceChangeID int64
 }
 
+// DeviceKeysEqual returns true if the device keys updates contain the
+// same display name and key JSON. This will return false if either of
+// the updates is not a device keys update, or if the user ID/device ID
+// differ between the two.
+func (m1 *DeviceMessage) DeviceKeysEqual(m2 *DeviceMessage) bool {
+	if m1.DeviceKeys == nil || m2.DeviceKeys == nil {
+		return false
+	}
+	if m1.UserID != m2.UserID || m1.DeviceID != m2.DeviceID {
+		return false
+	}
+	if m1.DisplayName != m2.DisplayName {
+		return false // different display names
+	}
+	if len(m1.KeyJSON) == 0 || len(m2.KeyJSON) == 0 {
+		return false // either is empty
+	}
+	return bytes.Equal(m1.KeyJSON, m2.KeyJSON)
+}
+
 // DeviceKeys represents a set of device keys for a single device
 // https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-keys-upload
 type DeviceKeys struct {
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 0c264b718..dc3c404bd 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -718,7 +718,7 @@ func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.Device
 		for _, existingKey := range existing {
 			// Do not treat the absence of keys as equal, or else we will not emit key changes
 			// when users delete devices which never had a key to begin with as both KeyJSONs are nil.
-			if bytes.Equal(existingKey.KeyJSON, newKey.KeyJSON) && len(existingKey.KeyJSON) > 0 {
+			if existingKey.DeviceKeysEqual(&newKey) {
 				exists = true
 				break
 			}

From cfff1b0aaafc68efbac27f366360667983f7696a Mon Sep 17 00:00:00 2001
From: S7evinK <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 22 Feb 2022 16:50:04 +0100
Subject: [PATCH 10/12] Remote banned user is kicked and may not rejoin until
 unbanned (#2216)

* Remote banned user is kicked and may not rejoin until unbanned

* Use gmsl constant
---
 clientapi/routing/state.go | 2 +-
 sytest-whitelist           | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go
index 088e412c6..d25ee8237 100644
--- a/clientapi/routing/state.go
+++ b/clientapi/routing/state.go
@@ -235,7 +235,7 @@ func OnIncomingStateTypeRequest(
 		}
 		// If the user has never been in the room then stop at this point.
 		// We won't tell the user about a room they have never joined.
-		if !membershipRes.HasBeenInRoom {
+		if !membershipRes.HasBeenInRoom || membershipRes.Membership == gomatrixserverlib.Ban {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
diff --git a/sytest-whitelist b/sytest-whitelist
index 187a0f475..87904938c 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -596,4 +596,5 @@ Device list doesn't change if remote server is down
 /context/ on joined room works
 /context/ on non world readable room does not work
 /context/ returns correct number of events
-/context/ with lazy_load_members filter works
\ No newline at end of file
+/context/ with lazy_load_members filter works
+Remote banned user is kicked and may not rejoin until unbanned

From 41dc651b25019b9ecc2338ff2ebec202066aaf21 Mon Sep 17 00:00:00 2001
From: S7evinK <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 22 Feb 2022 17:34:53 +0100
Subject: [PATCH 11/12] Send device update to local users if remote display
 name changes (#2215)

* Send device_list update to satisfy sytest

* Fix build issue from merged in change

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 keyserver/internal/device_list_update.go | 6 +++---
 keyserver/internal/internal.go           | 8 ++++++--
 sytest-whitelist                         | 3 +++
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index b208f0ce5..974d0196b 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -224,7 +224,7 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib.
 	}).Info("DeviceListUpdater.Update")
 
 	// if we haven't missed anything update the database and notify users
-	if exists {
+	if exists || event.Deleted {
 		k := event.Keys
 		if event.Deleted {
 			k = nil
@@ -267,7 +267,7 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib.
 			return false, fmt.Errorf("failed to store remote device keys for %s (%s): %w", event.UserID, event.DeviceID, err)
 		}
 
-		if err = emitDeviceKeyChanges(u.producer, existingKeys, keys); err != nil {
+		if err = emitDeviceKeyChanges(u.producer, existingKeys, keys, false); err != nil {
 			return false, fmt.Errorf("failed to produce device key changes for %s (%s): %w", event.UserID, event.DeviceID, err)
 		}
 		return false, nil
@@ -473,7 +473,7 @@ func (u *DeviceListUpdater) updateDeviceList(res *gomatrixserverlib.RespUserDevi
 	if err != nil {
 		return fmt.Errorf("failed to mark device list as fresh: %w", err)
 	}
-	err = emitDeviceKeyChanges(u.producer, existingKeys, keys)
+	err = emitDeviceKeyChanges(u.producer, existingKeys, keys, false)
 	if err != nil {
 		return fmt.Errorf("failed to emit key changes for fresh device list: %w", err)
 	}
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index dc3c404bd..0a8bef95d 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -648,7 +648,7 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per
 		}
 		return
 	}
-	err = emitDeviceKeyChanges(a.Producer, existingKeys, keysToStore)
+	err = emitDeviceKeyChanges(a.Producer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates)
 	if err != nil {
 		util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err)
 	}
@@ -710,7 +710,11 @@ func (a *KeyInternalAPI) uploadOneTimeKeys(ctx context.Context, req *api.Perform
 
 }
 
-func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.DeviceMessage) error {
+func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.DeviceMessage, onlyUpdateDisplayName bool) error {
+	// if we only want to update the display names, we can skip the checks below
+	if onlyUpdateDisplayName {
+		return producer.ProduceKeyChanges(new)
+	}
 	// find keys in new that are not in existing
 	var keysAdded []api.DeviceMessage
 	for _, newKey := range new {
diff --git a/sytest-whitelist b/sytest-whitelist
index 87904938c..d3144572d 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -597,4 +597,7 @@ Device list doesn't change if remote server is down
 /context/ on non world readable room does not work
 /context/ returns correct number of events
 /context/ with lazy_load_members filter works
+Can query remote device keys using POST after notification
+Device deletion propagates over federation
+Get left notifs in sync and /keys/changes when other user leaves
 Remote banned user is kicked and may not rejoin until unbanned

From 2b0a5adfafa46eb4df1ddb7ba909801366b920cf Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 22 Feb 2022 16:55:08 +0000
Subject: [PATCH 12/12] Version 0.6.4 (#2212)

* Version 0.6.4

* Tweaks

* Update changelog

* Update changelog one last time
---
 CHANGES.md          | 34 ++++++++++++++++++++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 4df8e869a..ee608194d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,39 @@
 # Changelog
 
+## Dendrite 0.6.4 (2022-02-21)
+
+### Features
+
+* All Client-Server API endpoints are now available under the `/v3` namespace
+* The `/whoami` response format now matches the latest Matrix spec version
+* Support added for the `/context` endpoint, which should help clients to render quote-replies correctly
+* Accounts now have an optional account type field, allowing admin accounts to be created
+* Server notices are now supported
+* Refactored the user API storage to deduplicate a significant amount of code, as well as merging both user API databases into a single database
+  * The account database is now used for all user API storage and the device database is now obsolete
+  * For some installations that have separate account and device databases, this may result in access tokens being revoked and client sessions being logged out — users may need to log in again
+  * The above can be avoided by moving the `device_devices` table into the account database manually
+* Guest registration can now be separately disabled with the new `client_api.guests_disabled` configuration option
+* Outbound connections now obey proxy settings from the environment, deprecating the `federation_api.proxy_outbound` configuration options
+
+### Fixes
+
+* The roomserver input API will now strictly consume only one database transaction per room, which should prevent situations where the roomserver can deadlock waiting for database connections to become available
+* Room joins will now fall back to federation if the local room state is insufficient to create a membership event
+* Create events are now correctly filtered from federation `/send` transactions
+* Excessive logging when federation is disabled should now be fixed
+* Dendrite will no longer panic if trying to retire an invite event that has not been seen yet
+* The device list updater will now wait for longer after a connection issue, rather than flooding the logs with errors
+* The device list updater will no longer produce unnecessary output events for federated key updates with no changes, which should help to reduce CPU usage
+* Local device name changes will now generate key change events correctly
+* The sync API will now try to share device list update notifications even if all state key NIDs cannot be fetched
+* An off-by-one error in the sync stream token handling which could result in a crash has been fixed
+* State events will no longer be re-sent unnecessary by the roomserver to other components if they have already been sent, which should help to reduce the NATS message sizes on the roomserver output topic in some cases
+* The roomserver input API now uses the process context and should handle graceful shutdowns better
+* Guest registration is now correctly disabled when the `client_api.registration_disabled` configuration option is set
+* One-time encryption keys are now cleaned up correctly when a device is logged out or removed
+* Invalid state snapshots in the state storage refactoring migration are now reset rather than causing a panic at startup
+
 ## Dendrite 0.6.3 (2022-02-10)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index a07f01b61..2ea1c5201 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 6
-	VersionPatch = 3
+	VersionPatch = 4
 	VersionTag   = "" // example: "rc1"
 )