Merge pull request #84 from globekeeper/release/upstream-v0.13.6-17

Release/upstream v0.13.6 17

.github/codecov.yaml

@@ -7,7 +7,7 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 0%
+        threshold: 0.1%
         base: auto
         flags:
           - unittests

CHANGES.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## Dendrite 0.13.6 (2024-01-26)
+
+Upgrading to this version is **highly** recommended, as it contains several QoL improvements.
+
+### Fixes
+
+- Use `AckExplicitPolicy` for JetStream consumers, so messages don't pile up in NATS
+- A rare panic when assigning a state key NID has been fixed
+- A rare panic when checking powerlevels has been fixed
+- Notary keys requests for all keys now work correctly
+- Spec compliance:
+  - Return `M_INVALID_PARAM` when querying room aliases
+  - Handle empty `from` parameter when requesting `/messages`
+  - Add CORP headers on media endpoints
+  - Remove `aliases` from `/publicRooms` responses
+  - Allow `+` in MXIDs (Contributed by [RosstheRoss](https://github.com/RosstheRoss))
+- Fixes membership transitions from `knock` to `join` in `knock_restricted` rooms
+- Incremental syncs now batch querying events (Contributed by [recht](https://github.com/recht))
+- Move `/joined_members` back to the clientAPI/roomserver, which should make bridges happier again
+- Backfilling from other servers now only uses at max 100 events instead of potentially thousands
+
 ## Dendrite 0.13.5 (2023-12-12)
 
 Upgrading to this version is **highly** recommended, as it fixes several long-standing bugs in

build/docker/Dockerfile.demo-pinecone

@@ -1,4 +1,5 @@
-FROM docker.io/golang:1.21-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM docker.io/golang:1.21-alpine3.18 AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/docker/Dockerfile.demo-yggdrasil

@@ -1,4 +1,5 @@
-FROM docker.io/golang:1.21-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM docker.io/golang:1.21-alpine3.18 AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

clientapi/clientapi_test.go

@@ -2148,3 +2148,130 @@ func TestKeyBackup(t *testing.T) {
 		}
 	})
 }
+
+func TestGetMembership(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	testCases := []struct {
+		name             string
+		roomID           string
+		user             *test.User
+		additionalEvents func(t *testing.T, room *test.Room)
+		request          func(t *testing.T, room *test.Room, accessToken string) *http.Request
+		wantOK           bool
+		wantMemberCount  int
+	}{
+
+		{
+			name: "/joined_members - Bob never joined",
+			user: bob,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Alice joined",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK:          true,
+			wantMemberCount: 1,
+		},
+		{
+			name: "/joined_members - Alice leaves, shouldn't be able to see members ",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
+					"membership": "leave",
+				}, test.WithStateKey(alice.ID))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Bob joins, Alice sees two members",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+					"membership": "join",
+				}, test.WithStateKey(bob.ID))
+			},
+			wantOK:          true,
+			wantMemberCount: 2,
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				room := test.NewRoom(t, alice)
+				t.Cleanup(func() {
+					t.Logf("running cleanup for %s", tc.name)
+				})
+				// inject additional events
+				if tc.additionalEvents != nil {
+					tc.additionalEvents(t, room)
+				}
+				if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+					t.Fatalf("failed to send events: %v", err)
+				}
+
+				w := httptest.NewRecorder()
+				routers.Client.ServeHTTP(w, tc.request(t, room, accessTokens[tc.user].accessToken))
+				if w.Code != 200 && tc.wantOK {
+					t.Logf("%s", w.Body.String())
+					t.Fatalf("got HTTP %d want %d", w.Code, 200)
+				}
+				t.Logf("[%s] Resp: %s", tc.name, w.Body.String())
+
+				// check we got the expected events
+				if tc.wantOK {
+					memberCount := len(gjson.GetBytes(w.Body.Bytes(), "joined").Map())
+					if memberCount != tc.wantMemberCount {
+						t.Fatalf("expected %d members, got %d", tc.wantMemberCount, memberCount)
+					}
+				}
+			})
+		}
+	})
+}

clientapi/routing/directory.go

@@ -55,7 +55,7 @@ func DirectoryRoom(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"),
+			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 
@@ -134,7 +134,7 @@ func SetLocalAlias(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"),
+			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 

clientapi/routing/memberships.go

@@ -0,0 +1,139 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+)
+
+// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
+type getJoinedMembersResponse struct {
+	Joined map[string]joinedMember `json:"joined"`
+}
+
+type joinedMember struct {
+	DisplayName string `json:"display_name"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// The database stores 'displayname' without an underscore.
+// Deserialize into this and then change to the actual API response
+type databaseJoinedMember struct {
+	DisplayName string `json:"displayname"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// GetJoinedMembers implements
+//
+//	GET /rooms/{roomId}/joined_members
+func GetJoinedMembers(
+	req *http.Request, device *userapi.Device, roomID string,
+	rsAPI api.ClientRoomserverAPI,
+) util.JSONResponse {
+	// Validate the userID
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("Device UserID is invalid"),
+		}
+	}
+
+	// Validate the roomID
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("RoomID is invalid"),
+		}
+	}
+
+	// Get the current memberships for the requesting user to determine
+	// if they are allowed to query this endpoint.
+	queryReq := api.QueryMembershipForUserRequest{
+		RoomID: validRoomID.String(),
+		UserID: *userID,
+	}
+
+	var queryRes api.QueryMembershipForUserResponse
+	if queryErr := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); queryErr != nil {
+		util.GetLogger(req.Context()).WithError(queryErr).Error("rsAPI.QueryMembershipsForRoom failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	if !queryRes.HasBeenInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	if !queryRes.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	// Get the current membership events
+	var membershipsForRoomResp api.QueryMembershipsForRoomResponse
+	if err = rsAPI.QueryMembershipsForRoom(req.Context(), &api.QueryMembershipsForRoomRequest{
+		JoinedOnly: true,
+		RoomID:     validRoomID.String(),
+	}, &membershipsForRoomResp); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryEventsByID failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	var res getJoinedMembersResponse
+	res.Joined = make(map[string]joinedMember)
+	for _, ev := range membershipsForRoomResp.JoinEvents {
+		var content databaseJoinedMember
+		if err := json.Unmarshal(ev.Content, &content); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(ev.Sender))
+		if err != nil || userID == nil {
+			util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		res.Joined[userID.String()] = joinedMember(content)
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}

clientapi/routing/register.go

@@ -635,6 +635,7 @@ func handleGuestRegistration(
 		AccessToken:       token,
 		IPAddr:            req.RemoteAddr,
 		UserAgent:         req.UserAgent(),
+		FromRegistration:  true,
 	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
@@ -982,6 +983,7 @@ func completeRegistration(
 		DeviceID:          deviceID,
 		IPAddr:            ipAddr,
 		UserAgent:         userAgent,
+		FromRegistration:  true,
 	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{

clientapi/routing/routing.go

@@ -1526,4 +1526,14 @@ func Setup(
 			return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomID}/joined_members",
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetJoinedMembers(req, device, vars["roomID"], rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
 }

federationapi/federationapi_test.go

@@ -5,11 +5,14 @@ import (
 	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -17,7 +20,10 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
@@ -362,3 +368,126 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 		}
 	}
 }
+
+func TestNotaryServer(t *testing.T) {
+	testCases := []struct {
+		name          string
+		httpBody      string
+		pubKeyRequest *gomatrixserverlib.PublicKeyNotaryLookupRequest
+		validateFunc  func(t *testing.T, response util.JSONResponse)
+	}{
+		{
+			name: "empty httpBody",
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusBadRequest, resp.Code)
+				nk, ok := resp.JSON.(spec.MatrixError)
+				assert.True(t, ok)
+				assert.Equal(t, spec.ErrorBadJSON, nk.ErrCode)
+			},
+		},
+		{
+			name:     "valid but empty httpBody",
+			httpBody: "{}",
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				want := util.JSONResponse{
+					Code: http.StatusOK,
+					JSON: routing.NotaryKeysResponse{ServerKeys: []json.RawMessage{}},
+				}
+				assert.Equal(t, want, resp)
+			},
+		},
+		{
+			name:     "request all keys using an empty criteria",
+			httpBody: `{"server_keys":{"servera":{}}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
+				assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
+			},
+		},
+		{
+			name:     "request all keys using null as the criteria",
+			httpBody: `{"server_keys":{"servera":null}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
+				assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
+			},
+		},
+		{
+			name:     "request specific key",
+			httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}}}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
+				assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
+			},
+		},
+		{
+			name:     "request multiple servers",
+			httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}},"serverb":{"ed25519:someID":{}}}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				wantServers := map[string]struct{}{
+					"servera": {},
+					"serverb": {},
+				}
+				for _, js := range nk.ServerKeys {
+					serverName := gjson.GetBytes(js, "server_name").Str
+					_, ok = wantServers[serverName]
+					assert.True(t, ok, "unexpected servername: %s", serverName)
+					delete(wantServers, serverName)
+					assert.True(t, gjson.GetBytes(js, "verify_keys.ed25519:someID").Exists())
+				}
+				if len(wantServers) > 0 {
+					t.Fatalf("expected response to also contain: %#v", wantServers)
+				}
+			},
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		fc := &fedClient{
+			keys: map[spec.ServerName]struct {
+				key   ed25519.PrivateKey
+				keyID gomatrixserverlib.KeyID
+			}{
+				"servera": {
+					key:   test.PrivateKeyA,
+					keyID: "ed25519:someID",
+				},
+				"serverb": {
+					key:   test.PrivateKeyB,
+					keyID: "ed25519:someID",
+				},
+			},
+		}
+
+		fedAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, fc, nil, caches, nil, true)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tc.httpBody))
+				req.Host = string(cfg.Global.ServerName)
+
+				resp := routing.NotaryKeys(req, &cfg.FederationAPI, fedAPI, tc.pubKeyRequest)
+				// assert that we received the expected response
+				tc.validateFunc(t, resp)
+			})
+		}
+
+	})
+}

federationapi/internal/query.go

@@ -43,6 +43,15 @@ func (a *FederationInternalAPI) fetchServerKeysFromCache(
 	ctx context.Context, req *api.QueryServerKeysRequest,
 ) ([]gomatrixserverlib.ServerKeys, error) {
 	var results []gomatrixserverlib.ServerKeys
+
+	// We got a request for _all_ server keys, return them.
+	if len(req.KeyIDToCriteria) == 0 {
+		serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{})
+		if len(serverKeysResponses) == 0 {
+			return nil, fmt.Errorf("failed to find server key response for server %s", req.ServerName)
+		}
+		return serverKeysResponses, nil
+	}
 	for keyID, criteria := range req.KeyIDToCriteria {
 		serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{keyID})
 		if len(serverKeysResponses) == 0 {

federationapi/routing/backfill.go

@@ -95,6 +95,12 @@ func Backfill(
 		}
 	}
 
+	// Enforce a limit of 100 events, as not to hit the DB to hard.
+	// Synapse has a hard limit of 100 events as well.
+	if req.Limit > 100 {
+		req.Limit = 100
+	}
+
 	// Query the Roomserver.
 	if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed")

federationapi/routing/keys.go

@@ -197,6 +197,10 @@ func localKeys(cfg *config.FederationAPI, serverName spec.ServerName) (*gomatrix
 	return &keys, err
 }
 
+type NotaryKeysResponse struct {
+	ServerKeys []json.RawMessage `json:"server_keys"`
+}
+
 func NotaryKeys(
 	httpReq *http.Request, cfg *config.FederationAPI,
 	fsAPI federationAPI.FederationInternalAPI,
@@ -217,10 +221,9 @@ func NotaryKeys(
 		}
 	}
 
-	var response struct {
+	response := NotaryKeysResponse{
-		ServerKeys []json.RawMessage `json:"server_keys"`
+		ServerKeys: []json.RawMessage{},
 	}
-	response.ServerKeys = []json.RawMessage{}
 
 	for serverName, kidToCriteria := range req.ServerKeys {
 		var keyList []gomatrixserverlib.ServerKeys

federationapi/routing/routing.go

@@ -647,6 +647,8 @@ func MakeFedAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
+			hub = hub.Clone()
 			hub.Scope().SetTag("origin", string(fedReq.Origin()))
 			hub.Scope().SetTag("uri", fedReq.RequestURI())
 		}

go.mod

@@ -22,12 +22,12 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20231212115925-41497b7563eb
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20240109180417-3495e573f2b7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/matryer/is v1.4.1
 	github.com/mattn/go-sqlite3 v1.14.17
-	github.com/nats-io/nats-server/v2 v2.9.23
+	github.com/nats-io/nats-server/v2 v2.10.7
-	github.com/nats-io/nats.go v1.28.0
+	github.com/nats-io/nats.go v1.31.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/opentracing/opentracing-go v1.2.0
@@ -42,12 +42,12 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
 	go.uber.org/atomic v1.10.0
-	golang.org/x/crypto v0.14.0
+	golang.org/x/crypto v0.17.0
 	golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
 	golang.org/x/image v0.10.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
 	golang.org/x/sync v0.3.0
-	golang.org/x/term v0.13.0
+	golang.org/x/term v0.15.0
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.4.0
@@ -93,7 +93,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v1.0.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
@@ -103,7 +103,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
-	github.com/nats-io/jwt/v2 v2.5.0 // indirect
+	github.com/nats-io/jwt/v2 v2.5.3 // indirect
 	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
@@ -120,9 +120,9 @@ require (
 	go.etcd.io/bbolt v1.3.6 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.12.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect

go.sum

@@ -171,8 +171,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -185,8 +185,8 @@ github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBE
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20231212115925-41497b7563eb h1:Nn+Fr96oi7bIfdOwX5A2L6A2MZCM+lqwLe4/+3+nYj8=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240109180417-3495e573f2b7 h1:EaUvK2ay6cxMxeshC1p6QswS9+rQFbUc2YerkRFyVXQ=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20231212115925-41497b7563eb/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240109180417-3495e573f2b7/go.mod h1:HZGsVJ3bUE+DkZtufkH9H0mlsvbhEGK5CpX0Zlavylg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66/go.mod h1:iBI1foelCqA09JJgPV0FYz4qA5dUXYOxMi57FxKBdd4=
 github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ=
@@ -217,12 +217,12 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
-github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak=
+github.com/nats-io/jwt/v2 v2.5.3 h1:/9SWvzc6hTfamcgXJ3uYRpgj+QuY2aLNqRiqrKcrpEo=
-github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
+github.com/nats-io/jwt/v2 v2.5.3/go.mod h1:iysuPemFcc7p4IoYots3IuELSI4EDe9Y0bQMe+I3Bf4=
-github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU=
+github.com/nats-io/nats-server/v2 v2.10.7 h1:f5VDy+GMu7JyuFA0Fef+6TfulfCs5nBTgq7MMkFJx5Y=
-github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0=
+github.com/nats-io/nats-server/v2 v2.10.7/go.mod h1:V2JHOvPiPdtfDXTuEUsthUnCvSDeFrK4Xn9hRo6du7c=
-github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c=
+github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
-github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
+github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
 github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
 github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -316,8 +316,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -381,25 +381,26 @@ golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

helm/dendrite/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.6"
+version: "0.13.7"
-appVersion: "0.13.5"
+appVersion: "0.13.6"
 description: Dendrite Matrix Homeserver
 type: application
 keywords:

helm/dendrite/README.md

@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.6](https://img.shields.io/badge/Version-0.13.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.5](https://img.shields.io/badge/AppVersion-0.13.5-informational?style=flat-square)
+![Version: 0.13.7](https://img.shields.io/badge/Version-0.13.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.6](https://img.shields.io/badge/AppVersion-0.13.6-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**

internal/httputil/httpapi.go

@@ -76,6 +76,8 @@ func MakeAuthAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
+			hub = hub.Clone()
 			hub.Scope().SetUser(sentry.User{
 				Username: device.UserID,
 			})

internal/validate.go

@@ -38,9 +38,9 @@ var (
 	ErrPasswordTooLong    = fmt.Errorf("password too long: max %d characters", maxPasswordLength)
 	ErrPasswordWeak       = fmt.Errorf("password too weak: min %d characters", minPasswordLength)
 	ErrUsernameTooLong    = fmt.Errorf("username exceeds the maximum length of %d characters", maxUsernameLength)
-	ErrUsernameInvalid    = errors.New("username can only contain characters a-z, 0-9, or '_-./='")
+	ErrUsernameInvalid    = errors.New("username can only contain characters a-z, 0-9, or '_+-./='")
 	ErrUsernameUnderscore = errors.New("username cannot start with a '_'")
-	validUsernameRegex    = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
+	validUsernameRegex    = regexp.MustCompile(`^[0-9a-z_\-+=./]+$`)
 )
 
 // ValidatePassword returns an error if the password is invalid

internal/validate_test.go

@@ -129,6 +129,11 @@ func Test_validateUsername(t *testing.T) {
 			localpart: "i_am_allowed=1",
 			domain:    "localhost",
 		},
+		{
+			name:      "special characters are allowed 3",
+			localpart: "+55555555555",
+			domain:    "localhost",
+		},
 		{
 			name:      "not all special characters are allowed",
 			localpart: "notallowed#", // contains #
@@ -139,6 +144,16 @@ func Test_validateUsername(t *testing.T) {
 				JSON: spec.InvalidUsername(ErrUsernameInvalid.Error()),
 			},
 		},
+		{
+			name:      "not all special characters are allowed 2",
+			localpart: "<notallowed", // contains <
+			domain:    "localhost",
+			wantErr:   ErrUsernameInvalid,
+			wantJSON: &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidUsername(ErrUsernameInvalid.Error()),
+			},
+		},
 		{
 			name:      "username containing numbers",
 			localpart: "hello1337",

internal/version.go

@@ -18,7 +18,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 13
-	VersionPatch = 5
+	VersionPatch = 6
 	VersionTag   = "" // example: "rc1"
 
 	gitRevLen = 7 // 7 matches the displayed characters on github.com

mediaapi/routing/routing.go

@@ -123,6 +123,7 @@ func makeDownloadAPI(
 
 		// Set internal headers returned regardless of the outcome of the request
 		util.SetCORSHeaders(w)
+		w.Header().Set("Cross-Origin-Resource-Policy", "cross-origin")
 		// Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors
 		w.Header().Set("Content-Type", "application/json")
 

roomserver/api/perform.go

@@ -8,7 +8,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
-	"github.com/matrix-org/util"
 )
 
 type PerformCreateRoomRequest struct {
@@ -91,14 +90,44 @@ type PerformBackfillRequest struct {
 	VirtualHost spec.ServerName `json:"virtual_host"`
 }
 
-// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order.
+// limitPrevEventIDs is the maximum of eventIDs we
+// return when calling PrevEventIDs.
+const limitPrevEventIDs = 100
+
+// PrevEventIDs returns the prev_event IDs of either 100 backwards extremities or
+// len(r.BackwardsExtremities). Limited to 100, due to Synapse/Dendrite stopping after reaching
+// this limit. (which sounds sane)
 func (r *PerformBackfillRequest) PrevEventIDs() []string {
-	var prevEventIDs []string
+	var uniqueIDs map[string]struct{}
-	for _, pes := range r.BackwardsExtremities {
+
-		prevEventIDs = append(prevEventIDs, pes...)
+	// Create a unique eventID map of either 100 or len(r.BackwardsExtremities).
+	// 100 since Synapse/Dendrite stops after reaching 100 events.
+	if len(r.BackwardsExtremities) > limitPrevEventIDs {
+		uniqueIDs = make(map[string]struct{}, limitPrevEventIDs)
+	} else {
+		uniqueIDs = make(map[string]struct{}, len(r.BackwardsExtremities))
 	}
-	prevEventIDs = util.UniqueStrings(prevEventIDs)
+
-	return prevEventIDs
+outerLoop:
+	for _, pes := range r.BackwardsExtremities {
+		for _, evID := range pes {
+			uniqueIDs[evID] = struct{}{}
+			// We found enough unique eventIDs.
+			if len(uniqueIDs) >= limitPrevEventIDs {
+				break outerLoop
+			}
+		}
+	}
+
+	// map -> []string
+	result := make([]string, len(uniqueIDs))
+	i := 0
+	for evID := range uniqueIDs {
+		result[i] = evID
+		i++
+	}
+
+	return result
 }
 
 // PerformBackfillResponse is a response to PerformBackfill.

roomserver/api/perform_test.go

@@ -0,0 +1,81 @@
+package api
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func BenchmarkPrevEventIDs(b *testing.B) {
+	for _, x := range []int64{1, 10, 100, 500, 1000, 2000} {
+		benchPrevEventIDs(b, int(x))
+	}
+}
+
+func benchPrevEventIDs(b *testing.B, count int) {
+	bwExtrems := generateBackwardsExtremities(b, count)
+	backfiller := PerformBackfillRequest{
+		BackwardsExtremities: bwExtrems,
+	}
+
+	b.Run(fmt.Sprintf("Original%d", count), func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			prevIDs := backfiller.PrevEventIDs()
+			_ = prevIDs
+		}
+	})
+}
+
+type testLike interface {
+	Helper()
+}
+
+const randomIDCharsCount = 10
+
+func generateBackwardsExtremities(t testLike, count int) map[string][]string {
+	t.Helper()
+	result := make(map[string][]string, count)
+	for i := 0; i < count; i++ {
+		eventID := randomEventId(int64(i))
+		result[eventID] = []string{
+			randomEventId(int64(i + 1)),
+			randomEventId(int64(i + 2)),
+			randomEventId(int64(i + 3)),
+		}
+	}
+	return result
+}
+
+const alphanumerics = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+// randomEventId generates a pseudo-random string of length n.
+func randomEventId(src int64) string {
+	randSrc := rand.NewSource(src)
+	b := make([]byte, randomIDCharsCount)
+	for i := range b {
+		b[i] = alphanumerics[randSrc.Int63()%int64(len(alphanumerics))]
+	}
+	return string(b)
+}
+
+func TestPrevEventIDs(t *testing.T) {
+	// generate 10 backwards extremities
+	bwExtrems := generateBackwardsExtremities(t, 10)
+	backfiller := PerformBackfillRequest{
+		BackwardsExtremities: bwExtrems,
+	}
+
+	prevIDs := backfiller.PrevEventIDs()
+	// Given how "generateBackwardsExtremities" works, this
+	// generates 12 unique event IDs
+	assert.Equal(t, 12, len(prevIDs))
+
+	// generate 200 backwards extremities
+	backfiller.BackwardsExtremities = generateBackwardsExtremities(t, 200)
+	prevIDs = backfiller.PrevEventIDs()
+	// PrevEventIDs returns at max 100 event IDs
+	assert.Equal(t, 100, len(prevIDs))
+}

roomserver/internal/input/input.go

@@ -108,20 +108,27 @@ type worker struct {
 	r            *Inputer
 	roomID       string
 	subscription *nats.Subscription
+	sentryHub    *sentry.Hub
 }
 
 func (r *Inputer) startWorkerForRoom(roomID string) {
 	v, loaded := r.workers.LoadOrStore(roomID, &worker{
-		r:      r,
+		r:         r,
-		roomID: roomID,
+		roomID:    roomID,
+		sentryHub: sentry.CurrentHub().Clone(),
 	})
 	w := v.(*worker)
 	w.Lock()
 	defer w.Unlock()
 	if !loaded || w.subscription == nil {
+		streamName := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent)
 		consumer := r.Cfg.Matrix.JetStream.Prefixed("RoomInput" + jetstream.Tokenise(w.roomID))
 		subject := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEventSubj(w.roomID))
 
+		logger := logrus.WithFields(logrus.Fields{
+			"stream_name": streamName,
+			"consumer":    consumer,
+		})
 		// Create the consumer. We do this as a specific step rather than
 		// letting PullSubscribe create it for us because we need the consumer
 		// to outlive the subscription. If we do it this way, we can Bind in the
@@ -135,21 +142,62 @@ func (r *Inputer) startWorkerForRoom(roomID string) {
 		// before it. This is necessary because otherwise our consumer will never
 		// acknowledge things we filtered out for other subjects and therefore they
 		// will linger around forever.
-		if _, err := w.r.JetStream.AddConsumer(
+
-			r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent),
+		info, err := w.r.JetStream.ConsumerInfo(streamName, consumer)
-			&nats.ConsumerConfig{
+		if err != nil && !errors.Is(err, nats.ErrConsumerNotFound) {
-				Durable:           consumer,
+			// log and return, we will retry anyway
-				AckPolicy:         nats.AckAllPolicy,
+			logger.WithError(err).Errorf("failed to get consumer info")
-				DeliverPolicy:     nats.DeliverAllPolicy,
-				FilterSubject:     subject,
-				AckWait:           MaximumMissingProcessingTime + (time.Second * 10),
-				InactiveThreshold: inactiveThreshold,
-			},
-		); err != nil {
-			logrus.WithError(err).Errorf("Failed to create consumer for room %q", w.roomID)
 			return
 		}
 
+		consumerConfig := &nats.ConsumerConfig{
+			Durable:           consumer,
+			AckPolicy:         nats.AckExplicitPolicy,
+			DeliverPolicy:     nats.DeliverAllPolicy,
+			FilterSubject:     subject,
+			AckWait:           MaximumMissingProcessingTime + (time.Second * 10),
+			InactiveThreshold: inactiveThreshold,
+		}
+
+		// The consumer already exists, try to update if necessary.
+		if info != nil {
+			// Not using reflect.DeepEqual here, since consumerConfig does not explicitly set
+			// e.g. the consumerName, which is added by NATS later. So this would result
+			// in constantly updating/recreating the consumer.
+			switch {
+			case info.Config.AckWait.Nanoseconds() != consumerConfig.AckWait.Nanoseconds():
+				// Initially we had a AckWait of 2m 10s, now we have 5m 10s, so we need to update
+				// existing consumers.
+				fallthrough
+			case info.Config.AckPolicy != consumerConfig.AckPolicy:
+				// We've changed the AckPolicy from AckAll to AckExplicit, this needs a
+				// recreation of the consumer. (Note: Only a few changes actually need a recreat)
+				logger.Warn("Consumer already exists, trying to update it.")
+				// Try updating the consumer first
+				if _, err = w.r.JetStream.UpdateConsumer(streamName, consumerConfig); err != nil {
+					// We failed to update the consumer, recreate it
+					logger.WithError(err).Warn("Unable to update consumer, recreating...")
+					if err = w.r.JetStream.DeleteConsumer(streamName, consumer); err != nil {
+						logger.WithError(err).Fatal("Unable to delete consumer")
+						return
+					}
+					// Set info to nil, so it can be recreated with the correct config.
+					info = nil
+				}
+			}
+		}
+
+		if info == nil {
+			// Create the consumer with the correct config
+			if _, err = w.r.JetStream.AddConsumer(
+				r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent),
+				consumerConfig,
+			); err != nil {
+				logger.WithError(err).Errorf("Failed to create consumer for room %q", w.roomID)
+				return
+			}
+		}
+
 		// Bind to our durable consumer. We want to receive all messages waiting
 		// for this subject and we want to manually acknowledge them, so that we
 		// can ensure they are only cleaned up when we are done processing them.
@@ -162,7 +210,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) {
 			nats.InactiveThreshold(inactiveThreshold),
 		)
 		if err != nil {
-			logrus.WithError(err).Errorf("Failed to subscribe to stream for room %q", w.roomID)
+			logger.WithError(err).Errorf("Failed to subscribe to stream for room %q", w.roomID)
 			return
 		}
 
@@ -219,9 +267,9 @@ func (w *worker) _next() {
 	// Look up what the next event is that's waiting to be processed.
 	ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute)
 	defer cancel()
-	if scope := sentry.CurrentHub().Scope(); scope != nil {
+	w.sentryHub.ConfigureScope(func(scope *sentry.Scope) {
 		scope.SetTag("room_id", w.roomID)
-	}
+	})
 	msgs, err := w.subscription.Fetch(1, nats.Context(ctx))
 	switch err {
 	case nil:
@@ -263,21 +311,23 @@ func (w *worker) _next() {
 		return
 	}
 
+	// Since we either Ack() or Term() the message at this point, we can defer decrementing the room backpressure
+	defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec()
+
 	// Try to unmarshal the input room event. If the JSON unmarshalling
 	// fails then we'll terminate the message — this notifies NATS that
 	// we are done with the message and never want to see it again.
 	msg := msgs[0]
 	var inputRoomEvent api.InputRoomEvent
 	if err = json.Unmarshal(msg.Data, &inputRoomEvent); err != nil {
-		_ = msg.Term()
+		// using AckWait here makes the call synchronous; 5 seconds is the default value used by NATS
+		_ = msg.Term(nats.AckWait(time.Second * 5))
 		return
 	}
 
-	if scope := sentry.CurrentHub().Scope(); scope != nil {
+	w.sentryHub.ConfigureScope(func(scope *sentry.Scope) {
 		scope.SetTag("event_id", inputRoomEvent.Event.EventID())
-	}
+	})
-	roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Inc()
-	defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec()
 
 	// Process the room event. If something goes wrong then we'll tell
 	// NATS to terminate the message. We'll store the error result as
@@ -299,7 +349,7 @@ func (w *worker) _next() {
 			}).Warn("Roomserver rejected event")
 		default:
 			if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
-				sentry.CaptureException(err)
+				w.sentryHub.CaptureException(err)
 			}
 			logrus.WithError(err).WithFields(logrus.Fields{
 				"room_id":  w.roomID,
@@ -307,10 +357,15 @@ func (w *worker) _next() {
 				"type":     inputRoomEvent.Event.Type(),
 			}).Warn("Roomserver failed to process event")
 		}
-		_ = msg.Term()
+		// Even though we failed to process this message (e.g. due to Dendrite restarting and receiving a context canceled),
+		// the message may already have been queued for redelivery or will be, so this makes sure that we still reprocess the msg
+		// after restarting. We only Ack if the context was not yet canceled.
+		if w.r.ProcessContext.Context().Err() == nil {
+			_ = msg.AckSync()
+		}
 		errString = err.Error()
 	} else {
-		_ = msg.Ack()
+		_ = msg.AckSync()
 	}
 
 	// If it was a synchronous input request then the "sync" field
@@ -381,6 +436,9 @@ func (r *Inputer) queueInputRoomEvents(
 			}).Error("Roomserver failed to queue async event")
 			return nil, fmt.Errorf("r.JetStream.PublishMsg: %w", err)
 		}
+
+		// Now that the event is queued, increment the room backpressure
+		roomserverInputBackpressure.With(prometheus.Labels{"room_id": roomID}).Inc()
 	}
 	return
 }

roomserver/internal/input/input_events.go

@@ -48,8 +48,10 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
 
-// TODO: Does this value make sense?
+// MaximumMissingProcessingTime is the maximum time we allow "processRoomEvent" to fetch
-const MaximumMissingProcessingTime = time.Minute * 2
+// e.g. missing auth/prev events. This duration is used for AckWait, and if it is exceeded
+// NATS queues the event for redelivery.
+const MaximumMissingProcessingTime = time.Minute * 5
 
 var processRoomEventDuration = prometheus.NewHistogramVec(
 	prometheus.HistogramOpts{

roomserver/internal/input/input_latest_events.go

@@ -298,6 +298,7 @@ func (u *latestEventsUpdater) latestState() error {
 		}).Warnf("State reset detected (removing %d events)", removed)
 		sentry.WithScope(func(scope *sentry.Scope) {
 			scope.SetLevel("warning")
+			scope.SetTag("room_id", u.event.RoomID().String())
 			scope.SetContext("State reset", map[string]interface{}{
 				"Event ID":      u.event.EventID(),
 				"Old state NID": fmt.Sprintf("%d", u.oldStateNID),

roomserver/roomserver_test.go

@@ -12,7 +12,9 @@ import (
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/internal/input"
 	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/tidwall/gjson"
 
@@ -1231,3 +1233,54 @@ func TestNewServerACLs(t *testing.T) {
 		assert.Equal(t, false, banned)
 	})
 }
+
+// Validate that changing the AckPolicy/AckWait of room consumers
+// results in their recreation
+func TestRoomConsumerRecreation(t *testing.T) {
+
+	alice := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	// As this is DB unrelated, just use SQLite
+	cfg, processCtx, closeDB := testrig.CreateConfig(t, test.DBTypeSQLite)
+	defer closeDB()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	natsInstance := &jetstream.NATSInstance{}
+
+	// Prepare a stream and consumer using the old configuration
+	jsCtx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+
+	streamName := cfg.Global.JetStream.Prefixed(jetstream.InputRoomEvent)
+	consumer := cfg.Global.JetStream.Prefixed("RoomInput" + jetstream.Tokenise(room.ID))
+	subject := cfg.Global.JetStream.Prefixed(jetstream.InputRoomEventSubj(room.ID))
+
+	consumerConfig := &nats.ConsumerConfig{
+		Durable:           consumer,
+		AckPolicy:         nats.AckAllPolicy,
+		DeliverPolicy:     nats.DeliverAllPolicy,
+		FilterSubject:     subject,
+		AckWait:           (time.Minute * 2) + (time.Second * 10),
+		InactiveThreshold: time.Hour * 24,
+	}
+
+	// Create the consumer with the old config
+	_, err := jsCtx.AddConsumer(streamName, consumerConfig)
+	assert.NoError(t, err)
+
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	// start JetStream listeners
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
+	rsAPI.SetFederationAPI(nil, nil)
+
+	// let the RS create the events, this also recreates the Consumers
+	err = api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false)
+	assert.NoError(t, err)
+
+	// Validate that AckPolicy and AckWait has changed
+	info, err := jsCtx.ConsumerInfo(streamName, consumer)
+	assert.NoError(t, err)
+	assert.Equal(t, nats.AckExplicitPolicy, info.Config.AckPolicy)
+
+	wantAckWait := input.MaximumMissingProcessingTime + (time.Second * 10)
+	assert.Equal(t, wantAckWait, info.Config.AckWait)
+}

roomserver/storage/shared/storage.go

@@ -889,10 +889,10 @@ func (d *Database) assignRoomNID(
 	}
 	// Check if we already have a numeric ID in the database.
 	roomNID, err := d.RoomsTable.SelectRoomNID(ctx, txn, roomID)
-	if err == sql.ErrNoRows {
+	if errors.Is(err, sql.ErrNoRows) {
 		// We don't have a numeric ID so insert one into the database.
 		roomNID, err = d.RoomsTable.InsertRoomNID(ctx, txn, roomID, roomVersion)
-		if err == sql.ErrNoRows {
+		if errors.Is(err, sql.ErrNoRows) {
 			// We raced with another insert so run the select again.
 			roomNID, err = d.RoomsTable.SelectRoomNID(ctx, txn, roomID)
 		}
@@ -914,10 +914,10 @@ func (d *Database) assignEventTypeNID(
 	}
 	// Check if we already have a numeric ID in the database.
 	eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType)
-	if err == sql.ErrNoRows {
+	if errors.Is(err, sql.ErrNoRows) {
 		// We don't have a numeric ID so insert one into the database.
 		eventTypeNID, err = d.EventTypesTable.InsertEventTypeNID(ctx, txn, eventType)
-		if err == sql.ErrNoRows {
+		if errors.Is(err, sql.ErrNoRows) {
 			// We raced with another insert so run the select again.
 			eventTypeNID, err = d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType)
 		}
@@ -938,16 +938,19 @@ func (d *EventDatabase) assignStateKeyNID(
 	}
 	// Check if we already have a numeric ID in the database.
 	eventStateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey)
-	if err == sql.ErrNoRows {
+	if errors.Is(err, sql.ErrNoRows) {
 		// We don't have a numeric ID so insert one into the database.
 		eventStateKeyNID, err = d.EventStateKeysTable.InsertEventStateKeyNID(ctx, txn, eventStateKey)
-		if err == sql.ErrNoRows {
+		if errors.Is(err, sql.ErrNoRows) {
 			// We raced with another insert so run the select again.
 			eventStateKeyNID, err = d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey)
 		}
 	}
+	if err != nil {
+		return 0, err
+	}
 	d.Cache.StoreEventStateKey(eventStateKeyNID, eventStateKey)
-	return eventStateKeyNID, err
+	return eventStateKeyNID, nil
 }
 
 func extractRoomVersionFromCreateEvent(event gomatrixserverlib.PDU) (

setup/base/base.go

@@ -50,6 +50,9 @@ import (
 //go:embed static/*.gotmpl
 var staticContent embed.FS
 
+//go:embed static/client/login
+var loginFallback embed.FS
+
 const HTTPServerTimeout = time.Minute * 5
 
 // CreateClient creates a new client (normally used for media fetch requests).
@@ -158,6 +161,14 @@ func SetupAndServeHTTP(
 		_, _ = w.Write(landingPage.Bytes())
 	})
 
+	// We only need the files beneath the static/client/login folder.
+	sub, err := fs.Sub(loginFallback, "static/client/login")
+	if err != nil {
+		logrus.Panicf("unable to read embedded files, this should never happen: %s", err)
+	}
+	// Serve a static page for login fallback
+	routers.Static.PathPrefix("/client/login/").Handler(http.StripPrefix("/_matrix/static/client/login/", http.FileServer(http.FS(sub))))
+
 	var clientHandler http.Handler
 	clientHandler = routers.Client
 	if cfg.Global.Sentry.Enabled {

setup/base/static/client/login/index.html

@@ -0,0 +1,47 @@
+<!doctype html>
+<html>
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <title> Login </title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="stylesheet" href="style.css">
+    <script src="js/jquery-3.4.1.min.js"></script>
+    <script src="js/login.js"></script>
+</head>
+<body onload="matrixLogin.onLoad()">
+    <div id="container">
+        <h1 id="title"></h1>
+
+        <span id="feedback"></span>
+
+        <div id="loading">
+            <img src="spinner.gif" />
+        </div>
+
+        <div id="sso_flow" class="login_flow" style="display: none;">
+            Single-sign on:
+            <form id="sso_form" action="/_matrix/client/v3/login/sso/redirect" method="get">
+                <input id="sso_redirect_url" type="hidden" name="redirectUrl" value=""/>
+                <input type="submit" value="Log in"/>
+            </form>
+        </div>
+
+        <div id="password_flow" class="login_flow" style="display: none;">
+            Password Authentication:
+            <form onsubmit="matrixLogin.passwordLogin(); return false;">
+                <input id="user_id" size="32" type="text" placeholder="Matrix ID (e.g. bob)" autocapitalize="off" autocorrect="off" />
+                <br/>
+                <input id="password" size="32" type="password" placeholder="Password"/>
+                <br/>
+
+                <input type="submit" value="Log in"/>
+            </form>
+        </div>
+
+        <div id="no_login_types" type="button" class="login_flow" style="display: none;">
+            Log in currently unavailable.
+        </div>
+    </div>
+</body>
+</html>

setup/base/static/client/login/js/login.js

@@ -0,0 +1,291 @@
+window.matrixLogin = {
+    endpoint: location.origin + "/_matrix/client/v3/login",
+    serverAcceptsPassword: false,
+    serverAcceptsSso: false,
+};
+
+// Titles get updated through the process to give users feedback.
+const TITLE_PRE_AUTH = "Log in with one of the following methods";
+const TITLE_POST_AUTH = "Logging in...";
+
+// The cookie used to store the original query parameters when using SSO.
+const COOKIE_KEY = "dendrite_login_fallback_qs";
+
+/*
+ * Submit a login request.
+ *
+ * type: The login type as a string (e.g. "m.login.foo").
+ * data: An object of data specific to the login type.
+ * extra: (Optional) An object to search for extra information to send with the
+ *     login request, e.g. device_id.
+ * callback: (Optional) Function to call on successful login.
+ */
+function submitLogin(type, data, extra, callback) {
+    console.log("Logging in with " + type);
+    setTitle(TITLE_POST_AUTH);
+
+    // Add the login type.
+    data.type = type;
+
+    // Add the device information, if it was provided.
+    if (extra.device_id) {
+        data.device_id = extra.device_id;
+    }
+    if (extra.initial_device_display_name) {
+        data.initial_device_display_name = extra.initial_device_display_name;
+    }
+
+    $.post(matrixLogin.endpoint, JSON.stringify(data), function(response) {
+        if (callback) {
+            callback();
+        }
+        matrixLogin.onLogin(response);
+    }).fail(errorFunc);
+}
+
+/*
+ * Display an error to the user and show the login form again.
+ */
+function errorFunc(err) {
+    // We want to show the error to the user rather than redirecting immediately to the
+    // SSO portal (if SSO is the only login option), so we inhibit the redirect.
+    showLogin(true);
+
+    if (err.responseJSON && err.responseJSON.error) {
+        setFeedbackString(err.responseJSON.error + " (" + err.responseJSON.errcode + ")");
+    }
+    else {
+        setFeedbackString("Request failed: " + err.status);
+    }
+}
+
+/*
+ * Display an error to the user.
+ */
+function setFeedbackString(text) {
+    $("#feedback").text(text);
+}
+
+/*
+ * (Maybe) Show the login forms.
+ *
+ * This actually does a few unrelated functions:
+ *
+ * * Configures the SSO redirect URL to come back to this page.
+ * * Configures and shows the SSO form, if the server supports SSO.
+ * * Otherwise, shows the password form.
+ */
+function showLogin(inhibitRedirect) {
+    setTitle(TITLE_PRE_AUTH);
+
+    // If inhibitRedirect is false, and SSO is the only supported login method,
+    // we can redirect straight to the SSO page.
+    if (matrixLogin.serverAcceptsSso) {
+        // Set the redirect to come back to this page, a login token will get
+        // added as a query parameter and handled after the redirect.
+        $("#sso_redirect_url").val(window.location.origin + window.location.pathname);
+
+        // Before submitting SSO, set the current query parameters into a cookie
+        // for retrieval later.
+        var qs = parseQsFromUrl();
+        setCookie(COOKIE_KEY, JSON.stringify(qs));
+
+        // If password is not supported and redirects are allowed, then submit
+        // the form (redirecting to the SSO provider).
+        if (!inhibitRedirect && !matrixLogin.serverAcceptsPassword) {
+            $("#sso_form").submit();
+            return;
+        }
+
+        // Otherwise, show the SSO form
+        $("#sso_flow").show();
+    }
+
+    if (matrixLogin.serverAcceptsPassword) {
+        $("#password_flow").show();
+    }
+
+    // If neither password or SSO are supported, show an error to the user.
+    if (!matrixLogin.serverAcceptsPassword && !matrixLogin.serverAcceptsSso) {
+        $("#no_login_types").show();
+    }
+
+    $("#loading").hide();
+}
+
+/*
+ * Hides the forms and shows a loading throbber.
+ */
+function showSpinner() {
+    $("#password_flow").hide();
+    $("#sso_flow").hide();
+    $("#no_login_types").hide();
+    $("#loading").show();
+}
+
+/*
+ * Helper to show the page's main title.
+ */
+function setTitle(title) {
+    $("#title").text(title);
+}
+
+/*
+ * Query the login endpoint for the homeserver's supported flows.
+ *
+ * This populates matrixLogin.serverAccepts* variables.
+ */
+function fetchLoginFlows(cb) {
+    $.get(matrixLogin.endpoint, function(response) {
+        for (var i = 0; i < response.flows.length; i++) {
+            var flow = response.flows[i];
+            if ("m.login.sso" === flow.type) {
+                matrixLogin.serverAcceptsSso = true;
+                console.log("Server accepts SSO");
+            }
+            if ("m.login.password" === flow.type) {
+                matrixLogin.serverAcceptsPassword = true;
+                console.log("Server accepts password");
+            }
+        }
+
+        cb();
+    }).fail(errorFunc);
+}
+
+/*
+ * Called on load to fetch login flows and attempt SSO login (if a token is available).
+ */
+matrixLogin.onLoad = function() {
+    fetchLoginFlows(function() {
+        // (Maybe) attempt logging in via SSO if a token is available.
+        if (!tryTokenLogin()) {
+            showLogin(false);
+        }
+    });
+};
+
+/*
+ * Submit simple user & password login.
+ */
+matrixLogin.passwordLogin = function() {
+    var user = $("#user_id").val();
+    var pwd = $("#password").val();
+
+    setFeedbackString("");
+
+    showSpinner();
+    submitLogin(
+        "m.login.password",
+        {user: user, password: pwd},
+        parseQsFromUrl());
+};
+
+/*
+ * The onLogin function gets called after a successful login.
+ *
+ * It is expected that implementations override this to be notified when the
+ * login is complete. The response to the login call is provided as the single
+ * parameter.
+ */
+matrixLogin.onLogin = function(response) {
+    // clobber this function
+    console.warn("onLogin - This function should be replaced to proceed.");
+};
+
+/*
+ * Process the query parameters from the current URL into an object.
+ */
+function parseQsFromUrl() {
+    var pos = window.location.href.indexOf("?");
+    if (pos == -1) {
+        return {};
+    }
+    var query = window.location.href.substr(pos + 1);
+
+    var result = {};
+    query.split("&").forEach(function(part) {
+        var item = part.split("=");
+        var key = item[0];
+        var val = item[1];
+
+        if (val) {
+            val = decodeURIComponent(val);
+        }
+        result[key] = val;
+    });
+    return result;
+}
+
+/*
+ * Process the cookies and return an object.
+ */
+function parseCookies() {
+    var allCookies = document.cookie;
+    var result = {};
+    allCookies.split(";").forEach(function(part) {
+        var item = part.split("=");
+        // Cookies might have arbitrary whitespace between them.
+        var key = item[0].trim();
+        // You can end up with a broken cookie that doesn't have an equals sign
+        // in it. Set to an empty value.
+        var val = (item[1] || "").trim();
+        // Values might be URI encoded.
+        if (val) {
+            val = decodeURIComponent(val);
+        }
+        result[key] = val;
+    });
+    return result;
+}
+
+/*
+ * Set a cookie that is valid for 1 hour.
+ */
+function setCookie(key, value) {
+    // The maximum age is set in seconds.
+    var maxAge = 60 * 60;
+    // Set the cookie, this defaults to the current domain and path.
+    document.cookie = key + "=" + encodeURIComponent(value) + ";max-age=" + maxAge + ";sameSite=lax";
+}
+
+/*
+ * Removes a cookie by key.
+ */
+function deleteCookie(key) {
+    // Delete a cookie by setting the expiration to 0. (Note that the value
+    // doesn't matter.)
+    document.cookie = key + "=deleted;expires=0";
+}
+
+/*
+ * Submits the login token if one is found in the query parameters. Returns a
+ * boolean of whether the login token was found or not.
+ */
+function tryTokenLogin() {
+    // Check if the login token is in the query parameters.
+    var qs = parseQsFromUrl();
+
+    var loginToken = qs.loginToken;
+    if (!loginToken) {
+        return false;
+    }
+
+    // Retrieve the original query parameters (from before the SSO redirect).
+    // They are stored as JSON in a cookie.
+    var cookies = parseCookies();
+    var originalQueryParams = JSON.parse(cookies[COOKIE_KEY] || "{}")
+
+    // If the login is successful, delete the cookie.
+    function callback() {
+        deleteCookie(COOKIE_KEY);
+    }
+
+    submitLogin(
+        "m.login.token",
+        {token: loginToken},
+        originalQueryParams,
+        callback);
+
+    return true;
+}

setup/base/static/client/login/style.css

@@ -0,0 +1,79 @@
+html {
+    height: 100%;
+}
+
+body {
+    height: 100%;
+    font-family: "Myriad Pro", "Myriad", Helvetica, Arial, sans-serif;
+    font-size: 12pt;
+    margin: 0px;
+}
+
+h1 {
+    font-size: 20pt;
+}
+
+a:link    { color: #666; }
+a:visited { color: #666; }
+a:hover   { color: #000; }
+a:active  { color: #000; }
+
+input {
+   margin: 5px;
+}
+
+textbox, input[type="text"], input[type="password"] {
+   width: 90%;
+}
+
+form {
+    text-align: center;
+    margin: 10px 0 0 0;
+}
+
+ul.radiobuttons {
+    text-align: left;
+    list-style: none;
+}
+
+/*
+ * Add some padding to the viewport.
+ */
+#container {
+    padding: 10px;
+}
+/*
+ * Center all direct children of the main form.
+ */
+#container > * {
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    text-align: center;
+}
+
+/*
+ * A wrapper around each login flow.
+ */
+.login_flow {
+    width: 300px;
+    text-align: left;
+    padding: 10px;
+    margin-bottom: 40px;
+
+    border-radius: 10px;
+    box-shadow: 0px 0px 20px 0px rgba(0,0,0,0.15);
+
+    background-color: #f8f8f8;
+    border: 1px #ccc solid;
+}
+
+/*
+ * Used to show error content.
+ */
+#feedback {
+    /* Red text. */
+    color: #ff0000;
+    /* A little space to not overlap the box-shadow. */
+    margin-bottom: 20px;
+}

syncapi/routing/memberships.go

@@ -15,7 +15,6 @@
 package routing
 
 import (
-	"encoding/json"
 	"math"
 	"net/http"
 
@@ -33,31 +32,13 @@ type getMembershipResponse struct {
 	Chunk []synctypes.ClientEvent `json:"chunk"`
 }
 
-// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
-type getJoinedMembersResponse struct {
-	Joined map[string]joinedMember `json:"joined"`
-}
-
-type joinedMember struct {
-	DisplayName string `json:"display_name"`
-	AvatarURL   string `json:"avatar_url"`
-}
-
-// The database stores 'displayname' without an underscore.
-// Deserialize into this and then change to the actual API response
-type databaseJoinedMember struct {
-	DisplayName string `json:"displayname"`
-	AvatarURL   string `json:"avatar_url"`
-}
-
 // GetMemberships implements
 //
 //	GET /rooms/{roomId}/members
-//	GET /rooms/{roomId}/joined_members
 func GetMemberships(
 	req *http.Request, device *userapi.Device, roomID string,
 	syncDB storage.Database, rsAPI api.SyncRoomserverAPI,
-	joinedOnly bool, membership, notMembership *string, at string,
+	membership, notMembership *string, at string,
 ) util.JSONResponse {
 	userID, err := spec.NewUserID(device.UserID, true)
 	if err != nil {
@@ -87,13 +68,6 @@ func GetMemberships(
 		}
 	}
 
-	if joinedOnly && !queryRes.IsInRoom {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
-		}
-	}
-
 	db, err := syncDB.NewDatabaseSnapshot(req.Context())
 	if err != nil {
 		return util.JSONResponse{
@@ -139,40 +113,6 @@ func GetMemberships(
 
 	result := qryRes.Events
 
-	if joinedOnly {
-		var res getJoinedMembersResponse
-		res.Joined = make(map[string]joinedMember)
-		for _, ev := range result {
-			var content databaseJoinedMember
-			if err := json.Unmarshal(ev.Content(), &content); err != nil {
-				util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
-				return util.JSONResponse{
-					Code: http.StatusInternalServerError,
-					JSON: spec.InternalServerError{},
-				}
-			}
-
-			userID, err := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID())
-			if err != nil || userID == nil {
-				util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
-				return util.JSONResponse{
-					Code: http.StatusInternalServerError,
-					JSON: spec.InternalServerError{},
-				}
-			}
-			if err != nil {
-				return util.JSONResponse{
-					Code: http.StatusForbidden,
-					JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"),
-				}
-			}
-			res.Joined[userID.String()] = joinedMember(content)
-		}
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: res,
-		}
-	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: getMembershipResponse{synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(result), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {

syncapi/routing/messages.go

@@ -135,13 +135,6 @@ func OnIncomingMessagesRequest(
 	var fromStream *types.StreamingToken
 	fromQuery := req.URL.Query().Get("from")
 	toQuery := req.URL.Query().Get("to")
-	emptyFromSupplied := fromQuery == ""
-	if emptyFromSupplied {
-		// NOTSPEC: We will pretend they used the latest sync token if no ?from= was provided.
-		// We do this to allow clients to get messages without having to call `/sync` e.g Cerulean
-		currPos := srp.Notifier.CurrentPosition()
-		fromQuery = currPos.String()
-	}
 
 	// Direction to return events from.
 	dir := req.URL.Query().Get("dir")
@@ -155,6 +148,23 @@ func OnIncomingMessagesRequest(
 	// to have one of the two accepted values (so dir == "f" <=> !backwardOrdering).
 	backwardOrdering := (dir == "b")
 
+	emptyFromSupplied := fromQuery == ""
+	if emptyFromSupplied {
+		// If "from" isn't provided, it defaults to either the earliest stream
+		// position (if we're going forward) or to the latest one (if we're
+		// going backward).
+
+		var from types.TopologyToken
+		if backwardOrdering {
+			from = types.TopologyToken{Depth: math.MaxInt64, PDUPosition: math.MaxInt64}
+		} else {
+			// go 1 earlier than the first event so we correctly fetch the earliest event
+			// this is because Database.GetEventsInTopologicalRange is exclusive of the lower-bound.
+			from = types.TopologyToken{}
+		}
+		fromQuery = from.String()
+	}
+
 	from, err := types.NewTopologyTokenFromString(fromQuery)
 	if err != nil {
 		var streamToken types.StreamingToken

syncapi/routing/routing.go

@@ -197,22 +197,10 @@ func Setup(
 			}
 
 			at := req.URL.Query().Get("at")
-			return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, false, membership, notMembership, at)
+			return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, membership, notMembership, at)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	v3mux.Handle("/rooms/{roomID}/joined_members",
-		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			at := req.URL.Query().Get("at")
-			membership := spec.Join
-			return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, true, &membership, nil, at)
-		}),
-	).Methods(http.MethodGet, http.MethodOptions)
-
 	v3mux.Handle("/rooms/{roomID}/location_sync",
 		httputil.MakeAuthAPI("location_sync", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))

syncapi/streams/stream_pdu.go

@@ -205,6 +205,12 @@ func (p *PDUStreamProvider) IncrementalSync(
 		req.Log.WithError(err).Error("unable to update event filter with ignored users")
 	}
 
+	dbEvents, err := p.getRecentEvents(ctx, stateDeltas, r, eventFilter, snapshot)
+	if err != nil {
+		req.Log.WithError(err).Error("unable to get recent events")
+		return r.From
+	}
+
 	newPos = from
 	for _, delta := range stateDeltas {
 		newRange := r
@@ -220,7 +226,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 			}
 		}
 		var pos types.StreamPosition
-		if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req); err != nil {
+		if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req, dbEvents); err != nil {
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
 			if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone {
 				return newPos
@@ -242,6 +248,66 @@ func (p *PDUStreamProvider) IncrementalSync(
 	return newPos
 }
 
+func (p *PDUStreamProvider) getRecentEvents(ctx context.Context, stateDeltas []types.StateDelta, r types.Range, eventFilter synctypes.RoomEventFilter, snapshot storage.DatabaseTransaction) (map[string]types.RecentEvents, error) {
+	var roomIDs []string
+	var newlyJoinedRoomIDs []string
+	for _, delta := range stateDeltas {
+		if delta.NewlyJoined {
+			newlyJoinedRoomIDs = append(newlyJoinedRoomIDs, delta.RoomID)
+		} else {
+			roomIDs = append(roomIDs, delta.RoomID)
+		}
+	}
+	dbEvents := make(map[string]types.RecentEvents)
+	if len(roomIDs) > 0 {
+		events, err := snapshot.RecentEvents(
+			ctx, roomIDs, r,
+			&eventFilter, true, true,
+		)
+		if err != nil {
+			if err != sql.ErrNoRows {
+				return nil, err
+			}
+		}
+		for k, v := range events {
+			dbEvents[k] = v
+		}
+	}
+	if len(newlyJoinedRoomIDs) > 0 {
+		// For rooms that were joined in this sync, try to fetch
+		// as much timeline events as allowed by the filter.
+
+		filter := eventFilter
+		// If we're going backwards, grep at least X events, this is mostly to satisfy Sytest
+		if eventFilter.Limit < recentEventBackwardsLimit {
+			filter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way
+			diff := r.From - r.To
+			if diff > 0 && diff < recentEventBackwardsLimit {
+				filter.Limit = int(diff)
+			}
+		}
+
+		events, err := snapshot.RecentEvents(
+			ctx, newlyJoinedRoomIDs, types.Range{
+				From:      r.To,
+				To:        0,
+				Backwards: true,
+			},
+			&filter, true, true,
+		)
+		if err != nil {
+			if err != sql.ErrNoRows {
+				return nil, err
+			}
+		}
+		for k, v := range events {
+			dbEvents[k] = v
+		}
+	}
+
+	return dbEvents, nil
+}
+
 // Limit the recent events to X when going backwards
 const recentEventBackwardsLimit = 100
 
@@ -255,29 +321,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	eventFilter *synctypes.RoomEventFilter,
 	stateFilter *synctypes.StateFilter,
 	req *types.SyncRequest,
+	dbEvents map[string]types.RecentEvents,
 ) (types.StreamPosition, error) {
 	var err error
-	originalLimit := eventFilter.Limit
-	// If we're going backwards, grep at least X events, this is mostly to satisfy Sytest
-	if r.Backwards && originalLimit < recentEventBackwardsLimit {
-		eventFilter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way
-		diff := r.From - r.To
-		if diff > 0 && diff < recentEventBackwardsLimit {
-			eventFilter.Limit = int(diff)
-		}
-	}
-
-	dbEvents, err := snapshot.RecentEvents(
-		ctx, []string{delta.RoomID}, r,
-		eventFilter, true, true,
-	)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return r.To, nil
-		}
-		return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err)
-	}
-
 	recentStreamEvents := dbEvents[delta.RoomID].Events
 	limited := dbEvents[delta.RoomID].Limited
 
@@ -339,9 +385,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		logrus.WithError(err).Error("unable to apply history visibility filter")
 	}
 
-	if r.Backwards && len(events) > originalLimit {
+	if r.Backwards && len(events) > eventFilter.Limit {
 		// We're going backwards and the events are ordered chronologically, so take the last `limit` events
-		events = events[len(events)-originalLimit:]
+		events = events[len(events)-eventFilter.Limit:]
 		limited = true
 	}
 

syncapi/syncapi_test.go

@@ -753,24 +753,6 @@ func TestGetMembership(t *testing.T) {
 			},
 			wantOK: false,
 		},
-		{
-			name: "/joined_members - Bob never joined",
-			request: func(t *testing.T, room *test.Room) *http.Request {
-				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
-					"access_token": bobDev.AccessToken,
-				}))
-			},
-			wantOK: false,
-		},
-		{
-			name: "/joined_members - Alice joined",
-			request: func(t *testing.T, room *test.Room) *http.Request {
-				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
-					"access_token": aliceDev.AccessToken,
-				}))
-			},
-			wantOK: true,
-		},
 		{
 			name: "Alice leaves before Bob joins, should not be able to see Bob",
 			request: func(t *testing.T, room *test.Room) *http.Request {
@@ -809,21 +791,6 @@ func TestGetMembership(t *testing.T) {
 			wantOK:          true,
 			wantMemberCount: 2,
 		},
-		{
-			name: "/joined_members - Alice leaves, shouldn't be able to see members ",
-			request: func(t *testing.T, room *test.Room) *http.Request {
-				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
-					"access_token": aliceDev.AccessToken,
-				}))
-			},
-			additionalEvents: func(t *testing.T, room *test.Room) {
-				room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
-					"membership": "leave",
-				}, test.WithStateKey(alice.ID))
-			},
-			useSleep: true,
-			wantOK:   false,
-		},
 		{
 			name: "'at' specified, returns memberships before Bob joins",
 			request: func(t *testing.T, room *test.Room) *http.Request {

userapi/api/api.go

@@ -388,6 +388,10 @@ type PerformDeviceCreationRequest struct {
 	// update for this account. Generally the only reason to do this is if the account
 	// is an appservice account.
 	NoDeviceListUpdate bool
+
+	// FromRegistration determines if this request comes from registering a new account
+	// and is in most cases false.
+	FromRegistration bool
 }
 
 // PerformDeviceCreationResponse is the response for PerformDeviceCreation
@@ -812,6 +816,10 @@ type PerformUploadKeysRequest struct {
 	// itself doesn't change but it's easier to pretend upload new keys and reuse the same code paths.
 	// Without this flag, requests to modify device display names would delete device keys.
 	OnlyDisplayNameUpdates bool
+
+	// FromRegistration is set if this key upload comes right after creating an account
+	// and determines if we need to inform downstream components.
+	FromRegistration bool
 }
 
 // PerformUploadKeysResponse is the response to PerformUploadKeys

userapi/internal/key_api.go

@@ -711,9 +711,15 @@ func (a *UserInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Pe
 		}
 		return
 	}
-	err = emitDeviceKeyChanges(a.KeyChangeProducer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates)
+
-	if err != nil {
+	// If the request does _not_ come right after registering an account
-		util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err)
+	// inform downstream components. However, we're fine with just creating the
+	// database entries above in other cases.
+	if !req.FromRegistration {
+		err = emitDeviceKeyChanges(a.KeyChangeProducer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates)
+		if err != nil {
+			util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err)
+		}
 	}
 }
 

userapi/internal/user_api.go

@@ -316,7 +316,7 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe
 		return nil
 	}
 	// create empty device keys and upload them to trigger device list changes
-	return a.deviceListUpdate(dev.UserID, []string{dev.ID})
+	return a.deviceListUpdate(dev.UserID, []string{dev.ID}, req.FromRegistration)
 }
 
 func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.PerformDeviceDeletionRequest, res *api.PerformDeviceDeletionResponse) error {
@@ -356,10 +356,10 @@ func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.Pe
 		return fmt.Errorf("a.KeyAPI.PerformDeleteKeys: %w", err)
 	}
 	// create empty device keys and upload them to delete what was once there and trigger device list changes
-	return a.deviceListUpdate(req.UserID, deletedDeviceIDs)
+	return a.deviceListUpdate(req.UserID, deletedDeviceIDs, false)
 }
 
-func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) error {
+func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string, fromRegistration bool) error {
 	deviceKeys := make([]api.DeviceKeys, len(deviceIDs))
 	for i, did := range deviceIDs {
 		deviceKeys[i] = api.DeviceKeys{
@@ -371,8 +371,9 @@ func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) er
 
 	var uploadRes api.PerformUploadKeysResponse
 	if err := a.PerformUploadKeys(context.Background(), &api.PerformUploadKeysRequest{
-		UserID:     userID,
+		UserID:           userID,
-		DeviceKeys: deviceKeys,
+		DeviceKeys:       deviceKeys,
+		FromRegistration: fromRegistration,
 	}, &uploadRes); err != nil {
 		return err
 	}