Merge branch 'main' into patch-1

.github/workflows/schedules.yaml

@@ -219,7 +219,7 @@ jobs:
           flags: complement
           fail_ci_if_error: true
 
-  element_web:
+  element-web:
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -257,3 +257,42 @@ jobs:
         env:
           PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
           TMPDIR: ${{ runner.temp }}
+
+  element-web-pinecone:
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tecolicom/actions-use-apt-tools@v1
+        with:
+          # Our test suite includes some screenshot tests with unusual diacritics, which are
+          # supposed to be covered by STIXGeneral.
+          tools: fonts-stix
+      - uses: actions/checkout@v2
+        with:
+          repository: matrix-org/matrix-react-sdk
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+      - name: Fetch layered build
+        run: scripts/ci/layered.sh
+      - name: Copy config
+        run: cp element.io/develop/config.json config.json
+        working-directory: ./element-web
+      - name: Build
+        env:
+          CI_PACKAGE: true
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        run: yarn build
+        working-directory: ./element-web
+      - name: Edit Test Config
+        run: |
+          sed -i '/HOMESERVER/c\        HOMESERVER: "dendritePinecone",' cypress.config.ts
+      - name: "Run cypress tests"
+        uses: cypress-io/github-action@v4.1.1
+        with:
+          browser: chrome
+          start: npx serve -p 8080 ./element-web/webapp
+          wait-on: 'http://localhost:8080'
+        env:
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          TMPDIR: ${{ runner.temp }}

appservice/appservice_test.go

@@ -16,10 +16,12 @@ import (
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/test/testrig"
 )
@@ -212,3 +214,86 @@ func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, w
 		t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
 	}
 }
+
+// Tests that the roomserver consumer only receives one invite
+func TestRoomserverConsumerOneInvite(t *testing.T) {
+
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	// Invite Bob
+	room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "invite",
+	}, test.WithStateKey(bob.ID))
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		natsInstance := &jetstream.NATSInstance{}
+
+		evChan := make(chan struct{})
+		// create a dummy AS url, handling the events
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var txn gomatrixserverlib.ApplicationServiceTransaction
+			err := json.NewDecoder(r.Body).Decode(&txn)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, ev := range txn.Events {
+				if ev.Type != gomatrixserverlib.MRoomMember {
+					continue
+				}
+				// Usually we would check the event content for the membership, but since
+				// we only invited bob, this should be fine for this test.
+				if ev.StateKey != nil && *ev.StateKey == bob.ID {
+					evChan <- struct{}{}
+				}
+			}
+		}))
+		defer srv.Close()
+
+		// Create a dummy application service
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
+			{
+				ID:              "someID",
+				URL:             srv.URL,
+				ASToken:         "",
+				HSToken:         "",
+				SenderLocalpart: "senderLocalPart",
+				NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+					"users":   {{RegexpObject: regexp.MustCompile(bob.ID)}},
+					"aliases": {{RegexpObject: regexp.MustCompile(room.ID)}},
+				},
+			},
+		}
+
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		// Create required internal APIs
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		// start the consumer
+		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
+
+		// Create the room
+		if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+		var seenInvitesForBob int
+	waitLoop:
+		for {
+			select {
+			case <-time.After(time.Millisecond * 50): // wait for the AS to process the events
+				break waitLoop
+			case <-evChan:
+				seenInvitesForBob++
+				if seenInvitesForBob != 1 {
+					t.Fatalf("received unexpected invites: %d", seenInvitesForBob)
+				}
+			}
+		}
+		close(evChan)
+	})
+}

appservice/consumers/roomserver.go

@@ -140,12 +140,6 @@ func (s *OutputRoomEventConsumer) onMessage(
 				}
 			}
 
-		case api.OutputTypeNewInviteEvent:
-			if output.NewInviteEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewInviteEvent.Event, state.ApplicationService) {
-				continue
-			}
-			events = append(events, output.NewInviteEvent.Event)
-
 		default:
 			continue
 		}

clientapi/admin_test.go

@@ -4,16 +4,17 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"testing"
 	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi"
@@ -52,39 +53,12 @@ func TestAdminResetPassword(t *testing.T) {
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
 		// Create the users in the userapi and login
-		accessTokens := map[*test.User]string{
+		accessTokens := map[*test.User]userDevice{
-			aliceAdmin: "",
+			aliceAdmin: {},
-			bob:        "",
+			bob:        {},
-			vhUser:     "",
+			vhUser:     {},
-		}
-		for u := range accessTokens {
-			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
-			userRes := &uapi.PerformAccountCreationResponse{}
-			password := util.RandomString(8)
-			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
-				AccountType: u.AccountType,
-				Localpart:   localpart,
-				ServerName:  serverName,
-				Password:    password,
-			}, userRes); err != nil {
-				t.Errorf("failed to create account: %s", err)
-			}
-
-			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
-				"type": authtypes.LoginTypePassword,
-				"identifier": map[string]interface{}{
-					"type": "m.id.user",
-					"user": u.ID,
-				},
-				"password": password,
-			}))
-			rec := httptest.NewRecorder()
-			routers.Client.ServeHTTP(rec, req)
-			if rec.Code != http.StatusOK {
-				t.Fatalf("failed to login: %s", rec.Body.String())
-			}
-			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
 		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
 
 		testCases := []struct {
 			name           string
@@ -128,7 +102,7 @@ func TestAdminResetPassword(t *testing.T) {
 				}
 
 				if tc.withHeader {
-					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser])
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken)
 				}
 
 				rec := httptest.NewRecorder()
@@ -179,37 +153,10 @@ func TestPurgeRoom(t *testing.T) {
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
 		// Create the users in the userapi and login
-		accessTokens := map[*test.User]string{
+		accessTokens := map[*test.User]userDevice{
-			aliceAdmin: "",
+			aliceAdmin: {},
-		}
-		for u := range accessTokens {
-			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
-			userRes := &uapi.PerformAccountCreationResponse{}
-			password := util.RandomString(8)
-			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
-				AccountType: u.AccountType,
-				Localpart:   localpart,
-				ServerName:  serverName,
-				Password:    password,
-			}, userRes); err != nil {
-				t.Errorf("failed to create account: %s", err)
-			}
-
-			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
-				"type": authtypes.LoginTypePassword,
-				"identifier": map[string]interface{}{
-					"type": "m.id.user",
-					"user": u.ID,
-				},
-				"password": password,
-			}))
-			rec := httptest.NewRecorder()
-			routers.Client.ServeHTTP(rec, req)
-			if rec.Code != http.StatusOK {
-				t.Fatalf("failed to login: %s", rec.Body.String())
-			}
-			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
 		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
 
 		testCases := []struct {
 			name   string
@@ -226,7 +173,7 @@ func TestPurgeRoom(t *testing.T) {
 			t.Run(tc.name, func(t *testing.T) {
 				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/purgeRoom/"+tc.roomID)
 
-				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin])
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
 
 				rec := httptest.NewRecorder()
 				routers.DendriteAdmin.ServeHTTP(rec, req)
@@ -239,3 +186,237 @@ func TestPurgeRoom(t *testing.T) {
 
 	})
 }
+
+func TestAdminEvacuateRoom(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, aliceAdmin)
+
+	// Join Bob
+	room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// this starts the JetStream consumers
+		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
+		rsAPI.SetFederationAPI(fsAPI, nil)
+
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+
+		testCases := []struct {
+			name         string
+			roomID       string
+			wantOK       bool
+			wantAffected []string
+		}{
+			{name: "Can evacuate existing room", wantOK: true, roomID: room.ID, wantAffected: []string{aliceAdmin.ID, bob.ID}},
+			{name: "Can not evacuate non-existent room", wantOK: false, roomID: "!doesnotexist:localhost", wantAffected: []string{}},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateRoom/"+tc.roomID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
+
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+
+				affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array()
+				affected := make([]string, 0, len(affectedArr))
+				for _, x := range affectedArr {
+					affected = append(affected, x.Str)
+				}
+				if !reflect.DeepEqual(affected, tc.wantAffected) {
+					t.Fatalf("expected affected %#v, but got %#v", tc.wantAffected, affected)
+				}
+			})
+		}
+	})
+}
+
+func TestAdminEvacuateUser(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, aliceAdmin)
+	room2 := test.NewRoom(t, aliceAdmin)
+
+	// Join Bob
+	room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+	room2.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// this starts the JetStream consumers
+		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true)
+		rsAPI.SetFederationAPI(fsAPI, nil)
+
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room2.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+
+		testCases := []struct {
+			name              string
+			userID            string
+			wantOK            bool
+			wantAffectedRooms []string
+		}{
+			{name: "Can evacuate existing user", wantOK: true, userID: bob.ID, wantAffectedRooms: []string{room.ID, room2.ID}},
+			{name: "invalid userID is rejected", wantOK: false, userID: "!notauserid:test", wantAffectedRooms: []string{}},
+			{name: "Can not evacuate user from different server", wantOK: false, userID: "@doesnotexist:localhost", wantAffectedRooms: []string{}},
+			{name: "Can not evacuate non-existent user", wantOK: false, userID: "@doesnotexist:test", wantAffectedRooms: []string{}},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateUser/"+tc.userID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
+
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+
+				affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array()
+				affected := make([]string, 0, len(affectedArr))
+				for _, x := range affectedArr {
+					affected = append(affected, x.Str)
+				}
+				if !reflect.DeepEqual(affected, tc.wantAffectedRooms) {
+					t.Fatalf("expected affected %#v, but got %#v", tc.wantAffectedRooms, affected)
+				}
+
+			})
+		}
+		// Wait for the FS API to have consumed every message
+		js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		timeout := time.After(time.Second)
+		for {
+			select {
+			case <-timeout:
+				t.Fatalf("FS API didn't process all events in time")
+			default:
+			}
+			info, err := js.ConsumerInfo(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), cfg.Global.JetStream.Durable("FederationAPIRoomServerConsumer")+"Pull")
+			if err != nil {
+				time.Sleep(time.Millisecond * 10)
+				continue
+			}
+			if info.NumPending == 0 && info.NumAckPending == 0 {
+				break
+			}
+		}
+	})
+}
+
+func TestAdminMarkAsStale(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+
+		testCases := []struct {
+			name   string
+			userID string
+			wantOK bool
+		}{
+			{name: "local user is not allowed", userID: aliceAdmin.ID},
+			{name: "invalid userID", userID: "!notvalid:test"},
+			{name: "remote user is allowed", userID: "@alice:localhost", wantOK: true},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/refreshDevices/"+tc.userID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
+
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}

clientapi/clientapi_test.go

@@ -0,0 +1,373 @@
+package clientapi
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/tidwall/gjson"
+)
+
+type userDevice struct {
+	accessToken string
+	deviceID    string
+	password    string
+}
+
+func TestGetPutDevices(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		testCases := []struct {
+			name           string
+			requestUser    *test.User
+			deviceUser     *test.User
+			request        *http.Request
+			wantStatusCode int
+			validateFunc   func(t *testing.T, device userDevice, routers httputil.Routers)
+		}{
+			{
+				name:           "can get all devices",
+				requestUser:    alice,
+				request:        httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices", strings.NewReader("")),
+				wantStatusCode: http.StatusOK,
+			},
+			{
+				name:           "can get specific own device",
+				requestUser:    alice,
+				deviceUser:     alice,
+				request:        httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices/", strings.NewReader("")),
+				wantStatusCode: http.StatusOK,
+			},
+			{
+				name:           "can not get device for different user",
+				requestUser:    alice,
+				deviceUser:     bob,
+				request:        httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices/", strings.NewReader("")),
+				wantStatusCode: http.StatusNotFound,
+			},
+			{
+				name:           "can update own device",
+				requestUser:    alice,
+				deviceUser:     alice,
+				request:        httptest.NewRequest(http.MethodPut, "/_matrix/client/v3/devices/", strings.NewReader(`{"display_name":"my new displayname"}`)),
+				wantStatusCode: http.StatusOK,
+				validateFunc: func(t *testing.T, device userDevice, routers httputil.Routers) {
+					req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices/"+device.deviceID, strings.NewReader(""))
+					req.Header.Set("Authorization", "Bearer "+device.accessToken)
+					rec := httptest.NewRecorder()
+					routers.Client.ServeHTTP(rec, req)
+					if rec.Code != http.StatusOK {
+						t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
+					}
+					gotDisplayName := gjson.GetBytes(rec.Body.Bytes(), "display_name").Str
+					if gotDisplayName != "my new displayname" {
+						t.Fatalf("expected displayname '%s', got '%s'", "my new displayname", gotDisplayName)
+					}
+				},
+			},
+			{
+				// this should return "device does not exist"
+				name:           "can not update device for different user",
+				requestUser:    alice,
+				deviceUser:     bob,
+				request:        httptest.NewRequest(http.MethodPut, "/_matrix/client/v3/devices/", strings.NewReader(`{"display_name":"my new displayname"}`)),
+				wantStatusCode: http.StatusNotFound,
+			},
+		}
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				dev := accessTokens[tc.requestUser]
+				if tc.deviceUser != nil {
+					tc.request = httptest.NewRequest(tc.request.Method, tc.request.RequestURI+accessTokens[tc.deviceUser].deviceID, tc.request.Body)
+				}
+				tc.request.Header.Set("Authorization", "Bearer "+dev.accessToken)
+				rec := httptest.NewRecorder()
+				routers.Client.ServeHTTP(rec, tc.request)
+				if rec.Code != tc.wantStatusCode {
+					t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
+				}
+				if tc.wantStatusCode != http.StatusOK && rec.Code != http.StatusOK {
+					return
+				}
+				if tc.validateFunc != nil {
+					tc.validateFunc(t, dev, routers)
+				}
+			})
+		}
+	})
+}
+
+// Deleting devices requires the UIA dance, so do this in a different test
+func TestDeleteDevice(t *testing.T) {
+	alice := test.NewUser(t)
+	localpart, serverName, _ := gomatrixserverlib.SplitID('@', alice.ID)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+
+		natsInstance := jetstream.NATSInstance{}
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+		}
+
+		// create the account and an initial device
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		// create some more devices
+		accessToken := util.RandomString(8)
+		devRes := &uapi.PerformDeviceCreationResponse{}
+		if err := userAPI.PerformDeviceCreation(processCtx.Context(), &uapi.PerformDeviceCreationRequest{
+			Localpart:          localpart,
+			ServerName:         serverName,
+			AccessToken:        accessToken,
+			NoDeviceListUpdate: true,
+		}, devRes); err != nil {
+			t.Fatal(err)
+		}
+		if !devRes.DeviceCreated {
+			t.Fatalf("failed to create device")
+		}
+		secondDeviceID := devRes.Device.ID
+
+		// initiate UIA for the second device
+		rec := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodDelete, "/_matrix/client/v3/devices/"+secondDeviceID, strings.NewReader(""))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusUnauthorized {
+			t.Fatalf("expected HTTP 401, got %d: %s", rec.Code, rec.Body.String())
+		}
+		// get the session ID
+		sessionID := gjson.GetBytes(rec.Body.Bytes(), "session").Str
+
+		// prepare UIA request body
+		reqBody := bytes.Buffer{}
+		if err := json.NewEncoder(&reqBody).Encode(map[string]interface{}{
+			"auth": map[string]string{
+				"session":  sessionID,
+				"type":     authtypes.LoginTypePassword,
+				"user":     alice.ID,
+				"password": accessTokens[alice].password,
+			},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// copy the request body, so we can use it again for the successful delete
+		reqBody2 := reqBody
+
+		// do the same request again, this time with our UIA, but for a different device ID, this should fail
+		rec = httptest.NewRecorder()
+
+		req = httptest.NewRequest(http.MethodDelete, "/_matrix/client/v3/devices/"+accessTokens[alice].deviceID, &reqBody)
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusForbidden {
+			t.Fatalf("expected HTTP 403, got %d: %s", rec.Code, rec.Body.String())
+		}
+
+		// do the same request again, this time with our UIA, but for the correct device ID, this should be fine
+		rec = httptest.NewRecorder()
+		req = httptest.NewRequest(http.MethodDelete, "/_matrix/client/v3/devices/"+secondDeviceID, &reqBody2)
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
+		}
+
+		// verify devices are deleted
+		rec = httptest.NewRecorder()
+		req = httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices", strings.NewReader(""))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
+		}
+		for _, device := range gjson.GetBytes(rec.Body.Bytes(), "devices.#.device_id").Array() {
+			if device.Str == secondDeviceID {
+				t.Fatalf("expected device %s to be deleted, but wasn't", secondDeviceID)
+			}
+		}
+	})
+}
+
+// Deleting devices requires the UIA dance, so do this in a different test
+func TestDeleteDevices(t *testing.T) {
+	alice := test.NewUser(t)
+	localpart, serverName, _ := gomatrixserverlib.SplitID('@', alice.ID)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+
+		natsInstance := jetstream.NATSInstance{}
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+		}
+
+		// create the account and an initial device
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		// create some more devices
+		var devices []string
+		for i := 0; i < 10; i++ {
+			accessToken := util.RandomString(8)
+			devRes := &uapi.PerformDeviceCreationResponse{}
+			if err := userAPI.PerformDeviceCreation(processCtx.Context(), &uapi.PerformDeviceCreationRequest{
+				Localpart:          localpart,
+				ServerName:         serverName,
+				AccessToken:        accessToken,
+				NoDeviceListUpdate: true,
+			}, devRes); err != nil {
+				t.Fatal(err)
+			}
+			if !devRes.DeviceCreated {
+				t.Fatalf("failed to create device")
+			}
+			devices = append(devices, devRes.Device.ID)
+		}
+
+		// initiate UIA
+		rec := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/delete_devices", strings.NewReader(""))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusUnauthorized {
+			t.Fatalf("expected HTTP 401, got %d: %s", rec.Code, rec.Body.String())
+		}
+		// get the session ID
+		sessionID := gjson.GetBytes(rec.Body.Bytes(), "session").Str
+
+		// prepare UIA request body
+		reqBody := bytes.Buffer{}
+		if err := json.NewEncoder(&reqBody).Encode(map[string]interface{}{
+			"auth": map[string]string{
+				"session":  sessionID,
+				"type":     authtypes.LoginTypePassword,
+				"user":     alice.ID,
+				"password": accessTokens[alice].password,
+			},
+			"devices": devices[5:],
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// do the same request again, this time with our UIA,
+		rec = httptest.NewRecorder()
+		req = httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/delete_devices", &reqBody)
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
+		}
+
+		// verify devices are deleted
+		rec = httptest.NewRecorder()
+		req = httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices", strings.NewReader(""))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
+		}
+		for _, device := range gjson.GetBytes(rec.Body.Bytes(), "devices.#.device_id").Array() {
+			for _, deletedDevice := range devices[5:] {
+				if device.Str == deletedDevice {
+					t.Fatalf("expected device %s to be deleted, but wasn't", deletedDevice)
+				}
+			}
+		}
+	})
+}
+
+func createAccessTokens(t *testing.T, accessTokens map[*test.User]userDevice, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) {
+	t.Helper()
+	for u := range accessTokens {
+		localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+		userRes := &uapi.PerformAccountCreationResponse{}
+		password := util.RandomString(8)
+		if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+			AccountType: u.AccountType,
+			Localpart:   localpart,
+			ServerName:  serverName,
+			Password:    password,
+		}, userRes); err != nil {
+			t.Errorf("failed to create account: %s", err)
+		}
+		req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+			"type": authtypes.LoginTypePassword,
+			"identifier": map[string]interface{}{
+				"type": "m.id.user",
+				"user": u.ID,
+			},
+			"password": password,
+		}))
+		rec := httptest.NewRecorder()
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("failed to login: %s", rec.Body.String())
+		}
+		accessTokens[u] = userDevice{
+			accessToken: gjson.GetBytes(rec.Body.Bytes(), "access_token").String(),
+			deviceID:    gjson.GetBytes(rec.Body.Bytes(), "device_id").String(),
+			password:    password,
+		}
+	}
+}

clientapi/routing/admin.go

@@ -22,23 +22,16 @@ import (
 	"github.com/matrix-org/dendrite/userapi/api"
 )
 
-func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	roomID, ok := vars["roomID"]
-	if !ok {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting room ID."),
-		}
-	}
 	res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
 	if err := rsAPI.PerformAdminEvacuateRoom(
 		req.Context(),
 		&roomserverAPI.PerformAdminEvacuateRoomRequest{
-			RoomID: roomID,
+			RoomID: vars["roomID"],
 		},
 		res,
 	); err != nil {
@@ -55,18 +48,13 @@ func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Dev
 	}
 }
 
-func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	userID, ok := vars["userID"]
+	userID := vars["userID"]
-	if !ok {
+
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting user ID."),
-		}
-	}
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		return util.MessageResponse(http.StatusBadRequest, err.Error())
@@ -103,13 +91,8 @@ func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	roomID, ok := vars["roomID"]
+	roomID := vars["roomID"]
-	if !ok {
+
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting room ID."),
-		}
-	}
 	res := &roomserverAPI.PerformAdminPurgeRoomResponse{}
 	if err := rsAPI.PerformAdminPurgeRoom(
 		context.Background(),

clientapi/routing/deactivate.go

@@ -33,7 +33,7 @@ func Deactivate(
 		return *errRes
 	}
 
-	localpart, _, err := gomatrixserverlib.SplitID('@', login.Username())
+	localpart, serverName, err := gomatrixserverlib.SplitID('@', login.Username())
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
 		return jsonerror.InternalServerError()
@@ -41,7 +41,8 @@ func Deactivate(
 
 	var res api.PerformAccountDeactivationResponse
 	err = accountAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{
-		Localpart: localpart,
+		Localpart:  localpart,
+		ServerName: serverName,
 	}, &res)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformAccountDeactivation failed")

clientapi/routing/device.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"encoding/json"
 	"io"
 	"net"
 	"net/http"
@@ -146,12 +147,6 @@ func UpdateDeviceByID(
 			JSON: jsonerror.Forbidden("device does not exist"),
 		}
 	}
-	if performRes.Forbidden {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("device not owned by current user"),
-		}
-	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -189,7 +184,7 @@ func DeleteDeviceById(
 		if dev != deviceID {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("session & device mismatch"),
+				JSON: jsonerror.Forbidden("session and device mismatch"),
 			}
 		}
 	}
@@ -242,16 +237,37 @@ func DeleteDeviceById(
 
 // DeleteDevices handles POST requests to /delete_devices
 func DeleteDevices(
-	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
+	req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.ClientUserAPI, device *api.Device,
 ) util.JSONResponse {
 	ctx := req.Context()
-	payload := devicesDeleteJSON{}
 
-	if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil {
+	bodyBytes, err := io.ReadAll(req.Body)
-		return *resErr
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
+		}
+	}
+	defer req.Body.Close() // nolint:errcheck
+
+	// initiate UIA
+	login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, device)
+	if errRes != nil {
+		return *errRes
 	}
 
-	defer req.Body.Close() // nolint: errcheck
+	if login.Username() != device.UserID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("unable to delete devices for other user"),
+		}
+	}
+
+	payload := devicesDeleteJSON{}
+	if err = json.Unmarshal(bodyBytes, &payload); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("unable to unmarshal device deletion request")
+		return jsonerror.InternalServerError()
+	}
 
 	var res api.PerformDeviceDeletionResponse
 	if err := userAPI.PerformDeviceDeletion(ctx, &api.PerformDeviceDeletionRequest{

clientapi/routing/routing.go

@@ -155,15 +155,15 @@ func Setup(
 
 	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
 		httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateRoom(req, cfg, device, rsAPI)
+			return AdminEvacuateRoom(req, rsAPI)
 		}),
-	).Methods(http.MethodGet, http.MethodOptions)
+	).Methods(http.MethodPost, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}",
 		httputil.MakeAdminAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateUser(req, cfg, device, rsAPI)
+			return AdminEvacuateUser(req, cfg, rsAPI)
 		}),
-	).Methods(http.MethodGet, http.MethodOptions)
+	).Methods(http.MethodPost, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/purgeRoom/{roomID}",
 		httputil.MakeAdminAPI("admin_purge_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -1115,7 +1115,7 @@ func Setup(
 
 	v3mux.Handle("/delete_devices",
 		httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return DeleteDevices(req, userAPI, device)
+			return DeleteDevices(req, userInteractiveAuth, userAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 

cmd/dendrite-demo-pinecone/monolith/monolith.go

@@ -213,7 +213,7 @@ func (p *P2PMonolith) Stop() {
 }
 
 func (p *P2PMonolith) WaitForShutdown() {
-	p.ProcessCtx.WaitForShutdown()
+	base.WaitForShutdown(p.ProcessCtx)
 	p.closeAllResources()
 }
 

docs/administration/4_adminapi.md

@@ -32,7 +32,7 @@ UPDATE userapi_accounts SET account_type = 3 WHERE localpart = '$localpart';
 
 Where `$localpart` is the username only (e.g. `alice`).
 
-## GET `/_dendrite/admin/evacuateRoom/{roomID}`
+## POST `/_dendrite/admin/evacuateRoom/{roomID}`
 
 This endpoint will instruct Dendrite to part all local users from the given `roomID`
 in the URL. It may take some time to complete. A JSON body will be returned containing
@@ -41,7 +41,7 @@ the user IDs of all affected users.
 If the room has an alias set (e.g. is published), the room's ID will not be visible in the URL, but it can
 be found as the room's "internal ID" in Element Web (Settings -> Advanced)
 
-## GET `/_dendrite/admin/evacuateUser/{userID}`
+## POST `/_dendrite/admin/evacuateUser/{userID}`
 
 This endpoint will instruct Dendrite to part the given local `userID` in the URL from
 all rooms which they are currently joined. A JSON body will be returned containing

helm/dendrite/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: dendrite
-version: "0.12.1"
+version: "0.12.2"
 appVersion: "0.12.0"
 description: Dendrite Matrix Homeserver
 type: application

helm/dendrite/templates/_helpers.tpl

@@ -1,15 +1,9 @@
 {{- define "validate.config" }}
-{{- if not .Values.signing_key.create -}}
+{{- if and (not .Values.signing_key.create) (eq .Values.signing_key.existingSecret "") -}}
-{{-  fail "You must create a signing key for configuration.signing_key. (see https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#server-key-generation)" -}}
+{{-  fail "You must create a signing key for configuration.signing_key OR specify an existing secret name in .Values.signing_key.existingSecret to mount it. (see https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#server-key-generation)" -}}
 {{- end -}}
-{{- if not (or .Values.dendrite_config.global.database.host .Values.postgresql.enabled) -}}
+{{- if and (not .Values.postgresql.enabled) (eq .Values.dendrite_config.global.database.connection_string "") -}}
-{{-  fail "Database server must be set." -}}
+{{-  fail "Database connection string must be set." -}}
-{{- end -}}
-{{- if not (or .Values.dendrite_config.global.database.user .Values.postgresql.enabled) -}}
-{{-  fail "Database user must be set." -}}
-{{- end -}}
-{{- if not (or .Values.dendrite_config.global.database.password .Values.postgresql.enabled) -}}
-{{-  fail "Database password must be set." -}}
 {{- end -}}
 {{- end -}}
 

helm/dendrite/templates/deployment.yaml

@@ -12,16 +12,19 @@ spec:
     matchLabels:
       {{- include "dendrite.selectorLabels" . | nindent 6 }}
   replicas: 1
+  strategy:
+    type: {{ $.Values.strategy.type }}
+    {{- if eq $.Values.strategy.type "RollingUpdate" }}
+    rollingUpdate:
+      maxSurge: {{ $.Values.strategy.rollingUpdate.maxSurge }}
+      maxUnavailable: {{ $.Values.strategy.rollingUpdate.maxUnavailable }}
+    {{- end }}
   template:
     metadata:
       labels:
         {{- include "dendrite.selectorLabels" . | nindent 8 }}
       annotations:
-        confighash-global: secret-{{ .Values.global | toYaml | sha256sum | trunc 32 }}
+        confighash: secret-{{ .Values.dendrite_config | toYaml | sha256sum | trunc 32 }}
-        confighash-clientapi: clientapi-{{ .Values.clientapi | toYaml | sha256sum | trunc 32 }}
-        confighash-federationapi: federationapi-{{ .Values.federationapi | toYaml | sha256sum | trunc 32 }}
-        confighash-mediaapi: mediaapi-{{ .Values.mediaapi | toYaml | sha256sum | trunc 32 }}
-        confighash-syncapi: syncapi-{{ .Values.syncapi | toYaml | sha256sum | trunc 32 }}
     spec:
       volumes:
       - name: {{ include "dendrite.fullname" . }}-conf-vol
@@ -44,6 +47,9 @@ spec:
       - name: {{ include "dendrite.fullname" . }}-search
         persistentVolumeClaim:
           claimName: {{ default (print ( include "dendrite.fullname" . ) "-search-pvc") $.Values.persistence.search.existingClaim | quote }}
+      {{- with .Values.extraVolumes }}
+      {{ . | toYaml | nindent 6 }}
+      {{- end }}
       containers:
       - name: {{ .Chart.Name }}
         {{- include "image.name" . | nindent 8 }}
@@ -57,7 +63,7 @@ spec:
         {{- if $.Values.dendrite_config.global.profiling.enabled }}
         env:
           - name: PPROFLISTEN
-            value: "localhost:{{- $.Values.global.profiling.port -}}"
+            value: "localhost:{{- $.Values.dendrite_config.global.profiling.port -}}"
         {{- end }}
         resources:
         {{- toYaml $.Values.resources | nindent 10 }}
@@ -77,6 +83,9 @@ spec:
           name: {{ include "dendrite.fullname" . }}-jetstream
         - mountPath: {{ .Values.dendrite_config.sync_api.search.index_path }}
           name: {{ include "dendrite.fullname" . }}-search
+        {{- with .Values.extraVolumeMounts }}
+        {{ . | toYaml | nindent 8 }}
+        {{- end }}
         livenessProbe:
           initialDelaySeconds: 10
           periodSeconds: 10

helm/dendrite/values.yaml

@@ -43,6 +43,30 @@ persistence:
     # -- PVC Storage Request for the search volume
     capacity: "1Gi"
 
+# Add additional volumes to the Dendrite Pod
+extraVolumes: []
+# ex.
+# - name: extra-config
+#   secret:
+#     secretName: extra-config
+
+
+# Configure additional mount points volumes in the Dendrite Pod
+extraVolumeMounts: []
+# ex.
+# - mountPath: /etc/dendrite/extra-config
+#   name: extra-config
+
+strategy:
+  # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate)
+  # If you are using ReadWriteOnce volumes, you should probably use Recreate
+  type: RollingUpdate
+  rollingUpdate:
+    # -- Maximum number of pods that can be unavailable during the update process
+    maxUnavailable: 25%
+    # -- Maximum number of pods that can be scheduled above the desired number of pods
+    maxSurge: 25%
+
 dendrite_config:
   version: 2
   global:

internal/fulltext/bleve.go

@@ -18,6 +18,7 @@
 package fulltext
 
 import (
+	"regexp"
 	"strings"
 
 	"github.com/blevesearch/bleve/v2"
@@ -60,6 +61,7 @@ type Indexer interface {
 	Index(elements ...IndexElement) error
 	Delete(eventID string) error
 	Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (*bleve.SearchResult, error)
+	GetHighlights(result *bleve.SearchResult) []string
 	Close() error
 }
 
@@ -124,6 +126,47 @@ func (f *Search) Delete(eventID string) error {
 	return f.FulltextIndex.Delete(eventID)
 }
 
+var highlightMatcher = regexp.MustCompile("<mark>(.*?)</mark>")
+
+// GetHighlights extracts the highlights from a SearchResult.
+func (f *Search) GetHighlights(result *bleve.SearchResult) []string {
+	if result == nil {
+		return []string{}
+	}
+
+	seenMatches := make(map[string]struct{})
+
+	for _, hit := range result.Hits {
+		if hit.Fragments == nil {
+			continue
+		}
+		fragments, ok := hit.Fragments["Content"]
+		if !ok {
+			continue
+		}
+		for _, x := range fragments {
+			substringMatches := highlightMatcher.FindAllStringSubmatch(x, -1)
+			for _, matches := range substringMatches {
+				for i := range matches {
+					if i == 0 { // skip first match, this is the complete substring match
+						continue
+					}
+					if _, ok := seenMatches[matches[i]]; ok {
+						continue
+					}
+					seenMatches[matches[i]] = struct{}{}
+				}
+			}
+		}
+	}
+
+	res := make([]string, 0, len(seenMatches))
+	for m := range seenMatches {
+		res = append(res, m)
+	}
+	return res
+}
+
 // Search searches the index given a search term, roomIDs and keys.
 func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (*bleve.SearchResult, error) {
 	qry := bleve.NewConjunctionQuery()
@@ -163,6 +206,10 @@ func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, or
 		s.SortBy([]string{"-StreamPosition"})
 	}
 
+	// Highlight some words
+	s.Highlight = bleve.NewHighlight()
+	s.Highlight.Fields = []string{"Content"}
+
 	return f.FulltextIndex.Search(s)
 }
 

internal/fulltext/bleve_test.go

@@ -160,14 +160,16 @@ func TestSearch(t *testing.T) {
 		roomIndex        []int
 	}
 	tests := []struct {
-		name      string
+		name           string
-		args      args
+		args           args
-		wantCount int
+		wantCount      int
-		wantErr   bool
+		wantErr        bool
+		wantHighlights []string
 	}{
 		{
-			name:      "Can search for many results in one room",
+			name:           "Can search for many results in one room",
-			wantCount: 16,
+			wantCount:      16,
+			wantHighlights: []string{"lorem"},
 			args: args{
 				term:      "lorem",
 				roomIndex: []int{0},
@@ -175,8 +177,9 @@ func TestSearch(t *testing.T) {
 			},
 		},
 		{
-			name:      "Can search for one result in one room",
+			name:           "Can search for one result in one room",
-			wantCount: 1,
+			wantCount:      1,
+			wantHighlights: []string{"lorem"},
 			args: args{
 				term:      "lorem",
 				roomIndex: []int{16},
@@ -184,8 +187,9 @@ func TestSearch(t *testing.T) {
 			},
 		},
 		{
-			name:      "Can search for many results in multiple rooms",
+			name:           "Can search for many results in multiple rooms",
-			wantCount: 17,
+			wantCount:      17,
+			wantHighlights: []string{"lorem"},
 			args: args{
 				term:      "lorem",
 				roomIndex: []int{0, 16},
@@ -193,8 +197,9 @@ func TestSearch(t *testing.T) {
 			},
 		},
 		{
-			name:      "Can search for many results in all rooms, reversed",
+			name:           "Can search for many results in all rooms, reversed",
-			wantCount: 30,
+			wantCount:      30,
+			wantHighlights: []string{"lorem"},
 			args: args{
 				term:             "lorem",
 				limit:            30,
@@ -202,8 +207,9 @@ func TestSearch(t *testing.T) {
 			},
 		},
 		{
-			name:      "Can search for specific search room name",
+			name:           "Can search for specific search room name",
-			wantCount: 1,
+			wantCount:      1,
+			wantHighlights: []string{"testing"},
 			args: args{
 				term:      "testing",
 				roomIndex: []int{},
@@ -212,8 +218,9 @@ func TestSearch(t *testing.T) {
 			},
 		},
 		{
-			name:      "Can search for specific search room topic",
+			name:           "Can search for specific search room topic",
-			wantCount: 1,
+			wantCount:      1,
+			wantHighlights: []string{"fulltext"},
 			args: args{
 				term:      "fulltext",
 				roomIndex: []int{},
@@ -222,6 +229,7 @@ func TestSearch(t *testing.T) {
 			},
 		},
 	}
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			f, ctx := mustOpenIndex(t, "")
@@ -238,6 +246,12 @@ func TestSearch(t *testing.T) {
 				t.Errorf("Search() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
+
+			highlights := f.GetHighlights(got)
+			if !reflect.DeepEqual(highlights, tt.wantHighlights) {
+				t.Errorf("Search() got highligts = %v, want %v", highlights, tt.wantHighlights)
+			}
+
 			if !reflect.DeepEqual(len(got.Hits), tt.wantCount) {
 				t.Errorf("Search() got = %v, want %v", len(got.Hits), tt.wantCount)
 			}

internal/fulltext/bleve_wasm.go

@@ -33,6 +33,7 @@ type Indexer interface {
 	Index(elements ...IndexElement) error
 	Delete(eventID string) error
 	Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (SearchResult, error)
+	GetHighlights(result SearchResult) []string
 	Close() error
 }
 
@@ -71,3 +72,7 @@ func (f *Search) Delete(eventID string) error {
 func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (SearchResult, error) {
 	return SearchResult{}, nil
 }
+
+func (f *Search) GetHighlights(result SearchResult) []string {
+	return []string{}
+}

roomserver/internal/perform/perform_admin.go

@@ -227,6 +227,7 @@ func (r *Admin) PerformAdminEvacuateUser(
 			}
 			return nil
 		}
+		res.Affected = append(res.Affected, roomID)
 		if len(outputEvents) == 0 {
 			continue
 		}
@@ -237,8 +238,6 @@ func (r *Admin) PerformAdminEvacuateUser(
 			}
 			return nil
 		}
-
-		res.Affected = append(res.Affected, roomID)
 	}
 	return nil
 }

syncapi/routing/search.go

@@ -19,7 +19,6 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/blevesearch/bleve/v2/search"
@@ -123,8 +122,8 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: SearchResponse{
-				SearchCategories: SearchCategories{
+				SearchCategories: SearchCategoriesResponse{
-					RoomEvents: RoomEvents{
+					RoomEvents: RoomEventsResponse{
 						Count:     int(result.Total),
 						NextBatch: nil,
 					},
@@ -158,7 +157,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 	}
 
 	groups := make(map[string]RoomResult)
-	knownUsersProfiles := make(map[string]ProfileInfo)
+	knownUsersProfiles := make(map[string]ProfileInfoResponse)
 
 	// Sort the events by depth, as the returned values aren't ordered
 	if orderByTime {
@@ -180,7 +179,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 			return jsonerror.InternalServerError()
 		}
 
-		profileInfos := make(map[string]ProfileInfo)
+		profileInfos := make(map[string]ProfileInfoResponse)
 		for _, ev := range append(eventsBefore, eventsAfter...) {
 			profile, ok := knownUsersProfiles[event.Sender()]
 			if !ok {
@@ -192,7 +191,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 				if stateEvent == nil {
 					continue
 				}
-				profile = ProfileInfo{
+				profile = ProfileInfoResponse{
 					AvatarURL:   gjson.GetBytes(stateEvent.Content(), "avatar_url").Str,
 					DisplayName: gjson.GetBytes(stateEvent.Content(), "displayname").Str,
 				}
@@ -237,13 +236,13 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 	}
 
 	res := SearchResponse{
-		SearchCategories: SearchCategories{
+		SearchCategories: SearchCategoriesResponse{
-			RoomEvents: RoomEvents{
+			RoomEvents: RoomEventsResponse{
 				Count:      int(result.Total),
 				Groups:     Groups{RoomID: groups},
 				Results:    results,
 				NextBatch:  nextBatchResult,
-				Highlights: strings.Split(searchReq.SearchCategories.RoomEvents.SearchTerm, " "),
+				Highlights: fts.GetHighlights(result),
 				State:      stateForRooms,
 			},
 		},
@@ -286,30 +285,40 @@ func contextEvents(
 	return eventsBefore, eventsAfter, err
 }
 
+type EventContext struct {
+	AfterLimit     int  `json:"after_limit,omitempty"`
+	BeforeLimit    int  `json:"before_limit,omitempty"`
+	IncludeProfile bool `json:"include_profile,omitempty"`
+}
+
+type GroupBy struct {
+	Key string `json:"key"`
+}
+
+type Groupings struct {
+	GroupBy []GroupBy `json:"group_by"`
+}
+
+type RoomEvents struct {
+	EventContext EventContext                      `json:"event_context"`
+	Filter       gomatrixserverlib.RoomEventFilter `json:"filter"`
+	Groupings    Groupings                         `json:"groupings"`
+	IncludeState bool                              `json:"include_state"`
+	Keys         []string                          `json:"keys"`
+	OrderBy      string                            `json:"order_by"`
+	SearchTerm   string                            `json:"search_term"`
+}
+
+type SearchCategories struct {
+	RoomEvents RoomEvents `json:"room_events"`
+}
+
 type SearchRequest struct {
-	SearchCategories struct {
+	SearchCategories SearchCategories `json:"search_categories"`
-		RoomEvents struct {
-			EventContext struct {
-				AfterLimit     int  `json:"after_limit,omitempty"`
-				BeforeLimit    int  `json:"before_limit,omitempty"`
-				IncludeProfile bool `json:"include_profile,omitempty"`
-			} `json:"event_context"`
-			Filter    gomatrixserverlib.RoomEventFilter `json:"filter"`
-			Groupings struct {
-				GroupBy []struct {
-					Key string `json:"key"`
-				} `json:"group_by"`
-			} `json:"groupings"`
-			IncludeState bool     `json:"include_state"`
-			Keys         []string `json:"keys"`
-			OrderBy      string   `json:"order_by"`
-			SearchTerm   string   `json:"search_term"`
-		} `json:"room_events"`
-	} `json:"search_categories"`
 }
 
 type SearchResponse struct {
-	SearchCategories SearchCategories `json:"search_categories"`
+	SearchCategories SearchCategoriesResponse `json:"search_categories"`
 }
 type RoomResult struct {
 	NextBatch *string  `json:"next_batch,omitempty"`
@@ -332,15 +341,15 @@ type SearchContextResponse struct {
 	EventsAfter  []gomatrixserverlib.ClientEvent `json:"events_after"`
 	EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before"`
 	Start        string                          `json:"start"`
-	ProfileInfo  map[string]ProfileInfo          `json:"profile_info"`
+	ProfileInfo  map[string]ProfileInfoResponse  `json:"profile_info"`
 }
 
-type ProfileInfo struct {
+type ProfileInfoResponse struct {
 	AvatarURL   string `json:"avatar_url"`
 	DisplayName string `json:"display_name"`
 }
 
-type RoomEvents struct {
+type RoomEventsResponse struct {
 	Count      int                                        `json:"count"`
 	Groups     Groups                                     `json:"groups"`
 	Highlights []string                                   `json:"highlights"`
@@ -348,6 +357,6 @@ type RoomEvents struct {
 	Results    []Result                                   `json:"results"`
 	State      map[string][]gomatrixserverlib.ClientEvent `json:"state,omitempty"`
 }
-type SearchCategories struct {
+type SearchCategoriesResponse struct {
-	RoomEvents RoomEvents `json:"room_events"`
+	RoomEvents RoomEventsResponse `json:"room_events"`
 }

syncapi/routing/search_test.go

@@ -0,0 +1,264 @@
+package routing
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/fulltext"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSearch(t *testing.T) {
+	alice := test.NewUser(t)
+	aliceDevice := userapi.Device{UserID: alice.ID}
+	room := test.NewRoom(t, alice)
+	room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "context before"})
+	room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world3!"})
+	room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "context after"})
+
+	roomsFilter := []string{room.ID}
+	roomsFilterUnknown := []string{"!unknown"}
+
+	emptyFromString := ""
+	fromStringValid := "1"
+	fromStringInvalid := "iCantBeParsed"
+
+	testCases := []struct {
+		name              string
+		wantOK            bool
+		searchReq         SearchRequest
+		device            *userapi.Device
+		wantResponseCount int
+		from              *string
+	}{
+		{
+			name:      "no user ID",
+			searchReq: SearchRequest{},
+			device:    &userapi.Device{},
+		},
+		{
+			name:      "with alice ID",
+			wantOK:    true,
+			searchReq: SearchRequest{},
+			device:    &aliceDevice,
+		},
+		{
+			name:   "searchTerm specified, found at the beginning",
+			wantOK: true,
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "hello"}},
+			},
+			device:            &aliceDevice,
+			wantResponseCount: 1,
+		},
+		{
+			name:   "searchTerm specified, found at the end",
+			wantOK: true,
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "world3"}},
+			},
+			device:            &aliceDevice,
+			wantResponseCount: 1,
+		},
+		/* the following would need matchQuery.SetFuzziness(1) in bleve.go
+		{
+			name:   "searchTerm fuzzy search",
+			wantOK: true,
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "hell"}}, // this still should find hello world
+			},
+			device:            &aliceDevice,
+			wantResponseCount: 1,
+		},
+		*/
+		{
+			name:   "searchTerm specified but no result",
+			wantOK: true,
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "i don't match"}},
+			},
+			device: &aliceDevice,
+		},
+		{
+			name:   "filter on room",
+			wantOK: true,
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{
+					RoomEvents: RoomEvents{
+						SearchTerm: "hello",
+						Filter: gomatrixserverlib.RoomEventFilter{
+							Rooms: &roomsFilter,
+						},
+					},
+				},
+			},
+			device:            &aliceDevice,
+			wantResponseCount: 1,
+		},
+		{
+			name: "filter on unknown room",
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{
+					RoomEvents: RoomEvents{
+						SearchTerm: "hello",
+						Filter: gomatrixserverlib.RoomEventFilter{
+							Rooms: &roomsFilterUnknown,
+						},
+					},
+				},
+			},
+			device: &aliceDevice,
+		},
+		{
+			name:   "include state",
+			wantOK: true,
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{
+					RoomEvents: RoomEvents{
+						SearchTerm: "hello",
+						Filter: gomatrixserverlib.RoomEventFilter{
+							Rooms: &roomsFilter,
+						},
+						IncludeState: true,
+					},
+				},
+			},
+			device:            &aliceDevice,
+			wantResponseCount: 1,
+		},
+		{
+			name:   "empty from does not error",
+			wantOK: true,
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{
+					RoomEvents: RoomEvents{
+						SearchTerm: "hello",
+						Filter: gomatrixserverlib.RoomEventFilter{
+							Rooms: &roomsFilter,
+						},
+					},
+				},
+			},
+			wantResponseCount: 1,
+			device:            &aliceDevice,
+			from:              &emptyFromString,
+		},
+		{
+			name:   "valid from does not error",
+			wantOK: true,
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{
+					RoomEvents: RoomEvents{
+						SearchTerm: "hello",
+						Filter: gomatrixserverlib.RoomEventFilter{
+							Rooms: &roomsFilter,
+						},
+					},
+				},
+			},
+			wantResponseCount: 1,
+			device:            &aliceDevice,
+			from:              &fromStringValid,
+		},
+		{
+			name: "invalid from does error",
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{
+					RoomEvents: RoomEvents{
+						SearchTerm: "hello",
+						Filter: gomatrixserverlib.RoomEventFilter{
+							Rooms: &roomsFilter,
+						},
+					},
+				},
+			},
+			device: &aliceDevice,
+			from:   &fromStringInvalid,
+		},
+		{
+			name:   "order by stream position",
+			wantOK: true,
+			searchReq: SearchRequest{
+				SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "hello", OrderBy: "recent"}},
+			},
+			device:            &aliceDevice,
+			wantResponseCount: 1,
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+
+		// create requisites
+		fts, err := fulltext.New(processCtx, cfg.SyncAPI.Fulltext)
+		assert.NoError(t, err)
+		assert.NotNil(t, fts)
+
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		db, err := storage.NewSyncServerDatasource(processCtx.Context(), cm, &cfg.SyncAPI.Database)
+		assert.NoError(t, err)
+
+		elements := []fulltext.IndexElement{}
+		// store the events in the database
+		var sp types.StreamPosition
+		for _, x := range room.Events() {
+			var stateEvents []*gomatrixserverlib.HeaderedEvent
+			var stateEventIDs []string
+			if x.Type() == gomatrixserverlib.MRoomMember {
+				stateEvents = append(stateEvents, x)
+				stateEventIDs = append(stateEventIDs, x.EventID())
+			}
+			sp, err = db.WriteEvent(processCtx.Context(), x, stateEvents, stateEventIDs, nil, nil, false, gomatrixserverlib.HistoryVisibilityShared)
+			assert.NoError(t, err)
+			if x.Type() != "m.room.message" {
+				continue
+			}
+			elements = append(elements, fulltext.IndexElement{
+				EventID:        x.EventID(),
+				RoomID:         x.RoomID(),
+				Content:        string(x.Content()),
+				ContentType:    x.Type(),
+				StreamPosition: int64(sp),
+			})
+		}
+		// Index the events
+		err = fts.Index(elements...)
+		assert.NoError(t, err)
+
+		// run the tests
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				reqBody := &bytes.Buffer{}
+				err = json.NewEncoder(reqBody).Encode(tc.searchReq)
+				assert.NoError(t, err)
+				req := httptest.NewRequest(http.MethodPost, "/", reqBody)
+
+				res := Search(req, tc.device, db, fts, tc.from)
+				if !tc.wantOK && !res.Is2xx() {
+					return
+				}
+				resp, ok := res.JSON.(SearchResponse)
+				if !ok && !tc.wantOK {
+					t.Fatalf("not a SearchResponse: %T: %s", res.JSON, res.JSON)
+				}
+				assert.Equal(t, tc.wantResponseCount, resp.SearchCategories.RoomEvents.Count)
+
+				// if we requested state, it should not be empty
+				if tc.searchReq.SearchCategories.RoomEvents.IncludeState {
+					assert.NotEmpty(t, resp.SearchCategories.RoomEvents.State)
+				}
+			})
+		}
+	})
+}

userapi/api/api.go

@@ -224,7 +224,6 @@ type PerformDeviceUpdateRequest struct {
 }
 type PerformDeviceUpdateResponse struct {
 	DeviceExists bool
-	Forbidden    bool
 }
 
 type PerformDeviceDeletionRequest struct {

userapi/internal/user_api.go

@@ -386,11 +386,6 @@ func (a *UserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.Perf
 	}
 	res.DeviceExists = true
 
-	if dev.UserID != req.RequestingUserID {
-		res.Forbidden = true
-		return nil
-	}
-
 	err = a.DB.UpdateDevice(ctx, localpart, domain, req.DeviceID, req.DisplayName)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("deviceDB.UpdateDevice failed")