Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/sendleave

.dockerignore

@@ -1,3 +1,2 @@
 bin
 *.wasm
-.git

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -62,6 +62,6 @@ If you can identify any relevant log snippets from server logs, please include
 those (please be careful to remove any personal or private data). Please surround them with
 ``` (three backticks, on a line on their own), so that they are formatted legibly.
 
-Alternatively, please send logs to @kegan:matrix.org or @neilalexander:matrix.org
+Alternatively, please send logs to @kegan:matrix.org, @s7evink:matrix.org or @devonh:one.ems.host
 with a link to the respective Github issue, thanks!
 -->

.github/workflows/dendrite.yml

@@ -123,7 +123,7 @@ jobs:
         with:
           # Optional: pass GITHUB_TOKEN to avoid rate limiting.
           token: ${{ secrets.GITHUB_TOKEN }}
-      - run: go test -json -v ./... 2>&1 | gotestfmt
+      - run: go test -json -v ./... 2>&1 | gotestfmt -hide all
         env:
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
@@ -255,7 +255,7 @@ jobs:
           key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-stable-test-race-
-      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
+      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all
         env:
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
@@ -280,6 +280,8 @@ jobs:
         with:
           go-version: "stable"
           cache: true
+      - name: Docker version
+        run: docker version
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
       - name: Test upgrade (PostgreSQL)
@@ -300,6 +302,8 @@ jobs:
         with:
           go-version: "stable"
           cache: true
+      - name: Docker version
+        run: docker version
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
       - name: Test upgrade (PostgreSQL)
@@ -436,7 +440,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:

.github/workflows/docker.yml

@@ -32,10 +32,6 @@ jobs:
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -57,10 +53,9 @@ jobs:
         id: docker_build_monolith
         uses: docker/build-push-action@v3
         with:
-          cache-from: type=gha
+          cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache
-          cache-to: type=gha,mode=max
+          cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -75,7 +70,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -109,10 +103,6 @@ jobs:
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -137,7 +127,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -153,7 +142,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -176,10 +164,6 @@ jobs:
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -204,7 +188,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -220,7 +203,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true

.github/workflows/helm.yml

@@ -32,7 +32,7 @@ jobs:
           version: v3.10.0
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.4.1
+        uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         with:

.github/workflows/schedules.yaml

@@ -128,7 +128,7 @@ jobs:
         # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+          go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
       - name: Run actions/checkout@v3 for dendrite
         uses: actions/checkout@v3
         with:

.golangci.yml

@@ -180,7 +180,6 @@ linters-settings:
 linters:
   enable:
     - errcheck
-    - goconst
     - gocyclo
     - goimports # Does everything gofmt does
     - gosimple
@@ -211,6 +210,7 @@ linters:
     - stylecheck
     - typecheck # Should turn back on soon
     - unconvert # Should turn back on soon
+    - goconst # Slightly annoying, as it reports "issues" in SQL statements
   disable-all: false
   presets:
   fast: false

CHANGES.md

@@ -1,5 +1,123 @@
 # Changelog
 
+## Dendrite 0.13.4 (2023-10-25)
+
+Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution
+algorithm.
+
+### Fixes:
+
+- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also 
+  avoids spamming the logs when shutting down.)
+- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset"
+  out of rooms and other missing state events due to calculating the wrong state.
+- A bug when setting notifications from Element Android has been fixed by implementing MSC3987
+
+### Features
+
+- Updated dependencies
+  - Internal NATS Server has been updated from v2.9.19 to v2.9.23
+
+## Dendrite 0.13.3 (2023-09-28)
+
+### Fixes:
+
+- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir))
+- Invitations are now correctly pushed to devices
+- A bug which could result in the corruption of `m.direct` account data has been fixed 
+
+### Features
+
+- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response
+- Room version 11 is now supported
+- Clients can request the `federation` `event_format` when creating filters
+- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+
+### Other
+
+- Dendrite now requires Go 1.20 if building from source
+
+## Dendrite 0.13.2 (2023-08-23)
+
+### Fixes:
+
+- Migrations in SQLite are now prepared on the correct context (transaction or database)
+- The `InputRoomEvent` stream now has a maximum age of 24h, which should help with slow start up times of NATS JetStream (contributed by [neilalexander](https://github.com/neilalexander))
+- Event size checks are more in line with Synapse
+- Requests to `/messages` have been optimized, possibly reducing database round trips
+- Re-add the revision of Dendrite when building from source (Note: This only works if git is installed)
+- Getting local members to notify has been optimized, which should significantly reduce memory allocation and cache usage
+- When getting queried about user profiles, we now return HTTP404 if the user/profiles does not exist
+- Background federated joins should now be fixed and not timeout after a short time
+- Database connections are now correctly re-used
+- Restored the old behavior of the `/purgeRoom` admin endpoint (does not evacuate the room before purging)
+- Don't expose information about the system when trying to download files that don't exist
+
+### Features
+
+- Further improvements and fixes for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+  - Lookup correct prev events in the sync API
+  - Populate `prev_sender` correctly in the sync API
+  - Event federation should work better
+- Added new `dendrite_up` Prometheus metric, containing the version of Dendrite
+- Space summaries ([MSC2946](https://github.com/matrix-org/matrix-spec-proposals/pull/2946)) have been moved from MSC to being natively supported
+- For easier issue investigation, logs for application services now contain the application service ID (contributed by [maxberger](https://github.com/maxberger))
+- The default room version to use when creating rooms can now be configured using `room_server.default_room_version`
+
+## Dendrite 0.13.1 (2023-07-06)
+
+This releases fixes a long-standing "off-by-one" error which could result in state resets. Upgrading to this version is **highly** recommended.
+
+When deduplicating state events, we were checking if the event in question was already in a state snapshot. If it was in a previous state snapshot, we would 
+then remove it from the list of events to store. If this happened, we were, unfortunately, skipping the next event to check. This resulted in 
+events getting stored in state snapshots where they may not be needed. When we now compared two of those state snapshots, one of them
+contained the skipped event, while the other didn't. This difference possibly shouldn't exist, resulting in unexpected state resets and explains
+reports of missing state events as well.
+
+Rooms where a state reset occurred earlier should, hopefully, reconcile over time.
+
+### Fixes:
+
+- A long-standing "off-by-one" error has been fixed, which could result in state resets
+- Roomserver Prometheus Metrics are available again
+
+### Features
+
+- Updated dependencies
+  - Internal NATS Server has been updated from v2.9.15 to v2.9.19
+
+## Dendrite 0.13.0 (2023-06-30)
+
+### Features
+
+- Results in responses to `/search` now highlight words more accurately and not only the search terms as before
+- Support for connecting to appservices listening on unix sockets has been added (contributed by [cyberb](https://github.com/cyberb))
+- Admin APIs for token authenticated registration have been added (contributed by [santhoshivan23](https://github.com/santhoshivan23))
+- Initial support for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+  - This is **highly experimental**, things like changing usernames/avatars, inviting users, upgrading rooms isn't working
+
+### Fixes
+
+- `m.upload.size` is now optional, finally allowing uploads with unlimited file size
+- A bug while resolving server names has been fixed (contributed by [anton-molyboha](https://github.com/anton-molyboha))
+- Application services should only receive one invitation instead of 2 (or worse), which could result in state resets previously
+- Several admin endpoints are now using `POST` instead of `GET`
+- `/delete_devices` now uses user-interactive authentication
+- Several "membership" (e.g `/kick`, `/ban`) endpoints are using less heavy database queries to check if the user is allowed to perform this action
+- `/3pid` endpoints are now available on `/v3` instead of the `/unstable` prefix
+- Upgrading rooms ignores state events of other users, which could result in failed upgrades before
+- Uploading key backups with a wrong version now returns `M_WRONG_ROOM_KEYS_VERSION`
+- A potential state reset when joining the same room multiple times in short sequence has been fixed
+- A bug where we returned the full event as `redacted_because` in redaction events has been fixed
+- The `displayname` and `avatar_url` can now be set to empty strings
+- Unsafe hotserving of files has been fixed (contributed by [joshqou](https://github.com/joshqou))
+- Joining new rooms would potentially return "redacted" events, due to history visibility not being set correctly, this could result in events being rejected 
+- Backfilling resulting in `unsuported room version ''` should now be solved
+
+### Other
+
+- Huge refactoring of Dendrite and gomatrixserverlib
+
 ## Dendrite 0.12.0 (2023-03-13)
 
 ### Features

Dockerfile

@@ -3,8 +3,8 @@
 #
 # base installs required dependencies and runs go mod download to cache dependencies
 #
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine AS base
-RUN apk --update --no-cache add bash build-base curl
+RUN apk --update --no-cache add bash build-base curl git
 
 #
 # build creates all needed binaries
@@ -13,7 +13,6 @@ FROM --platform=${BUILDPLATFORM} base AS build
 WORKDIR /src
 ARG TARGETOS
 ARG TARGETARCH
-ARG FLAGS
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
@@ -21,7 +20,7 @@ RUN --mount=target=. \
     GOARCH="$TARGETARCH" \
     GOOS="linux" \
     CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
-    go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
+    go build -v -trimpath -o /out/ ./cmd/...
 
 
 #

README.md

@@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for
 more information on requirements.
 
-To build Dendrite, you will need Go 1.18 or later.
+To build Dendrite, you will need Go 1.20 or later.
 
 For a usable federating Dendrite deployment, you will also need:
 
@@ -47,7 +47,7 @@ For a usable federating Dendrite deployment, you will also need:
 Also recommended are:
 
 - A PostgreSQL database engine, which will perform better than SQLite with many users and/or larger rooms
-- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/master/docs/nginx/monolith-sample.conf)
+- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/dendrite-sample.conf)
 
 The [Federation Tester](https://federationtester.matrix.org) can be used to verify your deployment.
 

appservice/appservice_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/matrix-org/dendrite/appservice"
@@ -32,6 +33,10 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 )
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestAppserviceInternalAPI(t *testing.T) {
 
 	// Set expected results
@@ -134,7 +139,6 @@ func TestAppserviceInternalAPI(t *testing.T) {
 		}
 		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
 		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
-
 		t.Cleanup(func() {
 			ctx.ShutdownDendrite()
 			ctx.WaitForShutdown()
@@ -144,7 +148,8 @@ func TestAppserviceInternalAPI(t *testing.T) {
 		natsInstance := jetstream.NATSInstance{}
 		cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 		runCases(t, asAPI)
@@ -238,7 +243,8 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) {
 	natsInstance := jetstream.NATSInstance{}
 	cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 	rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+	rsAPI.SetFederationAPI(nil, nil)
+	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 	t.Run("UserIDExists", func(t *testing.T) {
@@ -377,7 +383,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 		// Create required internal APIs
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// start the consumer
 		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
 

appservice/consumers/roomserver.go

@@ -128,7 +128,7 @@ func (s *OutputRoomEventConsumer) onMessage(
 			if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
 				newEventID := output.NewRoomEvent.Event.EventID()
 				eventsReq := &api.QueryEventsByIDRequest{
-					RoomID:   output.NewRoomEvent.Event.RoomID(),
+					RoomID:   output.NewRoomEvent.Event.RoomID().String(),
 					EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
 				}
 				eventsRes := &api.QueryEventsByIDResponse{}
@@ -181,7 +181,9 @@ func (s *OutputRoomEventConsumer) sendEvents(
 	// Create the transaction body.
 	transaction, err := json.Marshal(
 		ApplicationServiceTransaction{
-			Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatAll),
+			Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+				return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+			}),
 		},
 	)
 	if err != nil {
@@ -233,12 +235,18 @@ func (s *appserviceState) backoffAndPause(err error) error {
 //
 // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
 func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool {
+	user := ""
+	userID, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
+	if err == nil {
+		user = userID.String()
+	}
+
 	switch {
 	case appservice.URL == "":
 		return false
-	case appservice.IsInterestedInUserID(event.Sender()):
+	case appservice.IsInterestedInUserID(user):
 		return true
-	case appservice.IsInterestedInRoomID(event.RoomID()):
+	case appservice.IsInterestedInRoomID(event.RoomID().String()):
 		return true
 	}
 
@@ -249,7 +257,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	}
 
 	// Check all known room aliases of the room the event came from
-	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
+	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID().String()}
 	var queryRes api.GetAliasesForRoomIDResponse
 	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
 		for _, alias := range queryRes.Aliases {
@@ -260,7 +268,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID(),
+			"room_id":    event.RoomID().String(),
 		}).WithError(err).Errorf("Unable to get aliases for room")
 	}
 
@@ -276,7 +284,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 	// until we have a lighter way of checking the state before the event that
 	// doesn't involve state res, then this is probably OK.
 	membershipReq := &api.QueryMembershipsForRoomRequest{
-		RoomID:     event.RoomID(),
+		RoomID:     event.RoomID().String(),
 		JoinedOnly: true,
 	}
 	membershipRes := &api.QueryMembershipsForRoomResponse{}
@@ -305,7 +313,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID(),
+			"room_id":    event.RoomID().String(),
 		}).WithError(err).Errorf("Unable to get membership for room")
 	}
 	return false

appservice/query/query.go

@@ -217,7 +217,7 @@ func (a *AppServiceQueryAPI) Locations(
 		}
 
 		if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
-			log.WithError(err).Error("unable to get 'locations' from application service")
+			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service")
 			continue
 		}
 
@@ -252,7 +252,7 @@ func (a *AppServiceQueryAPI) User(
 		}
 
 		if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
-			log.WithError(err).Error("unable to get 'user' from application service")
+			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service")
 			continue
 		}
 
@@ -290,7 +290,7 @@ func (a *AppServiceQueryAPI) Protocols(
 		for _, as := range a.Cfg.Derived.ApplicationServices {
 			var proto api.ASProtocolResponse
 			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil {
-				log.WithError(err).Error("unable to get 'protocol' from application service")
+				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
 
@@ -320,7 +320,7 @@ func (a *AppServiceQueryAPI) Protocols(
 		for _, p := range as.Protocols {
 			var proto api.ASProtocolResponse
 			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil {
-				log.WithError(err).Error("unable to get 'protocol' from application service")
+				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
 			existing, ok := response[p]

are-we-synapse-yet.list

@@ -945,3 +945,11 @@ rmv User can invite remote user to room with version 10
 rmv Remote user can backfill in a room with version 10
 rmv Can reject invites over federation for rooms with version 10
 rmv Can receive redactions from regular users over federation in room version 10
+rmv User can create and send/receive messages in a room with version 11
+rmv local user can join room with version 11
+rmv User can invite local user to room with version 11
+rmv remote user can join room with version 11
+rmv User can invite remote user to room with version 11
+rmv Remote user can backfill in a room with version 11
+rmv Can reject invites over federation for rooms with version 11
+rmv Can receive redactions from regular users over federation in room version 11

build/dendritejs-pinecone/main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -190,13 +191,13 @@ func startup() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fedSenderAPI.IsBlacklistedOrBackingOff)
 
 	asQuery := appservice.NewInternalAPI(
 		processCtx, cfg, &natsInstance, userAPI, rsAPI,
 	)
 	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
 	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
 
 	monolith := setup.Monolith{

build/docker/Dockerfile.demo-pinecone

@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.19-alpine AS base
+FROM docker.io/golang:1.21-alpine AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/docker/Dockerfile.demo-yggdrasil

@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.19-alpine AS base
+FROM docker.io/golang:1.21-alpine AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/gobind-yggdrasil/monolith.go

@@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() {
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)

build/scripts/complement.sh

@@ -15,5 +15,5 @@ tar -xzf master.tar.gz
 
 # Run the tests!
 cd complement-master
-COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests
+COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi
 

clientapi/admin_test.go

@@ -2,6 +2,7 @@ package clientapi
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -23,12 +24,654 @@ import (
 	"github.com/matrix-org/util"
 	"github.com/tidwall/gjson"
 
+	capi "github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
+func TestAdminCreateToken(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser))
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		cfg.ClientAPI.RegistrationRequiresToken = true
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+			bob:        {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+		testCases := []struct {
+			name           string
+			requestingUser *test.User
+			requestOpt     test.HTTPRequestOpt
+			wantOK         bool
+			withHeader     bool
+		}{
+			{
+				name:           "Missing auth",
+				requestingUser: bob,
+				wantOK:         false,
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"token": "token1",
+				},
+				),
+			},
+			{
+				name:           "Bob is denied access",
+				requestingUser: bob,
+				wantOK:         false,
+				withHeader:     true,
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"token": "token2",
+				},
+				),
+			},
+			{
+				name:           "Alice can create a token without specifyiing any information",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+				requestOpt:     test.WithJSONBody(t, map[string]interface{}{}),
+			},
+			{
+				name:           "Alice can to create a token specifying a name",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"token": "token3",
+				},
+				),
+			},
+			{
+				name:           "Alice cannot to create a token that already exists",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"token": "token3",
+				},
+				),
+			},
+			{
+				name:           "Alice can create a token specifying valid params",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"token":        "token4",
+					"uses_allowed": 5,
+					"expiry_time":  time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond),
+				},
+				),
+			},
+			{
+				name:           "Alice cannot create a token specifying invalid name",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"token": "token@",
+				},
+				),
+			},
+			{
+				name:           "Alice cannot create a token specifying invalid uses_allowed",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"token":        "token5",
+					"uses_allowed": -1,
+				},
+				),
+			},
+			{
+				name:           "Alice cannot create a token specifying invalid expiry_time",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"token":       "token6",
+					"expiry_time": time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond),
+				},
+				),
+			},
+			{
+				name:           "Alice cannot to create a token specifying invalid length",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"length": 80,
+				},
+				),
+			},
+		}
+
+		for _, tc := range testCases {
+			tc := tc
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/registrationTokens/new")
+				if tc.requestOpt != nil {
+					req = test.NewRequest(t, http.MethodPost, "/_dendrite/admin/registrationTokens/new", tc.requestOpt)
+				}
+				if tc.withHeader {
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken)
+				}
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}
+
+func TestAdminListRegistrationTokens(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser))
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		cfg.ClientAPI.RegistrationRequiresToken = true
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+			bob:        {},
+		}
+		tokens := []capi.RegistrationToken{
+			{
+				Token:       getPointer("valid"),
+				UsesAllowed: getPointer(int32(10)),
+				ExpiryTime:  getPointer(time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond)),
+				Pending:     getPointer(int32(0)),
+				Completed:   getPointer(int32(0)),
+			},
+			{
+				Token:       getPointer("invalid"),
+				UsesAllowed: getPointer(int32(10)),
+				ExpiryTime:  getPointer(time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond)),
+				Pending:     getPointer(int32(0)),
+				Completed:   getPointer(int32(0)),
+			},
+		}
+		for _, tkn := range tokens {
+			tkn := tkn
+			userAPI.PerformAdminCreateRegistrationToken(ctx, &tkn)
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+		testCases := []struct {
+			name             string
+			requestingUser   *test.User
+			valid            string
+			isValidSpecified bool
+			wantOK           bool
+			withHeader       bool
+		}{
+			{
+				name:             "Missing auth",
+				requestingUser:   bob,
+				wantOK:           false,
+				isValidSpecified: false,
+			},
+			{
+				name:             "Bob is denied access",
+				requestingUser:   bob,
+				wantOK:           false,
+				withHeader:       true,
+				isValidSpecified: false,
+			},
+			{
+				name:           "Alice can list all tokens",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+			},
+			{
+				name:             "Alice can list all valid tokens",
+				requestingUser:   aliceAdmin,
+				wantOK:           true,
+				withHeader:       true,
+				valid:            "true",
+				isValidSpecified: true,
+			},
+			{
+				name:             "Alice can list all invalid tokens",
+				requestingUser:   aliceAdmin,
+				wantOK:           true,
+				withHeader:       true,
+				valid:            "false",
+				isValidSpecified: true,
+			},
+			{
+				name:             "No response when valid has a bad value",
+				requestingUser:   aliceAdmin,
+				wantOK:           false,
+				withHeader:       true,
+				valid:            "trueee",
+				isValidSpecified: true,
+			},
+		}
+
+		for _, tc := range testCases {
+			tc := tc
+			t.Run(tc.name, func(t *testing.T) {
+				var path string
+				if tc.isValidSpecified {
+					path = fmt.Sprintf("/_dendrite/admin/registrationTokens?valid=%v", tc.valid)
+				} else {
+					path = "/_dendrite/admin/registrationTokens"
+				}
+				req := test.NewRequest(t, http.MethodGet, path)
+				if tc.withHeader {
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken)
+				}
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}
+
+func TestAdminGetRegistrationToken(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser))
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		cfg.ClientAPI.RegistrationRequiresToken = true
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+			bob:        {},
+		}
+		tokens := []capi.RegistrationToken{
+			{
+				Token:       getPointer("alice_token1"),
+				UsesAllowed: getPointer(int32(10)),
+				ExpiryTime:  getPointer(time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond)),
+				Pending:     getPointer(int32(0)),
+				Completed:   getPointer(int32(0)),
+			},
+			{
+				Token:       getPointer("alice_token2"),
+				UsesAllowed: getPointer(int32(10)),
+				ExpiryTime:  getPointer(time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond)),
+				Pending:     getPointer(int32(0)),
+				Completed:   getPointer(int32(0)),
+			},
+		}
+		for _, tkn := range tokens {
+			tkn := tkn
+			userAPI.PerformAdminCreateRegistrationToken(ctx, &tkn)
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+		testCases := []struct {
+			name           string
+			requestingUser *test.User
+			token          string
+			wantOK         bool
+			withHeader     bool
+		}{
+			{
+				name:           "Missing auth",
+				requestingUser: bob,
+				wantOK:         false,
+			},
+			{
+				name:           "Bob is denied access",
+				requestingUser: bob,
+				wantOK:         false,
+				withHeader:     true,
+			},
+			{
+				name:           "Alice can GET alice_token1",
+				token:          "alice_token1",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+			},
+			{
+				name:           "Alice can GET alice_token2",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+				token:          "alice_token2",
+			},
+			{
+				name:           "Alice cannot GET a token that does not exists",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				token:          "alice_token3",
+			},
+		}
+
+		for _, tc := range testCases {
+			tc := tc
+			t.Run(tc.name, func(t *testing.T) {
+				path := fmt.Sprintf("/_dendrite/admin/registrationTokens/%s", tc.token)
+				req := test.NewRequest(t, http.MethodGet, path)
+				if tc.withHeader {
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken)
+				}
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}
+
+func TestAdminDeleteRegistrationToken(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser))
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		cfg.ClientAPI.RegistrationRequiresToken = true
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+			bob:        {},
+		}
+		tokens := []capi.RegistrationToken{
+			{
+				Token:       getPointer("alice_token1"),
+				UsesAllowed: getPointer(int32(10)),
+				ExpiryTime:  getPointer(time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond)),
+				Pending:     getPointer(int32(0)),
+				Completed:   getPointer(int32(0)),
+			},
+			{
+				Token:       getPointer("alice_token2"),
+				UsesAllowed: getPointer(int32(10)),
+				ExpiryTime:  getPointer(time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond)),
+				Pending:     getPointer(int32(0)),
+				Completed:   getPointer(int32(0)),
+			},
+		}
+		for _, tkn := range tokens {
+			tkn := tkn
+			userAPI.PerformAdminCreateRegistrationToken(ctx, &tkn)
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+		testCases := []struct {
+			name           string
+			requestingUser *test.User
+			token          string
+			wantOK         bool
+			withHeader     bool
+		}{
+			{
+				name:           "Missing auth",
+				requestingUser: bob,
+				wantOK:         false,
+			},
+			{
+				name:           "Bob is denied access",
+				requestingUser: bob,
+				wantOK:         false,
+				withHeader:     true,
+			},
+			{
+				name:           "Alice can DELETE alice_token1",
+				token:          "alice_token1",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+			},
+			{
+				name:           "Alice can DELETE alice_token2",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+				token:          "alice_token2",
+			},
+		}
+
+		for _, tc := range testCases {
+			tc := tc
+			t.Run(tc.name, func(t *testing.T) {
+				path := fmt.Sprintf("/_dendrite/admin/registrationTokens/%s", tc.token)
+				req := test.NewRequest(t, http.MethodDelete, path)
+				if tc.withHeader {
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken)
+				}
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}
+
+func TestAdminUpdateRegistrationToken(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser))
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		cfg.ClientAPI.RegistrationRequiresToken = true
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+			bob:        {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+		tokens := []capi.RegistrationToken{
+			{
+				Token:       getPointer("alice_token1"),
+				UsesAllowed: getPointer(int32(10)),
+				ExpiryTime:  getPointer(time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond)),
+				Pending:     getPointer(int32(0)),
+				Completed:   getPointer(int32(0)),
+			},
+			{
+				Token:       getPointer("alice_token2"),
+				UsesAllowed: getPointer(int32(10)),
+				ExpiryTime:  getPointer(time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond)),
+				Pending:     getPointer(int32(0)),
+				Completed:   getPointer(int32(0)),
+			},
+		}
+		for _, tkn := range tokens {
+			tkn := tkn
+			userAPI.PerformAdminCreateRegistrationToken(ctx, &tkn)
+		}
+		testCases := []struct {
+			name           string
+			requestingUser *test.User
+			method         string
+			token          string
+			requestOpt     test.HTTPRequestOpt
+			wantOK         bool
+			withHeader     bool
+		}{
+			{
+				name:           "Missing auth",
+				requestingUser: bob,
+				wantOK:         false,
+				token:          "alice_token1",
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"uses_allowed": 10,
+				},
+				),
+			},
+			{
+				name:           "Bob is denied access",
+				requestingUser: bob,
+				wantOK:         false,
+				withHeader:     true,
+				token:          "alice_token1",
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"uses_allowed": 10,
+				},
+				),
+			},
+			{
+				name:           "Alice can UPDATE a token's uses_allowed property",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+				token:          "alice_token1",
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"uses_allowed": 10,
+				}),
+			},
+			{
+				name:           "Alice can UPDATE a token's expiry_time property",
+				requestingUser: aliceAdmin,
+				wantOK:         true,
+				withHeader:     true,
+				token:          "alice_token2",
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"expiry_time": time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond),
+				},
+				),
+			},
+			{
+				name:           "Alice can UPDATE a token's uses_allowed and expiry_time property",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				token:          "alice_token1",
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"uses_allowed": 20,
+					"expiry_time":  time.Now().Add(10*24*time.Hour).UnixNano() / int64(time.Millisecond),
+				},
+				),
+			},
+			{
+				name:           "Alice CANNOT update a token with invalid properties",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				token:          "alice_token2",
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"uses_allowed": -5,
+					"expiry_time":  time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond),
+				},
+				),
+			},
+			{
+				name:           "Alice CANNOT UPDATE a token that does not exist",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				token:          "alice_token9",
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"uses_allowed": 100,
+				},
+				),
+			},
+			{
+				name:           "Alice can UPDATE token specifying uses_allowed as null - Valid for infinite uses",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				token:          "alice_token1",
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"uses_allowed": nil,
+				},
+				),
+			},
+			{
+				name:           "Alice can UPDATE token specifying expiry_time AS null - Valid for infinite time",
+				requestingUser: aliceAdmin,
+				wantOK:         false,
+				withHeader:     true,
+				token:          "alice_token1",
+				requestOpt: test.WithJSONBody(t, map[string]interface{}{
+					"expiry_time": nil,
+				},
+				),
+			},
+		}
+
+		for _, tc := range testCases {
+			tc := tc
+			t.Run(tc.name, func(t *testing.T) {
+				path := fmt.Sprintf("/_dendrite/admin/registrationTokens/%s", tc.token)
+				req := test.NewRequest(t, http.MethodPut, path)
+				if tc.requestOpt != nil {
+					req = test.NewRequest(t, http.MethodPut, path, tc.requestOpt)
+				}
+				if tc.withHeader {
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken)
+				}
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}
+
+func getPointer[T any](s T) *T {
+	return &s
+}
+
 func TestAdminResetPassword(t *testing.T) {
 	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
 	bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser))
@@ -48,8 +691,9 @@ func TestAdminResetPassword(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for changing the password/login
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -142,13 +786,14 @@ func TestPurgeRoom(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		// this starts the JetStream consumers
-		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
+
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
 			t.Fatalf("failed to send events: %v", err)
@@ -213,12 +858,13 @@ func TestAdminEvacuateRoom(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		// this starts the JetStream consumers
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
 			t.Fatalf("failed to send events: %v", err)
@@ -313,12 +959,13 @@ func TestAdminEvacuateUser(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		// this starts the JetStream consumers
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
 			t.Fatalf("failed to send events: %v", err)
@@ -407,7 +1054,8 @@ func TestAdminMarkAsStale(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)

clientapi/api/api.go

@@ -21,3 +21,11 @@ type ExtraPublicRoomsProvider interface {
 	// Rooms returns the extra rooms. This is called on-demand by clients, so cache appropriately.
 	Rooms() []fclient.PublicRoom
 }
+
+type RegistrationToken struct {
+	Token       *string `json:"token"`
+	UsesAllowed *int32  `json:"uses_allowed"`
+	Pending     *int32  `json:"pending"`
+	Completed   *int32  `json:"completed"`
+	ExpiryTime  *int64  `json:"expiry_time"`
+}

clientapi/clientapi_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/pushrules"
@@ -49,6 +50,10 @@ type userDevice struct {
 	password    string
 }
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestGetPutDevices(t *testing.T) {
 	alice := test.NewUser(t)
 	bob := test.NewUser(t)
@@ -120,7 +125,8 @@ func TestGetPutDevices(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -168,7 +174,8 @@ func TestDeleteDevice(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -272,7 +279,8 @@ func TestDeleteDevices(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -439,7 +447,7 @@ func TestSetDisplayname(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -551,7 +559,7 @@ func TestSetAvatarURL(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -629,7 +637,7 @@ func TestTyping(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -713,7 +721,7 @@ func TestMembership(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		rsAPI.SetUserAPI(userAPI)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -920,13 +928,17 @@ func TestCapabilities(t *testing.T) {
 		}
 	}
 
+	var tempRoomServerCfg config.RoomServer
+	tempRoomServerCfg.Defaults(config.DefaultOpts{})
+	defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion
+
 	expectedMap := map[string]interface{}{
 		"capabilities": map[string]interface{}{
 			"m.change_password": map[string]bool{
 				"enabled": true,
 			},
 			"m.room_versions": map[string]interface{}{
-				"default":   version.DefaultRoomVersion(),
+				"default":   defaultRoomVersion,
 				"available": versionsMap,
 			},
 		},
@@ -947,7 +959,8 @@ func TestCapabilities(t *testing.T) {
 
 		// Needed to create accounts
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -993,7 +1006,8 @@ func TestTurnserver(t *testing.T) {
 
 	// Needed to create accounts
 	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+	rsAPI.SetFederationAPI(nil, nil)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	//rsAPI.SetUserAPI(userAPI)
 	// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 	AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1090,7 +1104,8 @@ func Test3PID(t *testing.T) {
 
 		// Needed to create accounts
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -1265,7 +1280,8 @@ func TestPushRules(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1407,7 +1423,7 @@ func TestPushRules(t *testing.T) {
 				validateFunc: func(t *testing.T, respBody *bytes.Buffer) {
 					actions := gjson.GetBytes(respBody.Bytes(), "actions").Array()
 					// only a basic check
-					assert.Equal(t, 1, len(actions))
+					assert.Equal(t, 0, len(actions))
 				},
 			},
 			{
@@ -1651,7 +1667,8 @@ func TestKeys(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -2112,7 +2129,8 @@ func TestKeyBackup(t *testing.T) {
 		routers := httputil.NewRouters()
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)

clientapi/routing/account_data.go

@@ -145,8 +145,16 @@ func SaveReadMarker(
 	userAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
 ) util.JSONResponse {
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("userID for this device is invalid"),
+		}
+	}
+
 	// Verify that the user is a member of this room
-	resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}

clientapi/routing/admin.go

@@ -6,6 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"regexp"
+	"strconv"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -16,14 +18,254 @@ import (
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/exp/constraints"
 
+	clientapi "github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/userapi/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
+var validRegistrationTokenRegex = regexp.MustCompile("^[[:ascii:][:digit:]_]*$")
+
+func AdminCreateNewRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	if !cfg.RegistrationRequiresToken {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("Registration via tokens is not enabled on this homeserver"),
+		}
+	}
+	request := struct {
+		Token       string `json:"token"`
+		UsesAllowed *int32 `json:"uses_allowed,omitempty"`
+		ExpiryTime  *int64 `json:"expiry_time,omitempty"`
+		Length      int32  `json:"length"`
+	}{}
+
+	if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON(fmt.Sprintf("Failed to decode request body: %s", err)),
+		}
+	}
+
+	token := request.Token
+	usesAllowed := request.UsesAllowed
+	expiryTime := request.ExpiryTime
+	length := request.Length
+
+	if len(token) == 0 {
+		if length == 0 {
+			// length not provided in request. Assign default value of 16.
+			length = 16
+		}
+		// token not present in request body. Hence, generate a random token.
+		if length <= 0 || length > 64 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.BadJSON("length must be greater than zero and not greater than 64"),
+			}
+		}
+		token = util.RandomString(int(length))
+	}
+
+	if len(token) > 64 {
+		//Token present in request body, but is too long.
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("token must not be longer than 64"),
+		}
+	}
+
+	isTokenValid := validRegistrationTokenRegex.Match([]byte(token))
+	if !isTokenValid {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("token must consist only of characters matched by the regex [A-Za-z0-9-_]"),
+		}
+	}
+	// At this point, we have a valid token, either through request body or through random generation.
+	if usesAllowed != nil && *usesAllowed < 0 {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("uses_allowed must be a non-negative integer or null"),
+		}
+	}
+	if expiryTime != nil && spec.Timestamp(*expiryTime).Time().Before(time.Now()) {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("expiry_time must not be in the past"),
+		}
+	}
+	pending := int32(0)
+	completed := int32(0)
+	// If usesAllowed or expiryTime is 0, it means they are not present in the request. NULL (indicating unlimited uses / no expiration will be persisted in DB)
+	registrationToken := &clientapi.RegistrationToken{
+		Token:       &token,
+		UsesAllowed: usesAllowed,
+		Pending:     &pending,
+		Completed:   &completed,
+		ExpiryTime:  expiryTime,
+	}
+	created, err := userAPI.PerformAdminCreateRegistrationToken(req.Context(), registrationToken)
+	if !created {
+		return util.JSONResponse{
+			Code: http.StatusConflict,
+			JSON: map[string]string{
+				"error": fmt.Sprintf("token: %s already exists", token),
+			},
+		}
+	}
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: err,
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{
+			"token":        token,
+			"uses_allowed": getReturnValue(usesAllowed),
+			"pending":      pending,
+			"completed":    completed,
+			"expiry_time":  getReturnValue(expiryTime),
+		},
+	}
+}
+
+func getReturnValue[t constraints.Integer](in *t) any {
+	if in == nil {
+		return nil
+	}
+	return *in
+}
+
+func AdminListRegistrationTokens(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	queryParams := req.URL.Query()
+	returnAll := true
+	valid := true
+	validQuery, ok := queryParams["valid"]
+	if ok {
+		returnAll = false
+		validValue, err := strconv.ParseBool(validQuery[0])
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.BadJSON("invalid 'valid' query parameter"),
+			}
+		}
+		valid = validValue
+	}
+	tokens, err := userAPI.PerformAdminListRegistrationTokens(req.Context(), returnAll, valid)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.ErrorUnknown,
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{
+			"registration_tokens": tokens,
+		},
+	}
+}
+
+func AdminGetRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	tokenText := vars["token"]
+	token, err := userAPI.PerformAdminGetRegistrationToken(req.Context(), tokenText)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound(fmt.Sprintf("token: %s not found", tokenText)),
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: token,
+	}
+}
+
+func AdminDeleteRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	tokenText := vars["token"]
+	err = userAPI.PerformAdminDeleteRegistrationToken(req.Context(), tokenText)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: err,
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{},
+	}
+}
+
+func AdminUpdateRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	tokenText := vars["token"]
+	request := make(map[string]*int64)
+	if err = json.NewDecoder(req.Body).Decode(&request); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON(fmt.Sprintf("Failed to decode request body: %s", err)),
+		}
+	}
+	newAttributes := make(map[string]interface{})
+	usesAllowed, ok := request["uses_allowed"]
+	if ok {
+		// Only add usesAllowed to newAtrributes if it is present and valid
+		if usesAllowed != nil && *usesAllowed < 0 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.BadJSON("uses_allowed must be a non-negative integer or null"),
+			}
+		}
+		newAttributes["usesAllowed"] = usesAllowed
+	}
+	expiryTime, ok := request["expiry_time"]
+	if ok {
+		// Only add expiryTime to newAtrributes if it is present and valid
+		if expiryTime != nil && spec.Timestamp(*expiryTime).Time().Before(time.Now()) {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.BadJSON("expiry_time must not be in the past"),
+			}
+		}
+		newAttributes["expiryTime"] = expiryTime
+	}
+	if len(newAttributes) == 0 {
+		// No attributes to update. Return existing token
+		return AdminGetRegistrationToken(req, cfg, userAPI)
+	}
+	updatedToken, err := userAPI.PerformAdminUpdateRegistrationToken(req.Context(), tokenText, newAttributes)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound(fmt.Sprintf("token: %s not found", tokenText)),
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: *updatedToken,
+	}
+}
+
 func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {

clientapi/routing/aliases.go

@@ -55,9 +55,16 @@ func GetAliases(
 		visibility = content.HistoryVisibility
 	}
 	if visibility != spec.WorldReadable {
+		deviceUserID, err := spec.NewUserID(device.UserID, true)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
+			}
+		}
 		queryReq := api.QueryMembershipForUserRequest{
 			RoomID: roomID,
-			UserID: device.UserID,
+			UserID: *deviceUserID,
 		}
 		var queryRes api.QueryMembershipForUserResponse
 		if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil {

clientapi/routing/capabilities.go

@@ -17,6 +17,7 @@ package routing
 import (
 	"net/http"
 
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -24,7 +25,7 @@ import (
 
 // GetCapabilities returns information about the server's supported feature set
 // and other relevant capabilities to an authenticated user.
-func GetCapabilities() util.JSONResponse {
+func GetCapabilities(rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	versionsMap := map[gomatrixserverlib.RoomVersion]string{}
 	for v, desc := range version.SupportedRoomVersions() {
 		if desc.Stable() {
@@ -40,7 +41,7 @@ func GetCapabilities() util.JSONResponse {
 				"enabled": true,
 			},
 			"m.room_versions": map[string]interface{}{
-				"default":   version.DefaultRoomVersion(),
+				"default":   rsAPI.DefaultRoomVersion(),
 				"available": versionsMap,
 			},
 		},

clientapi/routing/createroom.go

@@ -171,7 +171,7 @@ func createRoom(
 
 	// Clobber keys: creator, room_version
 
-	roomVersion := roomserverVersion.DefaultRoomVersion()
+	roomVersion := rsAPI.DefaultRoomVersion()
 	if createRequest.RoomVersion != "" {
 		candidateVersion := gomatrixserverlib.RoomVersion(createRequest.RoomVersion)
 		_, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion)
@@ -224,6 +224,7 @@ func createRoom(
 		PrivateKey:      privateKey,
 		EventTime:       evTime,
 	}
+
 	roomAlias, createRes := rsAPI.PerformCreateRoom(ctx, *userID, *roomID, &req)
 	if createRes != nil {
 		return *createRes

clientapi/routing/directory.go

@@ -181,13 +181,39 @@ func SetLocalAlias(
 		return *resErr
 	}
 
-	queryReq := roomserverAPI.SetRoomAliasRequest{
+	roomID, err := spec.NewRoomID(r.RoomID)
-		UserID: device.UserID,
+	if err != nil {
-		RoomID: r.RoomID,
+		return util.JSONResponse{
-		Alias:  alias,
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("invalid room ID"),
 		}
-	var queryRes roomserverAPI.SetRoomAliasResponse
+	}
-	if err := rsAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
+
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *roomID, *userID)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("QuerySenderIDForUser failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	} else if senderID == nil {
+		util.GetLogger(req.Context()).WithField("roomID", *roomID).WithField("userID", *userID).Error("Sender ID not found")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	aliasAlreadyExists, err := rsAPI.SetRoomAlias(req.Context(), *senderID, *roomID, alias)
+	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -195,7 +221,7 @@ func SetLocalAlias(
 		}
 	}
 
-	if queryRes.AliasExists {
+	if aliasAlreadyExists {
 		return util.JSONResponse{
 			Code: http.StatusConflict,
 			JSON: spec.Unknown("The alias " + alias + " already exists."),
@@ -215,27 +241,88 @@ func RemoveLocalAlias(
 	alias string,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 ) util.JSONResponse {
-	queryReq := roomserverAPI.RemoveRoomAliasRequest{
+	userID, err := spec.NewUserID(device.UserID, true)
-		Alias:  alias,
+	if err != nil {
-		UserID: device.UserID,
-	}
-	var queryRes roomserverAPI.RemoveRoomAliasResponse
-	if err := rsAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
+			JSON: spec.InternalServerError{Err: "UserID for device is invalid"},
 		}
 	}
 
-	if !queryRes.Found {
+	roomIDReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: alias}
+	roomIDRes := roomserverAPI.GetRoomIDForAliasResponse{}
+	err = rsAPI.GetRoomIDForAlias(req.Context(), &roomIDReq, &roomIDRes)
+	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: spec.NotFound("The alias does not exist."),
 		}
 	}
 
-	if !queryRes.Removed {
+	validRoomID, err := spec.NewRoomID(roomIDRes.RoomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The alias does not exist."),
+		}
+	}
+
+	// This seems like the kind of auth check that should be done in the roomserver, but
+	// if this check fails (user is not in the room), then there will be no SenderID for the user
+	// for pseudo-ID rooms - it will just return "". However, we can't use lack of a sender ID
+	// as meaning they are not in the room, since lacking a sender ID could be caused by other bugs.
+	// TODO: maybe have QuerySenderIDForUser return richer errors?
+	var queryResp roomserverAPI.QueryMembershipForUserResponse
+	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+		RoomID: validRoomID.String(),
+		UserID: *userID,
+	}, &queryResp)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.QueryMembershipForUser failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+	if !queryResp.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You do not have permission to remove this alias."),
+		}
+	}
+
+	deviceSenderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *userID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The alias does not exist."),
+		}
+	}
+	// TODO: how to handle this case? missing user/room keys seem to be a whole new class of errors
+	if deviceSenderID == nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	aliasFound, aliasRemoved, err := rsAPI.RemoveRoomAlias(req.Context(), *deviceSenderID, alias)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	if !aliasFound {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The alias does not exist."),
+		}
+	}
+
+	if !aliasRemoved {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: spec.Forbidden("You do not have permission to remove this alias."),
@@ -288,7 +375,30 @@ func SetVisibility(
 	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, dev *userapi.Device,
 	roomID string,
 ) util.JSONResponse {
-	resErr := checkMemberInRoom(req.Context(), rsAPI, dev.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(dev.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("userID for this device is invalid"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("roomID is invalid")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
+	if err != nil || senderID == nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.Unknown("failed to find senderID for this user"),
+		}
+	}
+
+	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}
@@ -301,7 +411,7 @@ func SetVisibility(
 		}},
 	}
 	var queryEventsRes roomserverAPI.QueryLatestEventsAndStateResponse
-	err := rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
+	err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
 	if err != nil || len(queryEventsRes.StateEvents) == 0 {
 		util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
 		return util.JSONResponse{
@@ -312,7 +422,7 @@ func SetVisibility(
 
 	// NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event
 	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].PDU)
-	if power.UserLevel(dev.UserID) < power.EventLevel(spec.MRoomCanonicalAlias, true) {
+	if power.UserLevel(*senderID) < power.EventLevel(spec.MRoomCanonicalAlias, true) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: spec.Forbidden("userID doesn't have power level to change visibility"),

clientapi/routing/joined_rooms.go

@@ -33,23 +33,36 @@ func GetJoinedRooms(
 	device *userapi.Device,
 	rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
-	var res api.QueryRoomsForUserResponse
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
-	err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{
+	if err != nil {
-		UserID:         device.UserID,
+		util.GetLogger(req.Context()).WithError(err).Error("Invalid device user ID")
-		WantMembership: "join",
+		return util.JSONResponse{
-	}, &res)
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	rooms, err := rsAPI.QueryRoomsForUser(req.Context(), *deviceUserID, "join")
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
+			JSON: spec.Unknown("internal server error"),
 		}
 	}
-	if res.RoomIDs == nil {
+
-		res.RoomIDs = []string{}
+	var roomIDStrs []string
+	if rooms == nil {
+		roomIDStrs = []string{}
+	} else {
+		roomIDStrs = make([]string, len(rooms))
+		for i, roomID := range rooms {
+			roomIDStrs[i] = roomID.String()
 		}
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: getJoinedRoomsResponse{res.RoomIDs},
+		JSON: getJoinedRoomsResponse{roomIDStrs},
 	}
 }

clientapi/routing/joinroom_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -21,6 +22,10 @@ import (
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestJoinRoomByIDOrAlias(t *testing.T) {
 	alice := test.NewUser(t)
 	bob := test.NewUser(t)
@@ -35,9 +40,9 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		natsInstance := jetstream.NATSInstance{}
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
-		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
 		// Create the users in the userapi
 		for _, u := range []*test.User{alice, bob, charlie} {

clientapi/routing/leaveroom.go

@@ -29,10 +29,18 @@ func LeaveRoomByID(
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	roomID string,
 ) util.JSONResponse {
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.Unknown("device userID is invalid"),
+		}
+	}
+
 	// Prepare to ask the roomserver to perform the room join.
 	leaveReq := roomserverAPI.PerformLeaveRequest{
 		RoomID: roomID,
-		UserID: device.UserID,
+		Leaver: *userID,
 	}
 	leaveRes := roomserverAPI.PerformLeaveResponse{}
 

clientapi/routing/login_test.go

@@ -47,8 +47,9 @@ func TestLogin(t *testing.T) {
 		routers := httputil.NewRouters()
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for /login
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
 		Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics)

clientapi/routing/membership.go

@@ -22,10 +22,6 @@ import (
 	"time"
 
 	"github.com/getsentry/sentry-go"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/fclient"
-	"github.com/matrix-org/gomatrixserverlib/spec"
-
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -36,6 +32,9 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/util"
 )
@@ -57,7 +56,29 @@ func SendBan(
 		}
 	}
 
-	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to ban this user, bad userID"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
+	if err != nil || senderID == nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to ban this user, unknown senderID"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
@@ -66,7 +87,7 @@ func SendBan(
 	if errRes != nil {
 		return *errRes
 	}
-	allowedToBan := pl.UserLevel(device.UserID) >= pl.Ban
+	allowedToBan := pl.UserLevel(*senderID) >= pl.Ban
 	if !allowedToBan {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
@@ -133,16 +154,46 @@ func SendKick(
 		}
 	}
 
-	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
+	if err != nil || senderID == nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
 
+	bodyUserID, err := spec.NewUserID(body.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("body userID is invalid"),
+		}
+	}
+
 	pl, errRes := getPowerlevels(req, rsAPI, roomID)
 	if errRes != nil {
 		return *errRes
 	}
-	allowedToKick := pl.UserLevel(device.UserID) >= pl.Kick
+	allowedToKick := pl.UserLevel(*senderID) >= pl.Kick || bodyUserID.String() == deviceUserID.String()
 	if !allowedToKick {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
@@ -151,9 +202,9 @@ func SendKick(
 	}
 
 	var queryRes roomserverAPI.QueryMembershipForUserResponse
-	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
-		UserID: body.UserID,
+		UserID: *bodyUserID,
 	}, &queryRes)
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -185,15 +236,30 @@ func SendUnban(
 		}
 	}
 
-	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
 
+	bodyUserID, err := spec.NewUserID(body.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("body userID is invalid"),
+		}
+	}
 	var queryRes roomserverAPI.QueryMembershipForUserResponse
-	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
-		UserID: body.UserID,
+		UserID: *bodyUserID,
 	}, &queryRes)
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -244,7 +310,15 @@ func SendInvite(
 		}
 	}
 
-	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
@@ -264,22 +338,55 @@ func sendInvite(
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
 ) (util.JSONResponse, error) {
-	event, err := buildMembershipEvent(
+	validRoomID, err := spec.NewRoomID(roomID)
-		ctx, userID, reason, profileAPI, device, spec.Invite,
+	if err != nil {
-		roomID, false, cfg, evTime, rsAPI, asAPI,
+		return util.JSONResponse{
-	)
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("RoomID is invalid"),
+		}, err
+	}
+	inviter, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, err
+	}
+	invitee, err := spec.NewUserID(userID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("UserID is invalid"),
+		}, err
+	}
+	profile, err := loadProfile(ctx, userID, cfg, profileAPI, asAPI)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, err
+	}
+	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
 	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: spec.InternalServerError{},
 		}, err
 	}
-
 	err = rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{
-		Event:           event,
+		InviteInput: roomserverAPI.InviteInput{
+			RoomID:      *validRoomID,
+			Inviter:     *inviter,
+			Invitee:     *invitee,
+			DisplayName: profile.DisplayName,
+			AvatarURL:   profile.AvatarURL,
+			Reason:      reason,
+			IsDirect:    false,
+			KeyID:       identity.KeyID,
+			PrivateKey:  identity.PrivateKey,
+			EventTime:   evTime,
+		},
 		InviteRoomState: nil, // ask the roomserver to draw up invite room state for us
-		RoomVersion:     event.Version(),
 		SendAsServer:    string(device.UserDomain()),
 	})
 
@@ -312,17 +419,18 @@ func sendInvite(
 
 func buildMembershipEventDirect(
 	ctx context.Context,
-	targetUserID, reason string, userDisplayName, userAvatarURL string,
+	targetSenderID spec.SenderID, reason string, userDisplayName, userAvatarURL string,
-	sender string, senderDomain spec.ServerName,
+	sender spec.SenderID, senderDomain spec.ServerName,
 	membership, roomID string, isDirect bool,
 	keyID gomatrixserverlib.KeyID, privateKey ed25519.PrivateKey, evTime time.Time,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 ) (*types.HeaderedEvent, error) {
+	targetSenderString := string(targetSenderID)
 	proto := gomatrixserverlib.ProtoEvent{
-		Sender:   sender,
+		SenderID: string(sender),
 		RoomID:   roomID,
 		Type:     "m.room.member",
-		StateKey: &targetUserID,
+		StateKey: &targetSenderString,
 	}
 
 	content := gomatrixserverlib.MemberContent{
@@ -358,13 +466,39 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
-	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return nil, err
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return nil, err
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID)
+	if err != nil {
+		return nil, err
+	} else if senderID == nil {
+		return nil, fmt.Errorf("no sender ID for %s in %s", *userID, *validRoomID)
+	}
+
+	targetID, err := spec.NewUserID(targetUserID, true)
+	if err != nil {
+		return nil, err
+	}
+	targetSenderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *targetID)
+	if err != nil {
+		return nil, err
+	} else if targetSenderID == nil {
+		return nil, fmt.Errorf("no sender ID for %s in %s", *targetID, *validRoomID)
+	}
+
+	identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *userID)
 	if err != nil {
 		return nil, err
 	}
 
-	return buildMembershipEventDirect(ctx, targetUserID, reason, profile.DisplayName, profile.AvatarURL,
+	return buildMembershipEventDirect(ctx, *targetSenderID, reason, profile.DisplayName, profile.AvatarURL,
-		device.UserID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI)
+		*senderID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI)
 }
 
 // loadProfile lookups the profile of a given user from the database and returns
@@ -462,7 +596,7 @@ func checkAndProcessThreepid(
 	return
 }
 
-func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID, roomID string) *util.JSONResponse {
+func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID spec.UserID, roomID string) *util.JSONResponse {
 	var membershipRes roomserverAPI.QueryMembershipForUserResponse
 	err := rsAPI.QueryMembershipForUser(ctx, &roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
@@ -490,12 +624,21 @@ func SendForget(
 ) util.JSONResponse {
 	ctx := req.Context()
 	logger := util.GetLogger(ctx).WithField("roomID", roomID).WithField("userID", device.UserID)
+
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"),
+		}
+	}
+
 	var membershipRes roomserverAPI.QueryMembershipForUserResponse
 	membershipReq := roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
-		UserID: device.UserID,
+		UserID: *deviceUserID,
 	}
-	err := rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes)
+	err = rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes)
 	if err != nil {
 		logger.WithError(err).Error("QueryMembershipForUser: could not query membership for user")
 		return util.JSONResponse{

clientapi/routing/profile.go

@@ -16,6 +16,7 @@ package routing
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"time"
 
@@ -104,12 +105,6 @@ func SetAvatarURL(
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
-	if r.AvatarURL == "" {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("'avatar_url' must be supplied."),
-		}
-	}
 
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
@@ -151,7 +146,7 @@ func SetAvatarURL(
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, evTime)
 	if err != nil {
 		return response
 	}
@@ -199,12 +194,6 @@ func SetDisplayName(
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
-	if r.DisplayName == "" {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("'displayname' must be supplied."),
-		}
-	}
 
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
@@ -246,7 +235,7 @@ func SetDisplayName(
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, evTime)
 	if err != nil {
 		return response
 	}
@@ -260,13 +249,17 @@ func SetDisplayName(
 func updateProfile(
 	ctx context.Context, rsAPI api.ClientRoomserverAPI, device *userapi.Device,
 	profile *authtypes.Profile,
-	userID string, cfg *config.ClientAPI, evTime time.Time,
+	userID string, evTime time.Time,
 ) (util.JSONResponse, error) {
-	var res api.QueryRoomsForUserResponse
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
-	err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
+	if err != nil {
-		UserID:         device.UserID,
+		return util.JSONResponse{
-		WantMembership: "join",
+			Code: http.StatusInternalServerError,
-	}, &res)
+			JSON: spec.Unknown("internal server error"),
+		}, err
+	}
+
+	rooms, err := rsAPI.QueryRoomsForUser(ctx, *deviceUserID, "join")
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("QueryRoomsForUser failed")
 		return util.JSONResponse{
@@ -275,6 +268,11 @@ func updateProfile(
 		}, err
 	}
 
+	roomIDStrs := make([]string, len(rooms))
+	for i, room := range rooms {
+		roomIDStrs[i] = room.String()
+	}
+
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed")
@@ -285,7 +283,7 @@ func updateProfile(
 	}
 
 	events, err := buildMembershipEvents(
-		ctx, device, res.RoomIDs, *profile, userID, cfg, evTime, rsAPI,
+		ctx, roomIDStrs, *profile, userID, evTime, rsAPI,
 	)
 	switch e := err.(type) {
 	case nil:
@@ -302,7 +300,7 @@ func updateProfile(
 		}, e
 	}
 
-	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil {
+	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, false); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -356,19 +354,33 @@ func getProfile(
 
 func buildMembershipEvents(
 	ctx context.Context,
-	device *userapi.Device,
 	roomIDs []string,
-	newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
+	newProfile authtypes.Profile, userID string,
 	evTime time.Time, rsAPI api.ClientRoomserverAPI,
 ) ([]*types.HeaderedEvent, error) {
 	evs := []*types.HeaderedEvent{}
 
+	fullUserID, err := spec.NewUserID(userID, true)
+	if err != nil {
+		return nil, err
+	}
 	for _, roomID := range roomIDs {
+		validRoomID, err := spec.NewRoomID(roomID)
+		if err != nil {
+			return nil, err
+		}
+		senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID)
+		if err != nil {
+			return nil, err
+		} else if senderID == nil {
+			return nil, fmt.Errorf("sender ID not found for %s in %s", *fullUserID, *validRoomID)
+		}
+		senderIDString := string(*senderID)
 		proto := gomatrixserverlib.ProtoEvent{
-			Sender:   userID,
+			SenderID: senderIDString,
 			RoomID:   roomID,
 			Type:     "m.room.member",
-			StateKey: &userID,
+			StateKey: &senderIDString,
 		}
 
 		content := gomatrixserverlib.MemberContent{
@@ -378,16 +390,21 @@ func buildMembershipEvents(
 		content.DisplayName = newProfile.DisplayName
 		content.AvatarURL = newProfile.AvatarURL
 
-		if err := proto.SetContent(content); err != nil {
+		if err = proto.SetContent(content); err != nil {
 			return nil, err
 		}
 
-		identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+		user, err := spec.NewUserID(userID, true)
 		if err != nil {
 			return nil, err
 		}
 
-		event, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, rsAPI, nil)
+		identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *user)
+		if err != nil {
+			return nil, err
+		}
+
+		event, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, evTime, rsAPI, nil)
 		if err != nil {
 			return nil, err
 		}

clientapi/routing/redaction.go

@@ -35,6 +35,7 @@ import (
 
 type redactionContent struct {
 	Reason  string `json:"reason"`
+	Redacts string `json:"redacts"`
 }
 
 type redactionResponse struct {
@@ -47,11 +48,43 @@ func SendRedaction(
 	txnID *string,
 	txnCache *transactions.Cache,
 ) util.JSONResponse {
-	resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	deviceUserID, userIDErr := spec.NewUserID(device.UserID, true)
+	if userIDErr != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("userID doesn't have power level to redact"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, queryErr := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
+	if queryErr != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("userID doesn't have power level to redact"),
+		}
+	}
+
+	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}
 
+	// if user is member of room, and sender ID is nil, then this user doesn't have a pseudo ID for some reason,
+	// which is unexpected.
+	if senderID == nil {
+		util.GetLogger(req.Context()).WithField("userID", *deviceUserID).WithField("roomID", roomID).Error("missing sender ID for user, despite having membership")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
 	if txnID != nil {
 		// Try to fetch response from transactionsCache
 		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok {
@@ -66,7 +99,7 @@ func SendRedaction(
 			JSON: spec.NotFound("unknown event ID"), // TODO: is it ok to leak existence?
 		}
 	}
-	if ev.RoomID() != roomID {
+	if ev.RoomID().String() != roomID {
 		return util.JSONResponse{
 			Code: 400,
 			JSON: spec.NotFound("cannot redact event in another room"),
@@ -76,7 +109,7 @@ func SendRedaction(
 	// "Users may redact their own events, and any user with a power level greater than or equal
 	// to the redact power level of the room may redact events there"
 	// https://matrix.org/docs/spec/client_server/r0.6.1#put-matrix-client-r0-rooms-roomid-redact-eventid-txnid
-	allowedToRedact := ev.Sender() == device.UserID
+	allowedToRedact := ev.SenderID() == *senderID
 	if !allowedToRedact {
 		plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
 			EventType: spec.MRoomPowerLevels,
@@ -88,8 +121,8 @@ func SendRedaction(
 				JSON: spec.Forbidden("You don't have permission to redact this event, no power_levels event in this room."),
 			}
 		}
-		pl, err := plEvent.PowerLevels()
+		pl, plErr := plEvent.PowerLevels()
-		if err != nil {
+		if plErr != nil {
 			return util.JSONResponse{
 				Code: 403,
 				JSON: spec.Forbidden(
@@ -97,7 +130,7 @@ func SendRedaction(
 				),
 			}
 		}
-		allowedToRedact = pl.UserLevel(device.UserID) >= pl.Redact
+		allowedToRedact = pl.UserLevel(*senderID) >= pl.Redact
 	}
 	if !allowedToRedact {
 		return util.JSONResponse{
@@ -114,12 +147,17 @@ func SendRedaction(
 
 	// create the new event and set all the fields we can
 	proto := gomatrixserverlib.ProtoEvent{
-		Sender:  device.UserID,
+		SenderID: string(*senderID),
 		RoomID:   roomID,
 		Type:     spec.MRoomRedaction,
 		Redacts:  eventID,
 	}
-	err := proto.SetContent(r)
+
+	// Room version 11 expects the "redacts" field on the
+	// content field, so add it here as well
+	r.Redacts = eventID
+
+	err = proto.SetContent(r)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("proto.SetContent failed")
 		return util.JSONResponse{
@@ -128,7 +166,7 @@ func SendRedaction(
 		}
 	}
 
-	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+	identity, err := rsAPI.SigningIdentityFor(req.Context(), *validRoomID, *deviceUserID)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -137,7 +175,7 @@ func SendRedaction(
 	}
 
 	var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
-	e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, identity, time.Now(), rsAPI, &queryRes)
+	e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, &identity, time.Now(), rsAPI, &queryRes)
 	if errors.Is(err, eventutil.ErrRoomNoExists{}) {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,

clientapi/routing/register.go

@@ -236,7 +236,7 @@ type authDict struct {
 	// TODO: Lots of custom keys depending on the type
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
+// https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api
 type userInteractiveResponse struct {
 	Flows     []authtypes.Flow       `json:"flows"`
 	Completed []authtypes.LoginType  `json:"completed"`
@@ -256,7 +256,7 @@ func newUserInteractiveResponse(
 	}
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 type registerResponse struct {
 	UserID      string `json:"user_id"`
 	AccessToken string `json:"access_token,omitempty"`
@@ -462,7 +462,7 @@ func validateApplicationService(
 }
 
 // Register processes a /register request.
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 func Register(
 	req *http.Request,
 	userAPI userapi.ClientUserAPI,

clientapi/routing/register_test.go

@@ -298,13 +298,16 @@ func Test_register(t *testing.T) {
 		guestsDisabled       bool
 		enableRecaptcha      bool
 		captchaBody          string
-		wantResponse         util.JSONResponse
+		// in case of an error, the expected response
+		wantErrorResponse util.JSONResponse
+		// in case of success, the expected username assigned
+		wantUsername string
 	}{
 		{
 			name:           "disallow guests",
 			kind:           "guest",
 			guestsDisabled: true,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: spec.Forbidden(`Guest registration is disabled on "test"`),
 			},
@@ -312,11 +315,12 @@ func Test_register(t *testing.T) {
 		{
 			name:         "allow guests",
 			kind:         "guest",
+			wantUsername: "1",
 		},
 		{
 			name:      "unknown login type",
 			loginType: "im.not.known",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusNotImplemented,
 				JSON: spec.Unknown("unknown/unimplemented auth type"),
 			},
@@ -324,7 +328,7 @@ func Test_register(t *testing.T) {
 		{
 			name:                 "disabled registration",
 			registrationDisabled: true,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: spec.Forbidden(`Registration is disabled on "test"`),
 			},
@@ -334,15 +338,23 @@ func Test_register(t *testing.T) {
 			username:     "",
 			password:     "someRandomPassword",
 			forceEmpty:   true,
+			wantUsername: "2",
 		},
 		{
 			name:     "successful registration",
 			username: "success",
 		},
+		{
+			name:         "successful registration, sequential numeric ID",
+			username:     "",
+			password:     "someRandomPassword",
+			forceEmpty:   true,
+			wantUsername: "3",
+		},
 		{
 			name:     "failing registration - user already exists",
 			username: "success",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.UserInUse("Desired user ID is already taken."),
 			},
@@ -354,12 +366,12 @@ func Test_register(t *testing.T) {
 		{
 			name:              "invalid username",
 			username:          "#totalyNotValid",
-			wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
+			wantErrorResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
 		},
 		{
 			name:     "numeric username is forbidden",
 			username: "1337",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.InvalidUsername("Numeric user IDs are reserved"),
 			},
@@ -367,7 +379,7 @@ func Test_register(t *testing.T) {
 		{
 			name:      "disabled recaptcha login",
 			loginType: authtypes.LoginTypeRecaptcha,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: spec.Unknown(ErrCaptchaDisabled.Error()),
 			},
@@ -376,7 +388,7 @@ func Test_register(t *testing.T) {
 			name:            "enabled recaptcha, no response defined",
 			enableRecaptcha: true,
 			loginType:       authtypes.LoginTypeRecaptcha,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.BadJSON(ErrMissingResponse.Error()),
 			},
@@ -386,7 +398,7 @@ func Test_register(t *testing.T) {
 			enableRecaptcha: true,
 			loginType:       authtypes.LoginTypeRecaptcha,
 			captchaBody:     `notvalid`,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusUnauthorized,
 				JSON: spec.BadJSON(ErrInvalidCaptcha.Error()),
 			},
@@ -402,7 +414,7 @@ func Test_register(t *testing.T) {
 			enableRecaptcha:   true,
 			loginType:         authtypes.LoginTypeRecaptcha,
 			captchaBody:       `i should fail for other reasons`,
-			wantResponse:    util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}},
+			wantErrorResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}},
 		},
 	}
 
@@ -415,7 +427,8 @@ func Test_register(t *testing.T) {
 
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
@@ -485,8 +498,8 @@ func Test_register(t *testing.T) {
 						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows)
 					}
 				case spec.MatrixError:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
+					if !reflect.DeepEqual(tc.wantErrorResponse, resp) {
-						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse)
+						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantErrorResponse)
 					}
 					return
 				case registerResponse:
@@ -504,6 +517,13 @@ func Test_register(t *testing.T) {
 					if r.DeviceID == "" {
 						t.Fatalf("missing deviceID in response")
 					}
+					// if an expected username is provided, assert that it is a match
+					if tc.wantUsername != "" {
+						wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test"))
+						if wantUserID != r.UserID {
+							t.Fatalf("unexpected userID: %s, want %s", r.UserID, wantUserID)
+						}
+					}
 					return
 				default:
 					t.Logf("Got response: %T", resp.JSON)
@@ -540,45 +560,30 @@ func Test_register(t *testing.T) {
 
 				resp = Register(req, userAPI, &cfg.ClientAPI)
 
-				switch resp.JSON.(type) {
+				switch rr := resp.JSON.(type) {
-				case spec.InternalServerError:
+				case spec.InternalServerError, spec.MatrixError, util.JSONResponse:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
+					if !reflect.DeepEqual(tc.wantErrorResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
+						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantErrorResponse)
 					}
 					return
-				case spec.MatrixError:
+				case registerResponse:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
-					}
-					return
-				case util.JSONResponse:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
-					}
-					return
-				}
-
-				rr, ok := resp.JSON.(registerResponse)
-				if !ok {
-					t.Fatalf("expected a registerresponse, got %T", resp.JSON)
-				}
-
 					// validate the response
-				if tc.forceEmpty {
+					if tc.wantUsername != "" {
-					// when not supplying a username, one will be generated. Given this _SHOULD_ be
+						// if an expected username is provided, assert that it is a match
-					// the second user, set the username accordingly
+						wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test"))
-					reg.Username = "2"
-				}
-				wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test"))
 						if wantUserID != rr.UserID {
 							t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID)
 						}
+					}
 					if rr.DeviceID != *reg.DeviceID {
 						t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID)
 					}
 					if rr.AccessToken == "" {
 						t.Fatalf("missing accessToken in response")
 					}
+				default:
+					t.Fatalf("expected one of internalservererror, matrixerror, jsonresponse, registerresponse, got %T", resp.JSON)
+				}
 			})
 		}
 	})
@@ -594,7 +599,8 @@ func TestRegisterUserWithDisplayName(t *testing.T) {
 		natsInstance := jetstream.NATSInstance{}
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		deviceName, deviceID := "deviceName", "deviceID"
 		expectedDisplayName := "DisplayName"
 		response := completeRegistration(
@@ -634,7 +640,8 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		expectedDisplayName := "rabbit"
 		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)

clientapi/routing/room_hierarchy.go

@@ -0,0 +1,180 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+	"strconv"
+	"sync"
+
+	"github.com/google/uuid"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+	log "github.com/sirupsen/logrus"
+)
+
+// For storing pagination information for room hierarchies
+type RoomHierarchyPaginationCache struct {
+	cache map[string]roomserverAPI.RoomHierarchyWalker
+	mu    sync.Mutex
+}
+
+// Create a new, empty, pagination cache.
+func NewRoomHierarchyPaginationCache() RoomHierarchyPaginationCache {
+	return RoomHierarchyPaginationCache{
+		cache: map[string]roomserverAPI.RoomHierarchyWalker{},
+	}
+}
+
+// Get a cached page, or nil if there is no associated page in the cache.
+func (c *RoomHierarchyPaginationCache) Get(token string) *roomserverAPI.RoomHierarchyWalker {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	line, ok := c.cache[token]
+	if ok {
+		return &line
+	} else {
+		return nil
+	}
+}
+
+// Add a cache line to the pagination cache.
+func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.RoomHierarchyWalker) string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	token := uuid.NewString()
+	c.cache[token] = line
+	return token
+}
+
+// Query the hierarchy of a room/space
+//
+// Implements /_matrix/client/v1/rooms/{roomID}/hierarchy
+func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr string, rsAPI roomserverAPI.ClientRoomserverAPI, paginationCache *RoomHierarchyPaginationCache) util.JSONResponse {
+	parsedRoomID, err := spec.NewRoomID(roomIDStr)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.InvalidParam("room is unknown/forbidden"),
+		}
+	}
+	roomID := *parsedRoomID
+
+	suggestedOnly := false // Defaults to false (spec-defined)
+	switch req.URL.Query().Get("suggested_only") {
+	case "true":
+		suggestedOnly = true
+	case "false":
+	case "": // Empty string is returned when query param is not set
+	default:
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"),
+		}
+	}
+
+	limit := 1000 // Default to 1000
+	limitStr := req.URL.Query().Get("limit")
+	if limitStr != "" {
+		var maybeLimit int
+		maybeLimit, err = strconv.Atoi(limitStr)
+		if err != nil || maybeLimit < 0 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("query parameter 'limit', if set, must be a positive integer"),
+			}
+		}
+		limit = maybeLimit
+		if limit > 1000 {
+			limit = 1000 // Maximum limit of 1000
+		}
+	}
+
+	maxDepth := -1 // '-1' representing no maximum depth
+	maxDepthStr := req.URL.Query().Get("max_depth")
+	if maxDepthStr != "" {
+		var maybeMaxDepth int
+		maybeMaxDepth, err = strconv.Atoi(maxDepthStr)
+		if err != nil || maybeMaxDepth < 0 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("query parameter 'max_depth', if set, must be a positive integer"),
+			}
+		}
+		maxDepth = maybeMaxDepth
+	}
+
+	from := req.URL.Query().Get("from")
+
+	var walker roomserverAPI.RoomHierarchyWalker
+	if from == "" { // No pagination token provided, so start new hierarchy walker
+		walker = roomserverAPI.NewRoomHierarchyWalker(types.NewDeviceNotServerName(*device), roomID, suggestedOnly, maxDepth)
+	} else { // Attempt to resume cached walker
+		cachedWalker := paginationCache.Get(from)
+
+		if cachedWalker == nil || cachedWalker.SuggestedOnly != suggestedOnly || cachedWalker.MaxDepth != maxDepth {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("pagination not found for provided token ('from') with given 'max_depth', 'suggested_only' and room ID"),
+			}
+		}
+
+		walker = *cachedWalker
+	}
+
+	discoveredRooms, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit)
+
+	if err != nil {
+		switch err.(type) {
+		case roomserverAPI.ErrRoomUnknownOrNotAllowed:
+			util.GetLogger(req.Context()).WithError(err).Debugln("room unknown/forbidden when handling CS room hierarchy request")
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.Forbidden("room is unknown/forbidden"),
+			}
+		default:
+			log.WithError(err).Errorf("failed to fetch next page of room hierarchy (CS API)")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+	}
+
+	nextBatch := ""
+	// nextWalker will be nil if there's no more rooms left to walk
+	if nextWalker != nil {
+		nextBatch = paginationCache.AddLine(*nextWalker)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: RoomHierarchyClientResponse{
+			Rooms:     discoveredRooms,
+			NextBatch: nextBatch,
+		},
+	}
+
+}
+
+// Success response for /_matrix/client/v1/rooms/{roomID}/hierarchy
+type RoomHierarchyClientResponse struct {
+	Rooms     []fclient.RoomHierarchyRoom `json:"rooms"`
+	NextBatch string                      `json:"next_batch,omitempty"`
+}

clientapi/routing/routing.go

@@ -44,6 +44,19 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 )
 
+type WellKnownClientHomeserver struct {
+	BaseUrl string `json:"base_url"`
+}
+
+type WellKnownSlidingSyncProxy struct {
+	Url string `json:"url"`
+}
+
+type WellKnownClientResponse struct {
+	Homeserver       WellKnownClientHomeserver  `json:"m.homeserver"`
+	SlidingSyncProxy *WellKnownSlidingSyncProxy `json:"org.matrix.msc3575.proxy,omitempty"`
+}
+
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
 //
@@ -96,20 +109,22 @@ func Setup(
 
 	if cfg.Matrix.WellKnownClientName != "" {
 		logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName)
+		if cfg.Matrix.WellKnownSlidingSyncProxy != "" {
+			logrus.Infof("Setting org.matrix.msc3575.proxy url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownSlidingSyncProxy)
+		}
 		wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse {
+			response := WellKnownClientResponse{
+				Homeserver: WellKnownClientHomeserver{cfg.Matrix.WellKnownClientName},
+			}
+			if cfg.Matrix.WellKnownSlidingSyncProxy != "" {
+				response.SlidingSyncProxy = &WellKnownSlidingSyncProxy{
+					Url: cfg.Matrix.WellKnownSlidingSyncProxy,
+				}
+			}
+
 			return util.JSONResponse{
 				Code: http.StatusOK,
-				JSON: struct {
+				JSON: response,
-					HomeserverName struct {
-						BaseUrl string `json:"base_url"`
-					} `json:"m.homeserver"`
-				}{
-					HomeserverName: struct {
-						BaseUrl string `json:"base_url"`
-					}{
-						BaseUrl: cfg.Matrix.WellKnownClientName,
-					},
-				},
 			}
 		})).Methods(http.MethodGet, http.MethodOptions)
 	}
@@ -162,6 +177,36 @@ func Setup(
 			}),
 		).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 	}
+	dendriteAdminRouter.Handle("/admin/registrationTokens/new",
+		httputil.MakeAdminAPI("admin_registration_tokens_new", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminCreateNewRegistrationToken(req, cfg, userAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	dendriteAdminRouter.Handle("/admin/registrationTokens",
+		httputil.MakeAdminAPI("admin_list_registration_tokens", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminListRegistrationTokens(req, cfg, userAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	dendriteAdminRouter.Handle("/admin/registrationTokens/{token}",
+		httputil.MakeAdminAPI("admin_get_registration_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			switch req.Method {
+			case http.MethodGet:
+				return AdminGetRegistrationToken(req, cfg, userAPI)
+			case http.MethodPut:
+				return AdminUpdateRegistrationToken(req, cfg, userAPI)
+			case http.MethodDelete:
+				return AdminDeleteRegistrationToken(req, cfg, userAPI)
+			default:
+				return util.MatrixErrorResponse(
+					404,
+					string(spec.ErrorNotFound),
+					"unknown method",
+				)
+			}
+		}),
+	).Methods(http.MethodGet, http.MethodPut, http.MethodDelete, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
 		httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -258,6 +303,8 @@ func Setup(
 	// Note that 'apiversion' is chosen because it must not collide with a variable used in any of the routing!
 	v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
 
+	v1mux := publicAPIMux.PathPrefix("/v1/").Subrouter()
+
 	unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter()
 
 	v3mux.Handle("/createRoom",
@@ -475,6 +522,19 @@ func Setup(
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPut, http.MethodOptions)
 
+	// Defined outside of handler to persist between calls
+	// TODO: clear based on some criteria
+	roomHierarchyPaginationCache := NewRoomHierarchyPaginationCache()
+	v1mux.Handle("/rooms/{roomID}/hierarchy",
+		httputil.MakeAuthAPI("spaces", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return QueryRoomHierarchy(req, device, vars["roomID"], rsAPI, &roomHierarchyPaginationCache)
+		}, httputil.WithAllowGuests()),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
 		if r := rateLimits.Limit(req, nil); r != nil {
 			return *r
@@ -1211,7 +1271,7 @@ func Setup(
 			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
-			return GetCapabilities()
+			return GetCapabilities(rsAPI)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodGet, http.MethodOptions)
 

clientapi/routing/sendevent.go

@@ -23,19 +23,19 @@ import (
 	"sync"
 	"time"
 
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
-	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
-
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/synctypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 )
 
 // http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid
@@ -68,6 +68,8 @@ var sendEventDuration = prometheus.NewHistogramVec(
 //	/rooms/{roomID}/send/{eventType}
 //	/rooms/{roomID}/send/{eventType}/{txnID}
 //	/rooms/{roomID}/state/{eventType}/{stateKey}
+//
+// nolint: gocyclo
 func SendEvent(
 	req *http.Request,
 	device *userapi.Device,
@@ -91,6 +93,30 @@ func SendEvent(
 		}
 	}
 
+	// Translate user ID state keys to room keys in pseudo ID rooms
+	if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && stateKey != nil {
+		parsedRoomID, innerErr := spec.NewRoomID(roomID)
+		if innerErr != nil {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("invalid room ID"),
+			}
+		}
+
+		newStateKey, innerErr := synctypes.FromClientStateKey(*parsedRoomID, *stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+			return rsAPI.QuerySenderIDForUser(req.Context(), roomID, userID)
+		})
+		if innerErr != nil {
+			// TODO: work out better logic for failure cases (e.g. sender ID not found)
+			util.GetLogger(req.Context()).WithError(innerErr).Error("synctypes.FromClientStateKey failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+		stateKey = newStateKey
+	}
+
 	// create a mutex for the specific user in the specific room
 	// this avoids a situation where events that are received in quick succession are sent to the roomserver in a jumbled order
 	userID := device.UserID
@@ -121,6 +147,17 @@ func SendEvent(
 		delete(r, "join_authorised_via_users_server")
 	}
 
+	// for power level events we need to replace the userID with the pseudoID
+	if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && eventType == spec.MRoomPowerLevels {
+		err = updatePowerLevels(req, r, roomID, rsAPI)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{Err: err.Error()},
+			}
+		}
+	}
+
 	evTime, err := httputil.ParseTSParam(req)
 	if err != nil {
 		return util.JSONResponse{
@@ -129,7 +166,7 @@ func SendEvent(
 		}
 	}
 
-	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, cfg, rsAPI, evTime)
+	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, rsAPI, evTime)
 	if resErr != nil {
 		return *resErr
 	}
@@ -225,6 +262,35 @@ func SendEvent(
 	return res
 }
 
+func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID string, rsAPI api.ClientRoomserverAPI) error {
+	users, ok := r["users"]
+	if !ok {
+		return nil
+	}
+	userMap := users.(map[string]interface{})
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return err
+	}
+	for user, level := range userMap {
+		uID, err := spec.NewUserID(user, true)
+		if err != nil {
+			continue // we're modifying the map in place, so we're going to have invalid userIDs after the first iteration
+		}
+		senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *uID)
+		if err != nil {
+			return err
+		} else if senderID == nil {
+			util.GetLogger(req.Context()).Warnf("sender ID not found for %s in %s", uID, *validRoomID)
+			continue
+		}
+		userMap[string(*senderID)] = level
+		delete(userMap, user)
+	}
+	r["users"] = userMap
+	return nil
+}
+
 // stateEqual compares the new and the existing state event content. If they are equal, returns a *util.JSONResponse
 // with the existing event_id, making this an idempotent request.
 func stateEqual(ctx context.Context, rsAPI api.ClientRoomserverAPI, eventType, stateKey, roomID string, newContent map[string]interface{}) *util.JSONResponse {
@@ -261,21 +327,47 @@ func generateSendEvent(
 	r map[string]interface{},
 	device *userapi.Device,
 	roomID, eventType string, stateKey *string,
-	cfg *config.ClientAPI,
 	rsAPI api.ClientRoomserverAPI,
 	evTime time.Time,
 ) (gomatrixserverlib.PDU, *util.JSONResponse) {
 	// parse the incoming http request
-	userID := device.UserID
+	fullUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("Bad userID"),
+		}
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("RoomID is invalid"),
+		}
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.NotFound("internal server error"),
+		}
+	} else if senderID == nil {
+		// TODO: is it always the case that lack of a sender ID means they're not joined?
+		//       And should this logic be deferred to the roomserver somehow?
+		return nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("not joined to room"),
+		}
+	}
 
 	// create the new event and set all the fields we can
 	proto := gomatrixserverlib.ProtoEvent{
-		Sender:   userID,
+		SenderID: string(*senderID),
 		RoomID:   roomID,
 		Type:     eventType,
 		StateKey: stateKey,
 	}
-	err := proto.SetContent(r)
+	err = proto.SetContent(r)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("proto.SetContent failed")
 		return nil, &util.JSONResponse{
@@ -284,7 +376,7 @@ func generateSendEvent(
 		}
 	}
 
-	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+	identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *fullUserID)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -293,7 +385,7 @@ func generateSendEvent(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, rsAPI, &queryRes)
+	e, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, evTime, rsAPI, &queryRes)
 	switch specificErr := err.(type) {
 	case nil:
 	case eventutil.ErrRoomNoExists:
@@ -331,7 +423,9 @@ func generateSendEvent(
 		stateEvents[i] = queryRes.StateEvents[i].PDU
 	}
 	provider := gomatrixserverlib.NewAuthEvents(gomatrixserverlib.ToPDUs(stateEvents))
-	if err = gomatrixserverlib.Allowed(e.PDU, &provider); err != nil {
+	if err = gomatrixserverlib.Allowed(e.PDU, &provider, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return rsAPI.QueryUserIDForSender(ctx, *validRoomID, senderID)
+	}); err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: spec.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
@@ -348,7 +442,7 @@ func generateSendEvent(
 				JSON: spec.BadJSON("Cannot unmarshal the event content."),
 			}
 		}
-		if content["replacement_room"] == e.RoomID() {
+		if content["replacement_room"] == e.RoomID().String() {
 			return nil, &util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.InvalidParam("Cannot send tombstone event that points to the same room."),

clientapi/routing/sendevent_test.go

@@ -0,0 +1,275 @@
+package routing
+
+import (
+	"context"
+	"crypto/ed25519"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"gotest.tools/v3/assert"
+)
+
+// Mock roomserver API for testing
+//
+// Currently pretty specialised for the pseudo ID test, so will need
+// editing if future (other) sendevent tests are using this.
+type sendEventTestRoomserverAPI struct {
+	rsapi.ClientRoomserverAPI
+	t           *testing.T
+	roomIDStr   string
+	roomVersion gomatrixserverlib.RoomVersion
+	roomState   []*types.HeaderedEvent
+
+	// userID -> room key
+	senderMapping map[string]ed25519.PrivateKey
+
+	savedInputRoomEvents []rsapi.InputRoomEvent
+}
+
+func (s *sendEventTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) {
+	if roomID == s.roomIDStr {
+		return s.roomVersion, nil
+	} else {
+		s.t.Logf("room version queried for %s", roomID)
+		return "", fmt.Errorf("unknown room")
+	}
+}
+
+func (s *sendEventTestRoomserverAPI) QueryCurrentState(ctx context.Context, req *rsapi.QueryCurrentStateRequest, res *rsapi.QueryCurrentStateResponse) error {
+	res.StateEvents = map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{}
+	for _, stateKeyTuple := range req.StateTuples {
+		for _, stateEv := range s.roomState {
+			if stateEv.Type() == stateKeyTuple.EventType && stateEv.StateKey() != nil && *stateEv.StateKey() == stateKeyTuple.StateKey {
+				res.StateEvents[stateKeyTuple] = stateEv
+			}
+		}
+	}
+	return nil
+}
+
+func (s *sendEventTestRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error {
+	if req.RoomID == s.roomIDStr {
+		res.RoomExists = true
+		res.RoomVersion = s.roomVersion
+
+		res.StateEvents = make([]*types.HeaderedEvent, len(s.roomState))
+		copy(res.StateEvents, s.roomState)
+
+		res.LatestEvents = []string{}
+		res.Depth = 1
+		return nil
+	} else {
+		s.t.Logf("room event/state queried for %s", req.RoomID)
+		return fmt.Errorf("unknown room")
+	}
+
+}
+
+func (s *sendEventTestRoomserverAPI) QuerySenderIDForUser(
+	ctx context.Context,
+	roomID spec.RoomID,
+	userID spec.UserID,
+) (*spec.SenderID, error) {
+	if roomID.String() == s.roomIDStr {
+		if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs {
+			roomKey, ok := s.senderMapping[userID.String()]
+			if ok {
+				sender := spec.SenderIDFromPseudoIDKey(roomKey)
+				return &sender, nil
+			} else {
+				return nil, nil
+			}
+		} else {
+			senderID := spec.SenderIDFromUserID(userID)
+			return &senderID, nil
+		}
+	}
+
+	return nil, fmt.Errorf("room not found")
+}
+
+func (s *sendEventTestRoomserverAPI) QueryUserIDForSender(
+	ctx context.Context,
+	roomID spec.RoomID,
+	senderID spec.SenderID,
+) (*spec.UserID, error) {
+	if roomID.String() == s.roomIDStr {
+		if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs {
+			for uID, roomKey := range s.senderMapping {
+				if string(spec.SenderIDFromPseudoIDKey(roomKey)) == string(senderID) {
+					parsedUserID, err := spec.NewUserID(uID, true)
+					if err != nil {
+						s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err)
+					}
+					return parsedUserID, nil
+				}
+			}
+		} else {
+			userID := senderID.ToUserID()
+			if userID == nil {
+				return nil, fmt.Errorf("bad sender ID")
+			}
+			return userID, nil
+		}
+	}
+
+	return nil, fmt.Errorf("room not found")
+}
+
+func (s *sendEventTestRoomserverAPI) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, sender spec.UserID) (fclient.SigningIdentity, error) {
+	if s.roomIDStr == roomID.String() {
+		if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs {
+			roomKey, ok := s.senderMapping[sender.String()]
+			if !ok {
+				s.t.Logf("SigningIdentityFor used with unknown user ID: %v", sender.String())
+				return fclient.SigningIdentity{}, fmt.Errorf("could not get signing identity for %v", sender.String())
+			}
+			return fclient.SigningIdentity{PrivateKey: roomKey}, nil
+		} else {
+			return fclient.SigningIdentity{PrivateKey: ed25519.NewKeyFromSeed(make([]byte, 32))}, nil
+		}
+	}
+
+	return fclient.SigningIdentity{}, fmt.Errorf("room not found")
+}
+
+func (s *sendEventTestRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
+	s.savedInputRoomEvents = req.InputRoomEvents
+}
+
+// Test that user ID state keys are translated correctly
+func Test_SendEvent_PseudoIDStateKeys(t *testing.T) {
+	nonpseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10
+	pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs
+
+	senderKeySeed := make([]byte, 32)
+	senderUserID := "@testuser:domain"
+	senderPrivKey := ed25519.NewKeyFromSeed(senderKeySeed)
+	senderPseudoID := string(spec.SenderIDFromPseudoIDKey(senderPrivKey))
+
+	eventType := "com.example.test"
+	roomIDStr := "!id:domain"
+
+	device := &uapi.Device{
+		UserID: senderUserID,
+	}
+
+	t.Run("user ID state key are not translated to room key in non-pseudo ID room", func(t *testing.T) {
+		eventsJSON := []string{
+			fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderUserID, senderUserID, nonpseudoIDRoomVersion),
+			fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderUserID, roomIDStr, senderUserID),
+		}
+
+		roomState, err := createEvents(eventsJSON, nonpseudoIDRoomVersion)
+		if err != nil {
+			t.Fatalf("failed to prepare state events: %s", err.Error())
+		}
+
+		rsAPI := &sendEventTestRoomserverAPI{
+			t:           t,
+			roomIDStr:   roomIDStr,
+			roomVersion: nonpseudoIDRoomVersion,
+			roomState:   roomState,
+		}
+
+		req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}")))
+		if err != nil {
+			t.Fatalf("failed to make new request: %s", err.Error())
+		}
+
+		cfg := &config.ClientAPI{}
+
+		resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil)
+
+		if resp.Code != http.StatusOK {
+			t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp)
+		}
+
+		assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1)
+
+		ev := rsAPI.savedInputRoomEvents[0]
+		stateKey := ev.Event.StateKey()
+		if stateKey == nil {
+			t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderUserID)
+		}
+		if *stateKey != senderUserID {
+			t.Fatalf("expected submitted InputRoomEvent to have user ID state key\nfound: %v\nexpected: %v", *stateKey, senderUserID)
+		}
+	})
+
+	t.Run("user ID state key are translated to room key in pseudo ID room", func(t *testing.T) {
+		eventsJSON := []string{
+			fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderPseudoID, senderPseudoID, pseudoIDRoomVersion),
+			fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderPseudoID, roomIDStr, senderPseudoID),
+		}
+
+		roomState, err := createEvents(eventsJSON, pseudoIDRoomVersion)
+		if err != nil {
+			t.Fatalf("failed to prepare state events: %s", err.Error())
+		}
+
+		rsAPI := &sendEventTestRoomserverAPI{
+			t:           t,
+			roomIDStr:   roomIDStr,
+			roomVersion: pseudoIDRoomVersion,
+			senderMapping: map[string]ed25519.PrivateKey{
+				senderUserID: senderPrivKey,
+			},
+			roomState: roomState,
+		}
+
+		req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}")))
+		if err != nil {
+			t.Fatalf("failed to make new request: %s", err.Error())
+		}
+
+		cfg := &config.ClientAPI{}
+
+		resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil)
+
+		if resp.Code != http.StatusOK {
+			t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp)
+		}
+
+		assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1)
+
+		ev := rsAPI.savedInputRoomEvents[0]
+		stateKey := ev.Event.StateKey()
+		if stateKey == nil {
+			t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderPseudoID)
+		}
+		if *stateKey != senderPseudoID {
+			t.Fatalf("expected submitted InputRoomEvent to have pseudo ID state key\nfound: %v\nexpected: %v", *stateKey, senderPseudoID)
+		}
+	})
+}
+
+func createEvents(eventsJSON []string, roomVer gomatrixserverlib.RoomVersion) ([]*types.HeaderedEvent, error) {
+	events := make([]*types.HeaderedEvent, len(eventsJSON))
+
+	roomVerImpl, err := gomatrixserverlib.GetRoomVersion(roomVer)
+	if err != nil {
+		return nil, fmt.Errorf("no roomver impl: %s", err.Error())
+	}
+
+	for i, eventJSON := range eventsJSON {
+		pdu, evErr := roomVerImpl.NewEventFromTrustedJSON([]byte(eventJSON), false)
+		if evErr != nil {
+			return nil, fmt.Errorf("failed to make event: %s", err.Error())
+		}
+		ev := types.HeaderedEvent{PDU: pdu}
+		events[i] = &ev
+	}
+
+	return events, nil
+}

clientapi/routing/sendtyping.go

@@ -43,8 +43,16 @@ func SendTyping(
 		}
 	}
 
+	deviceUserID, err := spec.NewUserID(userID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
+		}
+	}
+
 	// Verify that the user is a member of this room
-	resErr := checkMemberInRoom(req.Context(), rsAPI, userID, roomID)
+	resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}

clientapi/routing/server_notices.go

@@ -28,7 +28,6 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/dendrite/roomserver/version"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -52,6 +51,7 @@ type sendServerNoticeRequest struct {
 	StateKey string `json:"state_key,omitempty"`
 }
 
+// nolint:gocyclo
 // SendServerNotice sends a message to a specific user. It can only be invoked by an admin.
 func SendServerNotice(
 	req *http.Request,
@@ -94,34 +94,42 @@ func SendServerNotice(
 		}
 	}
 
+	userID, err := spec.NewUserID(r.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("invalid user ID"),
+		}
+	}
+
 	// get rooms for specified user
-	allUserRooms := []string{}
+	allUserRooms := []spec.RoomID{}
-	userRooms := api.QueryRoomsForUserResponse{}
 	// Get rooms the user is either joined, invited or has left.
 	for _, membership := range []string{"join", "invite", "leave"} {
-		if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
+		userRooms, queryErr := rsAPI.QueryRoomsForUser(ctx, *userID, membership)
-			UserID:         r.UserID,
+		if queryErr != nil {
-			WantMembership: membership,
-		}, &userRooms); err != nil {
 			return util.ErrorResponse(err)
 		}
-		allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
+		allUserRooms = append(allUserRooms, userRooms...)
 	}
 
 	// get rooms of the sender
-	senderUserID := fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName)
+	senderUserID, err := spec.NewUserID(fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName), true)
-	senderRooms := api.QueryRoomsForUserResponse{}
+	if err != nil {
-	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
+		return util.JSONResponse{
-		UserID:         senderUserID,
+			Code: http.StatusInternalServerError,
-		WantMembership: "join",
+			JSON: spec.Unknown("internal server error"),
-	}, &senderRooms); err != nil {
+		}
+	}
+	senderRooms, err := rsAPI.QueryRoomsForUser(ctx, *senderUserID, "join")
+	if err != nil {
 		return util.ErrorResponse(err)
 	}
 
 	// check if we have rooms in common
-	commonRooms := []string{}
+	commonRooms := []spec.RoomID{}
 	for _, userRoomID := range allUserRooms {
-		for _, senderRoomID := range senderRooms.RoomIDs {
+		for _, senderRoomID := range senderRooms {
 			if userRoomID == senderRoomID {
 				commonRooms = append(commonRooms, senderRoomID)
 			}
@@ -134,12 +142,12 @@ func SendServerNotice(
 
 	var (
 		roomID      string
-		roomVersion = version.DefaultRoomVersion()
+		roomVersion = rsAPI.DefaultRoomVersion()
 	)
 
 	// create a new room for the user
 	if len(commonRooms) == 0 {
-		powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID)
+		powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID.String())
 		powerLevelContent.Users[r.UserID] = -10 // taken from Synapse
 		pl, err := json.Marshal(powerLevelContent)
 		if err != nil {
@@ -187,9 +195,17 @@ func SendServerNotice(
 		}
 	} else {
 		// we've found a room in common, check the membership
-		roomID = commonRooms[0]
+		deviceUserID, err := spec.NewUserID(r.UserID, true)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
+			}
+		}
+
+		roomID = commonRooms[0].String()
 		membershipRes := api.QueryMembershipForUserResponse{}
-		err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: r.UserID, RoomID: roomID}, &membershipRes)
+		err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: *deviceUserID, RoomID: roomID}, &membershipRes)
 		if err != nil {
 			util.GetLogger(ctx).WithError(err).Error("unable to query membership for user")
 			return util.JSONResponse{
@@ -212,7 +228,7 @@ func SendServerNotice(
 		"body":    r.Content.Body,
 		"msgtype": r.Content.MsgType,
 	}
-	e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, cfgClient, rsAPI, time.Now())
+	e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, rsAPI, time.Now())
 	if resErr != nil {
 		logrus.Errorf("failed to send message: %+v", resErr)
 		return *resErr
@@ -234,7 +250,7 @@ func SendServerNotice(
 		ctx, rsAPI,
 		api.KindNew,
 		[]*types.HeaderedEvent{
-			&types.HeaderedEvent{PDU: e},
+			{PDU: e},
 		},
 		device.UserDomain(),
 		cfgClient.Matrix.ServerName,
@@ -341,7 +357,7 @@ func getSenderDevice(
 	if len(deviceRes.Devices) > 0 {
 		// If there were changes to the profile, create a new membership event
 		if displayNameChanged || avatarChanged {
-			_, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, cfg, time.Now())
+			_, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, time.Now())
 			if err != nil {
 				return nil, err
 			}

clientapi/routing/state.go

@@ -99,9 +99,17 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 	if !worldReadable {
 		// The room isn't world-readable so try to work out based on the
 		// user's membership if we want the latest state or not.
-		err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{
+		userID, err := spec.NewUserID(device.UserID, true)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("UserID is invalid")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.Unknown("Device UserID is invalid"),
+			}
+		}
+		err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{
 			RoomID: roomID,
-			UserID: device.UserID,
+			UserID: *userID,
 		}, &membershipRes)
 		if err != nil {
 			util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser")
@@ -142,7 +150,9 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		for _, ev := range stateRes.StateEvents {
 			stateEvents = append(
 				stateEvents,
-				synctypes.ToClientEvent(ev, synctypes.FormatAll),
+				synctypes.ToClientEventDefault(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+					return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+				}, ev),
 			)
 		}
 	} else {
@@ -162,9 +172,16 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 			}
 		}
 		for _, ev := range stateAfterRes.StateEvents {
+			clientEvent, err := synctypes.ToClientEvent(ev, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+				return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+			})
+			if err != nil {
+				util.GetLogger(ctx).WithError(err).Error("Failed converting to ClientEvent")
+				continue
+			}
 			stateEvents = append(
 				stateEvents,
-				synctypes.ToClientEvent(ev, synctypes.FormatAll),
+				*clientEvent,
 			)
 		}
 	}
@@ -188,6 +205,37 @@ func OnIncomingStateTypeRequest(
 	var worldReadable bool
 	var wantLatestState bool
 
+	roomVer, err := rsAPI.QueryRoomVersionForRoom(ctx, roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
+		}
+	}
+
+	// Translate user ID state keys to room keys in pseudo ID rooms
+	if roomVer == gomatrixserverlib.RoomVersionPseudoIDs {
+		parsedRoomID, err := spec.NewRoomID(roomID)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: spec.InvalidParam("invalid room ID"),
+			}
+		}
+		newStateKey, err := synctypes.FromClientStateKey(*parsedRoomID, stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+			return rsAPI.QuerySenderIDForUser(ctx, roomID, userID)
+		})
+		if err != nil {
+			// TODO: work out better logic for failure cases (e.g. sender ID not found)
+			util.GetLogger(ctx).WithError(err).Error("synctypes.FromClientStateKey failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+		stateKey = *newStateKey
+	}
+
 	// Always fetch visibility so that we can work out whether to show
 	// the latest events or the last event from when the user was joined.
 	// Then include the requested event type and state key, assuming it
@@ -249,11 +297,19 @@ func OnIncomingStateTypeRequest(
 	// membershipRes will only be populated if the room is not world-readable.
 	var membershipRes api.QueryMembershipForUserResponse
 	if !worldReadable {
+		userID, err := spec.NewUserID(device.UserID, true)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("UserID is invalid")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.Unknown("Device UserID is invalid"),
+			}
+		}
 		// The room isn't world-readable so try to work out based on the
 		// user's membership if we want the latest state or not.
-		err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{
+		err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{
 			RoomID: roomID,
-			UserID: device.UserID,
+			UserID: *userID,
 		}, &membershipRes)
 		if err != nil {
 			util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser")
@@ -335,7 +391,9 @@ func OnIncomingStateTypeRequest(
 	}
 
 	stateEvent := stateEventInStateResp{
-		ClientEvent: synctypes.ToClientEvent(event, synctypes.FormatAll),
+		ClientEvent: synctypes.ToClientEventDefault(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+		}, event),
 	}
 
 	var res interface{}

clientapi/routing/state_test.go

@@ -0,0 +1,253 @@
+package routing
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"testing"
+
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+	"gotest.tools/v3/assert"
+)
+
+var ()
+
+type stateTestRoomserverAPI struct {
+	rsapi.RoomserverInternalAPI
+	t           *testing.T
+	roomState   map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent
+	roomIDStr   string
+	roomVersion gomatrixserverlib.RoomVersion
+	userIDStr   string
+	// userID -> senderID
+	senderMapping map[string]string
+}
+
+func (s stateTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) {
+	if roomID == s.roomIDStr {
+		return s.roomVersion, nil
+	} else {
+		s.t.Logf("room version queried for %s", roomID)
+		return "", fmt.Errorf("unknown room")
+	}
+}
+
+func (s stateTestRoomserverAPI) QueryLatestEventsAndState(
+	ctx context.Context,
+	req *rsapi.QueryLatestEventsAndStateRequest,
+	res *rsapi.QueryLatestEventsAndStateResponse,
+) error {
+	res.RoomExists = req.RoomID == s.roomIDStr
+	if !res.RoomExists {
+		return nil
+	}
+
+	res.StateEvents = []*types.HeaderedEvent{}
+	for _, stateKeyTuple := range req.StateToFetch {
+		val, ok := s.roomState[stateKeyTuple]
+		if ok && val != nil {
+			res.StateEvents = append(res.StateEvents, val)
+		}
+	}
+
+	return nil
+}
+
+func (s stateTestRoomserverAPI) QueryMembershipForUser(
+	ctx context.Context,
+	req *rsapi.QueryMembershipForUserRequest,
+	res *rsapi.QueryMembershipForUserResponse,
+) error {
+	if req.UserID.String() == s.userIDStr {
+		res.HasBeenInRoom = true
+		res.IsInRoom = true
+		res.RoomExists = true
+		res.Membership = spec.Join
+	}
+
+	return nil
+}
+
+func (s stateTestRoomserverAPI) QuerySenderIDForUser(
+	ctx context.Context,
+	roomID spec.RoomID,
+	userID spec.UserID,
+) (*spec.SenderID, error) {
+	sID, ok := s.senderMapping[userID.String()]
+	if ok {
+		sender := spec.SenderID(sID)
+		return &sender, nil
+	} else {
+		return nil, nil
+	}
+}
+
+func (s stateTestRoomserverAPI) QueryUserIDForSender(
+	ctx context.Context,
+	roomID spec.RoomID,
+	senderID spec.SenderID,
+) (*spec.UserID, error) {
+	for uID, sID := range s.senderMapping {
+		if sID == string(senderID) {
+			parsedUserID, err := spec.NewUserID(uID, true)
+			if err != nil {
+				s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err)
+			}
+			return parsedUserID, nil
+		}
+	}
+	return nil, nil
+}
+
+func (s stateTestRoomserverAPI) QueryStateAfterEvents(
+	ctx context.Context,
+	req *rsapi.QueryStateAfterEventsRequest,
+	res *rsapi.QueryStateAfterEventsResponse,
+) error {
+	return nil
+}
+
+func Test_OnIncomingStateTypeRequest(t *testing.T) {
+	var tempRoomServerCfg config.RoomServer
+	tempRoomServerCfg.Defaults(config.DefaultOpts{})
+	defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion
+	pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs
+	nonPseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10
+
+	userIDStr := "@testuser:domain"
+	eventType := "com.example.test"
+	stateKey := "testStateKey"
+	roomIDStr := "!id:domain"
+
+	device := &uapi.Device{
+		UserID: userIDStr,
+	}
+
+	t.Run("request simple state key", func(t *testing.T) {
+		ctx := context.Background()
+
+		rsAPI := stateTestRoomserverAPI{
+			roomVersion: defaultRoomVersion,
+			roomIDStr:   roomIDStr,
+			roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{
+				{
+					EventType: eventType,
+					StateKey:  stateKey,
+				}: mustCreateStatePDU(t, defaultRoomVersion, roomIDStr, eventType, stateKey, map[string]interface{}{
+					"foo": "bar",
+				}),
+			},
+			userIDStr: userIDStr,
+		}
+
+		jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateKey, false)
+
+		assert.DeepEqual(t, jsonResp, util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: spec.RawJSON(`{"foo":"bar"}`),
+		})
+	})
+
+	t.Run("user ID key translated to room key in pseudo ID rooms", func(t *testing.T) {
+		ctx := context.Background()
+
+		stateSenderUserID := "@sender:domain"
+		stateSenderRoomKey := "testsenderkey"
+
+		rsAPI := stateTestRoomserverAPI{
+			roomVersion: pseudoIDRoomVersion,
+			roomIDStr:   roomIDStr,
+			roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{
+				{
+					EventType: eventType,
+					StateKey:  stateSenderRoomKey,
+				}: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{
+					"foo": "bar",
+				}),
+				{
+					EventType: eventType,
+					StateKey:  stateSenderUserID,
+				}: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{
+					"not": "thisone",
+				}),
+			},
+			userIDStr: userIDStr,
+			senderMapping: map[string]string{
+				stateSenderUserID: stateSenderRoomKey,
+			},
+		}
+
+		jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false)
+
+		assert.DeepEqual(t, jsonResp, util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: spec.RawJSON(`{"foo":"bar"}`),
+		})
+	})
+
+	t.Run("user ID key not translated to room key in non-pseudo ID rooms", func(t *testing.T) {
+		ctx := context.Background()
+
+		stateSenderUserID := "@sender:domain"
+		stateSenderRoomKey := "testsenderkey"
+
+		rsAPI := stateTestRoomserverAPI{
+			roomVersion: nonPseudoIDRoomVersion,
+			roomIDStr:   roomIDStr,
+			roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{
+				{
+					EventType: eventType,
+					StateKey:  stateSenderRoomKey,
+				}: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{
+					"not": "thisone",
+				}),
+				{
+					EventType: eventType,
+					StateKey:  stateSenderUserID,
+				}: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{
+					"foo": "bar",
+				}),
+			},
+			userIDStr: userIDStr,
+			senderMapping: map[string]string{
+				stateSenderUserID: stateSenderUserID,
+			},
+		}
+
+		jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false)
+
+		assert.DeepEqual(t, jsonResp, util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: spec.RawJSON(`{"foo":"bar"}`),
+		})
+	})
+}
+
+func mustCreateStatePDU(t *testing.T, roomVer gomatrixserverlib.RoomVersion, roomID string, stateType string, stateKey string, stateContent map[string]interface{}) *types.HeaderedEvent {
+	t.Helper()
+	roomVerImpl := gomatrixserverlib.MustGetRoomVersion(roomVer)
+
+	evBytes, err := json.Marshal(map[string]interface{}{
+		"room_id":   roomID,
+		"type":      stateType,
+		"state_key": stateKey,
+		"content":   stateContent,
+	})
+	if err != nil {
+		t.Fatalf("failed to create event: %v", err)
+	}
+
+	ev, err := roomVerImpl.NewEventFromTrustedJSON(evBytes, false)
+	if err != nil {
+		t.Fatalf("failed to create event: %v", err)
+	}
+
+	return &types.HeaderedEvent{PDU: ev}
+}

clientapi/routing/upgrade_room.go

@@ -59,7 +59,15 @@ func UpgradeRoom(
 		}
 	}
 
-	newRoomID, err := rsAPI.PerformRoomUpgrade(req.Context(), roomID, device.UserID, gomatrixserverlib.RoomVersion(r.NewVersion))
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("device UserID is invalid")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+	newRoomID, err := rsAPI.PerformRoomUpgrade(req.Context(), roomID, *userID, gomatrixserverlib.RoomVersion(r.NewVersion))
 	switch e := err.(type) {
 	case nil:
 	case roomserverAPI.ErrNotAllowed:

clientapi/threepid/invites.go

@@ -355,8 +355,22 @@ func emit3PIDInviteEvent(
 	rsAPI api.ClientRoomserverAPI,
 	evTime time.Time,
 ) error {
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return err
+	}
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return err
+	}
+	sender, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID)
+	if err != nil {
+		return err
+	} else if sender == nil {
+		return fmt.Errorf("sender ID not found for %s in %s", *userID, *validRoomID)
+	}
 	proto := &gomatrixserverlib.ProtoEvent{
-		Sender:   device.UserID,
+		SenderID: string(*sender),
 		RoomID:   roomID,
 		Type:     "m.room.third_party_invite",
 		StateKey: &res.Token,
@@ -370,7 +384,7 @@ func emit3PIDInviteEvent(
 		PublicKeys:     res.PublicKeys,
 	}
 
-	if err := proto.SetContent(content); err != nil {
+	if err = proto.SetContent(content); err != nil {
 		return err
 	}
 

cmd/dendrite-demo-pinecone/monolith/monolith.go

@@ -98,7 +98,7 @@ func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir st
 	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(storageDir, dbPrefix)))
 	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(storageDir, dbPrefix)))
 	cfg.RelayAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-relayapi.db", filepath.Join(storageDir, dbPrefix)))
-	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+	cfg.MSCs.MSCs = []string{"msc2836"}
 	cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(storageDir, dbPrefix)))
 	cfg.ClientAPI.RegistrationDisabled = false
 	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
@@ -126,7 +126,7 @@ func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) {
 }
 
 func (p *P2PMonolith) SetupDendrite(
-	processCtx *process.ProcessContext, cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers,
+	processCtx *process.ProcessContext, cfg *config.Dendrite, cm *sqlutil.Connections, routers httputil.Routers,
 	port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) {
 
 	p.port = port
@@ -143,13 +143,12 @@ func (p *P2PMonolith) SetupDendrite(
 	fsAPI := federationapi.NewInternalAPI(
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
-	rsAPI.SetFederationAPI(fsAPI, keyRing)
-
 	userProvider := users.NewPineconeUserProvider(p.Router, p.Sessions, userAPI, federation)
 	roomProvider := rooms.NewPineconeRoomProvider(p.Router, p.Sessions, fsAPI, federation)
 
@@ -222,8 +221,8 @@ func (p *P2PMonolith) closeAllResources() {
 	p.httpServerMu.Lock()
 	if p.httpServer != nil {
 		_ = p.httpServer.Shutdown(context.Background())
-		p.httpServerMu.Unlock()
 	}
+	p.httpServerMu.Unlock()
 
 	select {
 	case p.stopHandlingEvents <- true:

cmd/dendrite-demo-yggdrasil/README.md

@@ -1,6 +1,6 @@
 # Yggdrasil Demo
 
-This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later.
+This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.20 or later.
 
 To run the homeserver, start at the root of the Dendrite repository and run:
 

cmd/dendrite-demo-yggdrasil/main.go

@@ -134,7 +134,7 @@ func main() {
 		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+		cfg.MSCs.MSCs = []string{"msc2836"}
 		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.ClientAPI.RegistrationDisabled = false
 		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
@@ -213,14 +213,15 @@ func main() {
 	natsInstance := jetstream.NATSInstance{}
 	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
-
-	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
-	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationapi.NewInternalAPI(
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
+
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
+	rsAPI.SetAppserviceAPI(asAPI)
+
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{

cmd/dendrite-upgrade-tests/main.go

@@ -7,7 +7,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -55,7 +54,7 @@ var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD"
 // due to the error:
 // When using COPY with more than one source file, the destination must be a directory and end with a /
 // We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
-const DockerfilePostgreSQL = `FROM golang:1.18-buster as build
+const DockerfilePostgreSQL = `FROM golang:1.20-bookworm as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 ARG BINARY
@@ -74,16 +73,16 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key
 # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password
 RUN sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml
 # No password when connecting over localhost
-RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/11/main/pg_hba.conf
+RUN sed -i "s%127.0.0.1/32            scram-sha-256%127.0.0.1/32            trust%g" /etc/postgresql/15/main/pg_hba.conf
 # Bump up max conns for moar concurrency
-RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/11/main/postgresql.conf
+RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/15/main/postgresql.conf
 RUN sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
 #!/bin/bash -eu \n\
 pg_lsclusters \n\
-pg_ctlcluster 11 main start \n\
+pg_ctlcluster 15 main start \n\
  \n\
 until pg_isready \n\
 do \n\
@@ -101,7 +100,7 @@ ENV BINARY=dendrite
 EXPOSE 8008 8448
 CMD /build/run_dendrite.sh`
 
-const DockerfileSQLite = `FROM golang:1.18-buster as build
+const DockerfileSQLite = `FROM golang:1.20-bookworm as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 ARG BINARY
@@ -119,7 +118,7 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key
 
 # Make sure the SQLite databases are in a persistent location, we're already mapping
 # the postgresql folder so let's just use that for simplicity
-RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/11\/main\/%g" dendrite.yaml
+RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/15\/main\/%g" dendrite.yaml
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
@@ -402,7 +401,7 @@ func runImage(dockerClient *client.Client, volumeName string, branchNameToImageI
 			{
 				Type:   mount.TypeVolume,
 				Source: volumeName,
-				Target: "/var/lib/postgresql/11/main",
+				Target: "/var/lib/postgresql/15/main",
 			},
 		},
 	}, nil, nil, "dendrite_upgrade_test_"+branchName)
@@ -515,7 +514,7 @@ func testCreateAccount(dockerClient *client.Client, version *semver.Version, con
 	}
 	defer response.Close()
 
-	data, err := ioutil.ReadAll(response.Reader)
+	data, err := io.ReadAll(response.Reader)
 	if err != nil {
 		return err
 	}
@@ -557,8 +556,8 @@ func cleanup(dockerClient *client.Client) {
 	})
 	for _, c := range containers {
 		log.Printf("Removing container: %v %v\n", c.ID, c.Names)
-		s := time.Second
+		timeout := 1
-		_ = dockerClient.ContainerStop(context.Background(), c.ID, &s)
+		_ = dockerClient.ContainerStop(context.Background(), c.ID, container.StopOptions{Timeout: &timeout})
 		_ = dockerClient.ContainerRemove(context.Background(), c.ID, types.ContainerRemoveOptions{
 			Force: true,
 		})
@@ -592,7 +591,7 @@ func main() {
 	branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions)
 
 	// make a shared postgres volume
-	volume, err := dockerClient.VolumeCreate(context.Background(), volume.VolumeCreateBody{
+	volume, err := dockerClient.VolumeCreate(context.Background(), volume.CreateOptions{
 		Name: "dendrite_upgrade_test",
 		Labels: map[string]string{
 			dendriteUpgradeTestLabel: "yes",

cmd/dendrite/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/appservice"
@@ -156,13 +157,14 @@ func main() {
 
 	keyRing := fsAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient)
-	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
-
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this
 	// dependency. Other components also need updating after their dependencies are up.
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
+
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
+
 	rsAPI.SetAppserviceAPI(asAPI)
 	rsAPI.SetUserAPI(userAPI)
 
@@ -187,6 +189,16 @@ func main() {
 		}
 	}
 
+	upCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "dendrite",
+		Name:      "up",
+		ConstLabels: map[string]string{
+			"version": internal.VersionString(),
+		},
+	})
+	upCounter.Add(1)
+	prometheus.MustRegister(upCounter)
+
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {
 		basepkg.SetupAndServeHTTP(processCtx, cfg, routers, httpAddr, nil, nil)

cmd/generate-config/main.go

@@ -74,7 +74,7 @@ func main() {
 			// don't hit matrix.org when running tests!!!
 			cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
 			cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))
-			cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"}
+			cfg.MSCs.MSCs = []string{"msc2836", "msc2444", "msc2753"}
 			cfg.Logging[0].Level = "trace"
 			cfg.Logging[0].Type = "std"
 			cfg.UserAPI.BCryptCost = bcrypt.MinCost

cmd/resolve-state/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 // This is a utility for inspecting state snapshots and running state resolution
@@ -32,6 +33,19 @@ var roomVersion = flag.String("roomversion", "5", "the room version to parse eve
 var filterType = flag.String("filtertype", "", "the event types to filter on")
 var difference = flag.Bool("difference", false, "whether to calculate the difference between snapshots")
 
+// dummyQuerier implements QuerySenderIDAPI. Does **NOT** do any "magic" for pseudoID rooms
+// to avoid having to "start" a full roomserver API.
+type dummyQuerier struct{}
+
+func (d dummyQuerier) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+	s := spec.SenderIDFromUserID(userID)
+	return &s, nil
+}
+
+func (d dummyQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	return senderID.ToUserID(), nil
+}
+
 // nolint:gocyclo
 func main() {
 	ctx := context.Background()
@@ -53,22 +67,31 @@ func main() {
 		}
 	}
 
-	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
-
 	processCtx := process.NewProcessContext()
 	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+
+	dbOpts := cfg.RoomServer.Database
+	if dbOpts.ConnectionString == "" {
+		dbOpts = cfg.Global.DatabaseOptions
+	}
+
+	fmt.Println("Opening database")
 	roomserverDB, err := storage.Open(
-		processCtx.Context(), cm, &cfg.RoomServer.Database,
+		processCtx.Context(), cm, &dbOpts,
-		caching.NewRistrettoCache(128*1024*1024, time.Hour, true),
+		caching.NewRistrettoCache(8*1024*1024, time.Minute*5, caching.DisableMetrics),
 	)
 	if err != nil {
 		panic(err)
 	}
 
+	rsAPI := dummyQuerier{}
+
 	roomInfo := &types.RoomInfo{
 		RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion),
 	}
-	stateres := state.NewStateResolution(roomserverDB, roomInfo)
+	stateres := state.NewStateResolution(roomserverDB, roomInfo, rsAPI)
+
+	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
 
 	if *difference {
 		if len(snapshotNIDs) != 2 {
@@ -179,10 +202,25 @@ func main() {
 		authEvents[i] = authEventEntries[i].PDU
 	}
 
+	// Get the roomNID
+	roomInfo, err = roomserverDB.RoomInfo(ctx, authEvents[0].RoomID().String())
+	if err != nil {
+		panic(err)
+	}
+
 	fmt.Println("Resolving state")
 	var resolved Events
 	resolved, err = gomatrixserverlib.ResolveConflicts(
-		gomatrixserverlib.RoomVersion(*roomVersion), events, authEvents,
+		gomatrixserverlib.RoomVersion(*roomVersion), events, authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+		},
+		func(eventID string) bool {
+			isRejected, rejectedErr := roomserverDB.IsEventRejected(ctx, roomInfo.RoomNID, eventID)
+			if rejectedErr != nil {
+				return true
+			}
+			return isRejected
+		},
 	)
 	if err != nil {
 		panic(err)

dendrite-sample.yaml

@@ -72,6 +72,10 @@ global:
   # The base URL to delegate client-server communications to e.g. https://localhost
   well_known_client_name: ""
 
+  # The server name to delegate sliding sync communications to, with optional port.
+  # Requires `well_known_client_name` to also be configured.
+  well_known_sliding_sync_proxy: ""
+
   # Lists of domains that the server will trust as identity servers to verify third
   # party identifiers such as phone numbers and email addresses.
   trusted_third_party_id_servers:
@@ -276,7 +280,6 @@ media_api:
 mscs:
   mscs:
   #  - msc2836  # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
-  #  - msc2946  # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
 
 # Configuration for the Sync API.
 sync_api:
@@ -322,6 +325,10 @@ user_api:
   auto_join_rooms:
   #  - "#main:matrix.org"
 
+  # The number of workers to start for the DeviceListUpdater. Defaults to 8.
+  # This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely.
+  # worker_count: 8
+
 # Configuration for Opentracing.
 # See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
 # how this works and how to set it up.

docs/FAQ.md

@@ -24,7 +24,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo
 
 Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be 
 specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting 
-the development efforts through [contributing](../development/contributing). 
+the development efforts through [contributing](./development/CONTRIBUTING.md). 
 
 ## Is there a migration path from Synapse to Dendrite?
 
@@ -64,16 +64,18 @@ Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially suppo
 
 ## Does Dendrite support Space Summaries?
 
-Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration:
+Yes
+
+## Does Dendrite support Threads?
+
+Yes, to enable them [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported.
 
 ```
 mscs:
   mscs:
-    - msc2946
+    - msc2836
 ```
 
-Similarly, [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported.
-
 Please note that MSCs should be considered experimental and can result in significant usability issues when enabled. If you'd like more details on how MSCs are ratified or the current status of MSCs, please see the [Matrix specification documentation](https://spec.matrix.org/proposals/) on the subject.
 
 ## Does Dendrite support push notifications?
@@ -103,7 +105,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom <ve
 
 ## How do I reset somebody's password on my server?
 
-Use the admin endpoint [resetpassword](./administration/adminapi#post-_dendriteadminresetpassworduserid)
+Use the admin endpoint [resetpassword](./administration/4_adminapi.md#post-_dendriteadminresetpassworduserid)
 
 ## Should I use PostgreSQL or SQLite for my databases?
 

docs/Gemfile.lock

@@ -14,7 +14,7 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.23.9)
+    commonmarker (0.23.10)
     concurrent-ruby (1.2.0)
     dnsruby (1.61.9)
       simpleidn (~> 0.1)

docs/administration/4_adminapi.md

@@ -75,7 +75,7 @@ This endpoint instructs Dendrite to immediately query `/devices/{userID}` on a f
 
 ## POST `/_dendrite/admin/purgeRoom/{roomID}`
 
-This endpoint instructs Dendrite to remove the given room from its database. Before doing so, it will evacuate all local users from the room. It does **NOT** remove media files. Depending on the size of the room, this may take a while. Will return an empty JSON once other components were instructed to delete the room.
+This endpoint instructs Dendrite to remove the given room from its database. It does **NOT** remove media files. Depending on the size of the room, this may take a while. Will return an empty JSON once other components were instructed to delete the room.
 
 ## POST `/_synapse/admin/v1/send_server_notice`
 

docs/administration/5_optimisation.md

@@ -95,7 +95,7 @@ Consider enabling the DNS cache by modifying the `global` section of your config
 ## Time synchronisation
 
 Matrix relies heavily on TLS which requires the system time to be correct. If the clock
-drifts then you may find that federation no works reliably (or at all) and clients may
+drifts then you may find that federation will not work reliably (or at all) and clients may
 struggle to connect to your Dendrite server.
 
 Ensure that the time is synchronised on your system by enabling NTP sync.

docs/development/CONTRIBUTING.md

@@ -109,7 +109,7 @@ To configure the connection to a remote Postgres, you can use the following envi
 
 ```bash
 POSTGRES_USER=postgres
-POSTGERS_PASSWORD=yourPostgresPassword
+POSTGRES_PASSWORD=yourPostgresPassword
 POSTGRES_HOST=localhost
 POSTGRES_DB=postgres # the superuser database to use
 ```

docs/installation/1_planning.md

@@ -59,7 +59,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie
 
 ### Go
 
-At this time, Dendrite supports being built with Go 1.18 or later. We do not support building
+At this time, Dendrite supports being built with Go 1.20 or later. We do not support building
 Dendrite with older versions of Go than this. If you are installing Go using a package manager,
 you should check (by running `go version`) that you are using a suitable version before you start.
 

docs/installation/manual/3_signingkey.md

@@ -2,7 +2,7 @@
 title: Generating signing keys
 parent: Manual
 grand_parent: Installation
-nav_order: 4
+nav_order: 3
 permalink: /installation/manual/signingkeys
 ---
 

docs/installation/manual/4_configuration.md

@@ -2,7 +2,7 @@
 title: Configuring Dendrite
 parent: Manual
 grand_parent: Installation
-nav_order: 3
+nav_order: 4
 permalink: /installation/manual/configuration
 ---
 
@@ -21,7 +21,7 @@ sections:
 
 First of all, you will need to configure the server name of your Matrix homeserver.
 This must match the domain name that you have selected whilst [configuring the domain
-name delegation](domainname#delegation).
+name delegation](../domainname#delegation).
 
 In the `global` section, set the `server_name` to your delegated domain name:
 

federationapi/api/api.go

@@ -27,7 +27,6 @@ type FederationInternalAPI interface {
 	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
 	LookupServerKeys(ctx context.Context, s spec.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
 	MSC2836EventRelationships(ctx context.Context, origin, dst spec.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res fclient.MSC2836EventRelationshipsResponse, err error)
-	MSC2946Spaces(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error)
 
 	// Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos.
 	PerformBroadcastEDU(
@@ -63,6 +62,8 @@ type RoomserverFederationAPI interface {
 	PerformLeave(ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse) error
 	// Handle sending an invite to a remote server.
 	SendInvite(ctx context.Context, event gomatrixserverlib.PDU, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error)
+	// Handle sending an invite to a remote server.
+	SendInviteV3(ctx context.Context, event gomatrixserverlib.ProtoEvent, invitee spec.UserID, version gomatrixserverlib.RoomVersion, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error)
 	// Handle an instruction to peek a room on a remote server.
 	PerformOutboundPeek(ctx context.Context, request *PerformOutboundPeekRequest, response *PerformOutboundPeekResponse) error
 	// Query the server names of the joined hosts in a room.
@@ -73,6 +74,8 @@ type RoomserverFederationAPI interface {
 	GetEventAuth(ctx context.Context, origin, s spec.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res fclient.RespEventAuth, err error)
 	GetEvent(ctx context.Context, origin, s spec.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
 	LookupMissingEvents(ctx context.Context, origin, s spec.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error)
+
+	RoomHierarchies(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.RoomHierarchyResponse, err error)
 }
 
 type P2PFederationAPI interface {

federationapi/consumers/keychange.go

@@ -117,19 +117,27 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 		return true
 	}
 
-	var queryRes roomserverAPI.QueryRoomsForUserResponse
+	userID, err := spec.NewUserID(m.UserID, true)
-	err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{
+	if err != nil {
-		UserID:         m.UserID,
+		sentry.CaptureException(err)
-		WantMembership: "join",
+		logger.WithError(err).Error("invalid user ID")
-	}, &queryRes)
+		return true
+	}
+
+	roomIDs, err := t.rsAPI.QueryRoomsForUser(t.ctx, *userID, "join")
 	if err != nil {
 		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to calculate joined rooms for user")
 		return true
 	}
 
+	roomIDStrs := make([]string, len(roomIDs))
+	for i, room := range roomIDs {
+		roomIDStrs[i] = room.String()
+	}
+
 	// send this key change to all servers who share rooms with this user.
-	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
+	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true)
 	if err != nil {
 		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in")
@@ -179,18 +187,27 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 	}
 	logger := logrus.WithField("user_id", output.UserID)
 
-	var queryRes roomserverAPI.QueryRoomsForUserResponse
+	outputUserID, err := spec.NewUserID(output.UserID, true)
-	err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{
+	if err != nil {
-		UserID:         output.UserID,
+		sentry.CaptureException(err)
-		WantMembership: "join",
+		logrus.WithError(err).Errorf("invalid user ID")
-	}, &queryRes)
+		return true
+	}
+
+	rooms, err := t.rsAPI.QueryRoomsForUser(t.ctx, *outputUserID, "join")
 	if err != nil {
 		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user")
 		return true
 	}
+
+	roomIDStrs := make([]string, len(rooms))
+	for i, room := range rooms {
+		roomIDStrs[i] = room.String()
+	}
+
 	// send this key change to all servers who share rooms with this user.
-	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
+	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true)
 	if err != nil {
 		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in")

federationapi/consumers/presence.go

@@ -29,6 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 )
@@ -94,16 +95,23 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
 		return true
 	}
 
-	var queryRes roomserverAPI.QueryRoomsForUserResponse
+	parsedUserID, err := spec.NewUserID(userID, true)
-	err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{
+	if err != nil {
-		UserID:         userID,
+		util.GetLogger(ctx).WithError(err).WithField("user_id", userID).Error("invalid user ID")
-		WantMembership: "join",
+		return true
-	}, &queryRes)
+	}
+
+	roomIDs, err := t.rsAPI.QueryRoomsForUser(t.ctx, *parsedUserID, "join")
 	if err != nil {
 		log.WithError(err).Error("failed to calculate joined rooms for user")
 		return true
 	}
 
+	roomIDStrs := make([]string, len(roomIDs))
+	for i, roomID := range roomIDs {
+		roomIDStrs[i] = roomID.String()
+	}
+
 	presence := msg.Header.Get("presence")
 
 	ts, err := strconv.Atoi(msg.Header.Get("last_active_ts"))
@@ -112,7 +120,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
 	}
 
 	// send this presence to all servers who share rooms with this user.
-	joined, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
+	joined, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true)
 	if err != nil {
 		log.WithError(err).Error("failed to get joined hosts")
 		return true

federationapi/consumers/roomserver.go

@@ -16,7 +16,9 @@ package consumers
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"time"
@@ -174,7 +176,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	// Finally, work out if there are any more events missing.
 	if len(missingEventIDs) > 0 {
 		eventsReq := &api.QueryEventsByIDRequest{
-			RoomID:   ore.Event.RoomID(),
+			RoomID:   ore.Event.RoomID().String(),
 			EventIDs: missingEventIDs,
 		}
 		eventsRes := &api.QueryEventsByIDResponse{}
@@ -192,7 +194,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 		evs[i] = addsStateEvents[i].PDU
 	}
 
-	addsJoinedHosts, err := JoinedHostsFromEvents(evs)
+	addsJoinedHosts, err := JoinedHostsFromEvents(s.ctx, evs, s.rsAPI)
 	if err != nil {
 		return err
 	}
@@ -203,7 +205,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	// talking to the roomserver
 	oldJoinedHosts, err := s.db.UpdateRoom(
 		s.ctx,
-		ore.Event.RoomID(),
+		ore.Event.RoomID().String(),
 		addsJoinedHosts,
 		ore.RemovesStateEventIDs,
 		rewritesState, // if we're re-writing state, nuke all joined hosts before adding
@@ -216,7 +218,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	if s.cfg.Matrix.Presence.EnableOutbound && len(addsJoinedHosts) > 0 && ore.Event.Type() == spec.MRoomMember && ore.Event.StateKey() != nil {
 		membership, _ := ore.Event.Membership()
 		if membership == spec.Join {
-			s.sendPresence(ore.Event.RoomID(), addsJoinedHosts)
+			s.sendPresence(ore.Event.RoomID().String(), addsJoinedHosts)
 		}
 	}
 
@@ -345,7 +347,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 		return nil, err
 	}
 
-	combinedAddsJoinedHosts, err := JoinedHostsFromEvents(combinedAddsEvents)
+	combinedAddsJoinedHosts, err := JoinedHostsFromEvents(s.ctx, combinedAddsEvents, s.rsAPI)
 	if err != nil {
 		return nil, err
 	}
@@ -374,7 +376,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 	}
 
 	// handle peeking hosts
-	inboundPeeks, err := s.db.GetInboundPeeks(s.ctx, ore.Event.PDU.RoomID())
+	inboundPeeks, err := s.db.GetInboundPeeks(s.ctx, ore.Event.PDU.RoomID().String())
 	if err != nil {
 		return nil, err
 	}
@@ -394,7 +396,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 // JoinedHostsFromEvents turns a list of state events into a list of joined hosts.
 // This errors if one of the events was invalid.
 // It should be impossible for an invalid event to get this far in the pipeline.
-func JoinedHostsFromEvents(evs []gomatrixserverlib.PDU) ([]types.JoinedHost, error) {
+func JoinedHostsFromEvents(ctx context.Context, evs []gomatrixserverlib.PDU, rsAPI api.FederationRoomserverAPI) ([]types.JoinedHost, error) {
 	var joinedHosts []types.JoinedHost
 	for _, ev := range evs {
 		if ev.Type() != "m.room.member" || ev.StateKey() == nil {
@@ -407,12 +409,26 @@ func JoinedHostsFromEvents(evs []gomatrixserverlib.PDU) ([]types.JoinedHost, err
 		if membership != spec.Join {
 			continue
 		}
-		_, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey())
+		var domain spec.ServerName
+		userID, err := rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*ev.StateKey()))
+		if err != nil {
+			if errors.As(err, new(base64.CorruptInputError)) {
+				// Fallback to using the "old" way of getting the user domain, avoids
+				// "illegal base64 data at input byte 0" errors
+				// FIXME: we should do this in QueryUserIDForSender instead
+				_, domain, err = gomatrixserverlib.SplitID('@', *ev.StateKey())
 				if err != nil {
 					return nil, err
 				}
+			} else {
+				return nil, err
+			}
+		} else {
+			domain = userID.Domain()
+		}
+
 		joinedHosts = append(joinedHosts, types.JoinedHost{
-			MemberEventID: ev.EventID(), ServerName: serverName,
+			MemberEventID: ev.EventID(), ServerName: domain,
 		})
 	}
 	return joinedHosts, nil
@@ -490,7 +506,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 	// At this point the missing events are neither the event itself nor are
 	// they present in our local database. Our only option is to fetch them
 	// from the roomserver using the query API.
-	eventReq := api.QueryEventsByIDRequest{EventIDs: missing, RoomID: event.RoomID()}
+	eventReq := api.QueryEventsByIDRequest{EventIDs: missing, RoomID: event.RoomID().String()}
 	var eventResp api.QueryEventsByIDResponse
 	if err := s.rsAPI.QueryEventsByID(s.ctx, &eventReq, &eventResp); err != nil {
 		return nil, err

federationapi/federationapi.go

@@ -24,7 +24,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/sirupsen/logrus"
 
-	"github.com/matrix-org/dendrite/federationapi/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/consumers"
 	"github.com/matrix-org/dendrite/federationapi/internal"
@@ -95,14 +94,14 @@ func AddPublicRoutes(
 func NewInternalAPI(
 	processContext *process.ProcessContext,
 	dendriteCfg *config.Dendrite,
-	cm sqlutil.Connections,
+	cm *sqlutil.Connections,
 	natsInstance *jetstream.NATSInstance,
 	federation fclient.FederationClient,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	caches *caching.Caches,
 	keyRing *gomatrixserverlib.KeyRing,
 	resetBlacklist bool,
-) api.FederationInternalAPI {
+) *internal.FederationInternalAPI {
 	cfg := &dendriteCfg.FederationAPI
 
 	federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName)
@@ -126,7 +125,7 @@ func NewInternalAPI(
 	queues := queue.NewOutgoingQueues(
 		federationDB, processContext,
 		cfg.Matrix.DisableFederation,
-		cfg.Matrix.ServerName, federation, rsAPI, &stats,
+		cfg.Matrix.ServerName, federation, &stats,
 		signingInfo,
 	)
 

federationapi/federationapi_test.go

@@ -33,7 +33,16 @@ import (
 type fedRoomserverAPI struct {
 	rsapi.FederationRoomserverAPI
 	inputRoomEvents   func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse)
-	queryRoomsForUser func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error
+	queryRoomsForUser func(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error)
+}
+
+func (f *fedRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	return spec.NewUserID(string(senderID), true)
+}
+
+func (f *fedRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+	senderID := spec.SenderID(userID.String())
+	return &senderID, nil
 }
 
 // PerformJoin will call this function
@@ -45,11 +54,11 @@ func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.Input
 }
 
 // keychange consumer calls this
-func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
+func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) {
 	if f.queryRoomsForUser == nil {
-		return nil
+		return nil, nil
 	}
-	return f.queryRoomsForUser(ctx, req, res)
+	return f.queryRoomsForUser(ctx, userID, desiredMembership)
 }
 
 // TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate
@@ -111,12 +120,13 @@ func (f *fedClient) MakeJoin(ctx context.Context, origin, s spec.ServerName, roo
 	defer f.fedClientMutex.Unlock()
 	for _, r := range f.allowJoins {
 		if r.ID == roomID {
+			senderIDString := userID
 			res.RoomVersion = r.Version
 			res.JoinEvent = gomatrixserverlib.ProtoEvent{
-				Sender:     userID,
+				SenderID:   senderIDString,
 				RoomID:     roomID,
 				Type:       "m.room.member",
-				StateKey:   &userID,
+				StateKey:   &senderIDString,
 				Content:    spec.RawJSON([]byte(`{"membership":"join"}`)),
 				PrevEvents: r.ForwardExtremities(),
 			}
@@ -136,7 +146,7 @@ func (f *fedClient) SendJoin(ctx context.Context, origin, s spec.ServerName, eve
 	f.fedClientMutex.Lock()
 	defer f.fedClientMutex.Unlock()
 	for _, r := range f.allowJoins {
-		if r.ID == event.RoomID() {
+		if r.ID == event.RoomID().String() {
 			r.InsertEvent(f.t, &types.HeaderedEvent{PDU: event})
 			f.t.Logf("Join event: %v", event.EventID())
 			res.StateEvents = types.NewEventJSONsFromHeaderedEvents(r.CurrentState())
@@ -189,18 +199,22 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 	fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID)
 	room := test.NewRoom(t, creator)
 
+	roomID, err := spec.NewRoomID(room.ID)
+	if err != nil {
+		t.Fatalf("Invalid room ID: %q", roomID)
+	}
+
 	rsapi := &fedRoomserverAPI{
 		inputRoomEvents: func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
 			if req.Asynchronous {
 				t.Errorf("InputRoomEvents from PerformJoin MUST be synchronous")
 			}
 		},
-		queryRoomsForUser: func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
+		queryRoomsForUser: func(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) {
-			if req.UserID == joiningUser.ID && req.WantMembership == "join" {
+			if userID.String() == joiningUser.ID && desiredMembership == "join" {
-				res.RoomIDs = []string{room.ID}
+				return []spec.RoomID{*roomID}, nil
-				return nil
 			}
-			return fmt.Errorf("unexpected queryRoomsForUser: %+v", *req)
+			return nil, fmt.Errorf("unexpected queryRoomsForUser: %v, %v", userID, desiredMembership)
 		},
 	}
 	fc := &fedClient{

federationapi/internal/api.go

@@ -54,11 +54,14 @@ func NewFederationInternalAPI(
 			KeyDatabase: serverKeyDB,
 		}
 
+		pubKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey)
 		addDirectFetcher := func() {
 			keyRing.KeyFetchers = append(
 				keyRing.KeyFetchers,
 				&gomatrixserverlib.DirectKeyFetcher{
 					Client:            federation,
+					IsLocalServerName: cfg.Matrix.IsLocalServerName,
+					LocalPublicKey:    []byte(pubKey),
 				},
 			)
 		}
@@ -109,7 +112,7 @@ func NewFederationInternalAPI(
 	}
 }
 
-func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) {
+func (a *FederationInternalAPI) IsBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) {
 	stats := a.statistics.ForServer(s)
 	if stats.Blacklisted() {
 		return stats, &api.FederationClientError{
@@ -148,7 +151,7 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti
 func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted(
 	s spec.ServerName, request func() (interface{}, error),
 ) (interface{}, error) {
-	stats, err := a.isBlacklistedOrBackingOff(s)
+	stats, err := a.IsBlacklistedOrBackingOff(s)
 	if err != nil {
 		return nil, err
 	}

federationapi/internal/federationclient.go

@@ -29,7 +29,7 @@ func (a *FederationInternalAPI) MakeJoin(
 func (a *FederationInternalAPI) SendJoin(
 	ctx context.Context, origin, s spec.ServerName, event gomatrixserverlib.PDU,
 ) (res gomatrixserverlib.SendJoinResponse, err error) {
-	ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
+	ctx, cancel := context.WithTimeout(ctx, time.Minute*5)
 	defer cancel()
 	ires, err := a.federation.SendJoin(ctx, origin, s, event)
 	if err != nil {
@@ -194,16 +194,16 @@ func (a *FederationInternalAPI) MSC2836EventRelationships(
 	return ires.(fclient.MSC2836EventRelationshipsResponse), nil
 }
 
-func (a *FederationInternalAPI) MSC2946Spaces(
+func (a *FederationInternalAPI) RoomHierarchies(
 	ctx context.Context, origin, s spec.ServerName, roomID string, suggestedOnly bool,
-) (res fclient.MSC2946SpacesResponse, err error) {
+) (res fclient.RoomHierarchyResponse, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
-		return a.federation.MSC2946Spaces(ctx, origin, s, roomID, suggestedOnly)
+		return a.federation.RoomHierarchy(ctx, origin, s, roomID, suggestedOnly)
 	})
 	if err != nil {
 		return res, err
 	}
-	return ires.(fclient.MSC2946SpacesResponse), nil
+	return ires.(fclient.RoomHierarchyResponse), nil
 }

federationapi/internal/federationclient_test.go

@@ -65,7 +65,7 @@ func TestFederationClientQueryKeys(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -96,7 +96,7 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -126,7 +126,7 @@ func TestFederationClientQueryKeysFailure(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -156,7 +156,7 @@ func TestFederationClientClaimKeys(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -187,7 +187,7 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{

federationapi/internal/keys.go

@@ -170,7 +170,7 @@ func (s *FederationInternalAPI) handleDatabaseKeys(
 		// in that case. If the key isn't valid right now, then by
 		// leaving it in the 'requests' map, we'll try to update the
 		// key using the fetchers in handleFetcherKeys.
-		if res.WasValidAt(now, true) {
+		if res.WasValidAt(now, gomatrixserverlib.StrictValiditySignatureCheck) {
 			delete(requests, req)
 		}
 	}

federationapi/internal/perform.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"crypto/ed25519"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -164,7 +165,31 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		PrivateKey: r.cfg.Matrix.PrivateKey,
 		KeyID:      r.cfg.Matrix.KeyID,
 		KeyRing:    r.keyRing,
-		EventProvider: federatedEventProvider(ctx, r.federation, r.keyRing, user.Domain(), serverName),
+		EventProvider: federatedEventProvider(ctx, r.federation, r.keyRing, user.Domain(), serverName, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return r.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+		}),
+		UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return r.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+		},
+		GetOrCreateSenderID: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) {
+			// assign a roomNID, otherwise we can't create a private key for the user
+			_, nidErr := r.rsAPI.AssignRoomNID(ctx, roomID, gomatrixserverlib.RoomVersion(roomVersion))
+			if nidErr != nil {
+				return "", nil, nidErr
+			}
+			key, keyErr := r.rsAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID)
+			if keyErr != nil {
+				return "", nil, keyErr
+			}
+			return spec.SenderIDFromPseudoIDKey(key), key, nil
+		},
+		StoreSenderIDFromPublicID: func(ctx context.Context, senderID spec.SenderID, userIDRaw string, roomID spec.RoomID) error {
+			storeUserID, userErr := spec.NewUserID(userIDRaw, true)
+			if userErr != nil {
+				return userErr
+			}
+			return r.rsAPI.StoreUserRoomPublicKey(ctx, senderID, *storeUserID, roomID)
+		},
 	}
 	response, joinErr := gomatrixserverlib.PerformJoin(ctx, r, joinInput)
 
@@ -187,7 +212,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	// joining a room, waiting for 200 OK then changing device keys and have those keys not be sent
 	// to other servers (this was a cause of a flakey sytest "Local device key changes get to remote servers")
 	// The events are trusted now as we performed auth checks above.
-	joinedHosts, err := consumers.JoinedHostsFromEvents(response.StateSnapshot.GetStateEvents().TrustedEvents(response.JoinEvent.Version(), false))
+	joinedHosts, err := consumers.JoinedHostsFromEvents(ctx, response.StateSnapshot.GetStateEvents().TrustedEvents(response.JoinEvent.Version(), false), r.rsAPI)
 	if err != nil {
 		return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err)
 	}
@@ -358,8 +383,11 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 
 	// authenticate the state returned (check its auth events etc)
 	// the equivalent of CheckSendJoinResponse()
+	userIDProvider := func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return r.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+	}
 	authEvents, stateEvents, err := gomatrixserverlib.CheckStateResponse(
-		ctx, &respPeek, respPeek.RoomVersion, r.keyRing, federatedEventProvider(ctx, r.federation, r.keyRing, r.cfg.Matrix.ServerName, serverName),
+		ctx, &respPeek, respPeek.RoomVersion, r.keyRing, federatedEventProvider(ctx, r.federation, r.keyRing, r.cfg.Matrix.ServerName, serverName, userIDProvider), userIDProvider,
 	)
 	if err != nil {
 		return fmt.Errorf("error checking state returned from peeking: %w", err)
@@ -406,7 +434,7 @@ func (r *FederationInternalAPI) PerformLeave(
 	request *api.PerformLeaveRequest,
 	response *api.PerformLeaveResponse,
 ) (err error) {
-	_, origin, err := r.cfg.Matrix.SplitLocalID('@', request.UserID)
+	userID, err := spec.NewUserID(request.UserID, true)
 	if err != nil {
 		return err
 	}
@@ -425,7 +453,7 @@ func (r *FederationInternalAPI) PerformLeave(
 		// request.
 		respMakeLeave, err := r.federation.MakeLeave(
 			ctx,
-			origin,
+			userID.Domain(),
 			serverName,
 			request.RoomID,
 			request.UserID,
@@ -446,9 +474,20 @@ func (r *FederationInternalAPI) PerformLeave(
 
 		// Set all the fields to be what they should be, this should be a no-op
 		// but it's possible that the remote server returned us something "odd"
+		roomID, err := spec.NewRoomID(request.RoomID)
+		if err != nil {
+			return err
+		}
+		senderID, err := r.rsAPI.QuerySenderIDForUser(ctx, *roomID, *userID)
+		if err != nil {
+			return err
+		} else if senderID == nil {
+			return fmt.Errorf("sender ID not found for %s in %s", *userID, *roomID)
+		}
+		senderIDString := string(*senderID)
 		respMakeLeave.LeaveEvent.Type = spec.MRoomMember
-		respMakeLeave.LeaveEvent.Sender = request.UserID
+		respMakeLeave.LeaveEvent.SenderID = senderIDString
-		respMakeLeave.LeaveEvent.StateKey = &request.UserID
+		respMakeLeave.LeaveEvent.StateKey = &senderIDString
 		respMakeLeave.LeaveEvent.RoomID = request.RoomID
 		respMakeLeave.LeaveEvent.Redacts = ""
 		leaveEB := verImpl.NewEventBuilderFromProtoEvent(&respMakeLeave.LeaveEvent)
@@ -470,7 +509,7 @@ func (r *FederationInternalAPI) PerformLeave(
 		// Build the leave event.
 		event, err := leaveEB.Build(
 			time.Now(),
-			origin,
+			userID.Domain(),
 			r.cfg.Matrix.KeyID,
 			r.cfg.Matrix.PrivateKey,
 		)
@@ -482,7 +521,7 @@ func (r *FederationInternalAPI) PerformLeave(
 		// Try to perform a send_leave using the newly built event.
 		err = r.federation.SendLeave(
 			ctx,
-			origin,
+			userID.Domain(),
 			serverName,
 			event,
 		)
@@ -509,7 +548,7 @@ func (r *FederationInternalAPI) SendInvite(
 	event gomatrixserverlib.PDU,
 	strippedState []gomatrixserverlib.InviteStrippedState,
 ) (gomatrixserverlib.PDU, error) {
-	_, origin, err := r.cfg.Matrix.SplitLocalID('@', event.Sender())
+	inviter, err := r.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
 	if err != nil {
 		return nil, err
 	}
@@ -532,7 +571,7 @@ func (r *FederationInternalAPI) SendInvite(
 	logrus.WithFields(logrus.Fields{
 		"event_id":     event.EventID(),
 		"user_id":      *event.StateKey(),
-		"room_id":      event.RoomID(),
+		"room_id":      event.RoomID().String(),
 		"room_version": event.Version(),
 		"destination":  destination,
 	}).Info("Sending invite")
@@ -542,7 +581,7 @@ func (r *FederationInternalAPI) SendInvite(
 		return nil, fmt.Errorf("gomatrixserverlib.NewInviteV2Request: %w", err)
 	}
 
-	inviteRes, err := r.federation.SendInviteV2(ctx, origin, destination, inviteReq)
+	inviteRes, err := r.federation.SendInviteV2(ctx, inviter.Domain(), destination, inviteReq)
 	if err != nil {
 		return nil, fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err)
 	}
@@ -558,6 +597,58 @@ func (r *FederationInternalAPI) SendInvite(
 	return inviteEvent, nil
 }
 
+// SendInviteV3 implements api.FederationInternalAPI
+func (r *FederationInternalAPI) SendInviteV3(
+	ctx context.Context,
+	event gomatrixserverlib.ProtoEvent,
+	invitee spec.UserID,
+	version gomatrixserverlib.RoomVersion,
+	strippedState []gomatrixserverlib.InviteStrippedState,
+) (gomatrixserverlib.PDU, error) {
+	validRoomID, err := spec.NewRoomID(event.RoomID)
+	if err != nil {
+		return nil, err
+	}
+	verImpl, err := gomatrixserverlib.GetRoomVersion(version)
+	if err != nil {
+		return nil, err
+	}
+
+	inviter, err := r.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(event.SenderID))
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO (devon): This should be allowed via a relay. Currently only transactions
+	// can be sent to relays. Would need to extend relays to handle invites.
+	if !r.shouldAttemptDirectFederation(invitee.Domain()) {
+		return nil, fmt.Errorf("relay servers have no meaningful response for invite.")
+	}
+
+	logrus.WithFields(logrus.Fields{
+		"user_id":      invitee.String(),
+		"room_id":      event.RoomID,
+		"room_version": version,
+		"destination":  invitee.Domain(),
+	}).Info("Sending invite")
+
+	inviteReq, err := fclient.NewInviteV3Request(event, version, strippedState)
+	if err != nil {
+		return nil, fmt.Errorf("gomatrixserverlib.NewInviteV3Request: %w", err)
+	}
+
+	inviteRes, err := r.federation.SendInviteV3(ctx, inviter.Domain(), invitee.Domain(), inviteReq, invitee)
+	if err != nil {
+		return nil, fmt.Errorf("r.federation.SendInviteV3: failed to send invite: %w", err)
+	}
+
+	inviteEvent, err := verImpl.NewEventFromUntrustedJSON(inviteRes.Event)
+	if err != nil {
+		return nil, fmt.Errorf("r.federation.SendInviteV3 failed to decode event response: %w", err)
+	}
+	return inviteEvent, nil
+}
+
 // PerformServersAlive implements api.FederationInternalAPI
 func (r *FederationInternalAPI) PerformBroadcastEDU(
 	ctx context.Context,
@@ -635,6 +726,7 @@ func checkEventsContainCreateEvent(events []gomatrixserverlib.PDU) error {
 func federatedEventProvider(
 	ctx context.Context, federation fclient.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier, origin, server spec.ServerName,
+	userIDForSender spec.UserIDForSender,
 ) gomatrixserverlib.EventProvider {
 	// A list of events that we have retried, if they were not included in
 	// the auth events supplied in the send_join.
@@ -684,7 +776,7 @@ func federatedEventProvider(
 				}
 
 				// Check the signatures of the event.
-				if err := gomatrixserverlib.VerifyEventSignatures(ctx, ev, keyRing); err != nil {
+				if err := gomatrixserverlib.VerifyEventSignatures(ctx, ev, keyRing, userIDForSender); err != nil {
 					return nil, fmt.Errorf("missingAuth VerifyEventSignatures: %w", err)
 				}
 

federationapi/internal/perform_test.go

@@ -16,6 +16,7 @@ package internal
 
 import (
 	"context"
+	"crypto/ed25519"
 	"testing"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
@@ -53,10 +54,14 @@ func TestPerformWakeupServers(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, offline)
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -65,7 +70,7 @@ func TestPerformWakeupServers(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -95,10 +100,14 @@ func TestQueryRelayServers(t *testing.T) {
 	err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers)
 	assert.NoError(t, err)
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -107,7 +116,7 @@ func TestQueryRelayServers(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -132,10 +141,14 @@ func TestRemoveRelayServers(t *testing.T) {
 	err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers)
 	assert.NoError(t, err)
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -144,7 +157,7 @@ func TestRemoveRelayServers(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -168,10 +181,14 @@ func TestRemoveRelayServers(t *testing.T) {
 func TestPerformDirectoryLookup(t *testing.T) {
 	testDB := test.NewInMemoryFederationDatabase()
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -180,7 +197,7 @@ func TestPerformDirectoryLookup(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -192,7 +209,7 @@ func TestPerformDirectoryLookup(t *testing.T) {
 		ServerName: "server",
 	}
 	res := api.PerformDirectoryLookupResponse{}
-	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	err = fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
 	assert.NoError(t, err)
 }
 
@@ -203,10 +220,14 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
 	testDB.SetServerAssumedOffline(context.Background(), server)
 	testDB.P2PAddRelayServersForServer(context.Background(), server, []spec.ServerName{"relay"})
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
-				ServerName: server,
+				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -215,7 +236,7 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -227,6 +248,6 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
 		ServerName: server,
 	}
 	res := api.PerformDirectoryLookupResponse{}
-	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	err = fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
 	assert.Error(t, err)
 }

federationapi/queue/destinationqueue.go

@@ -31,7 +31,6 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
-	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -53,7 +52,6 @@ type destinationQueue struct {
 	db                 storage.Database
 	process            *process.ProcessContext
 	signing            map[spec.ServerName]*fclient.SigningIdentity
-	rsAPI              api.FederationRoomserverAPI
 	client             fclient.FederationClient        // federation client
 	origin             spec.ServerName                 // origin of requests
 	destination        spec.ServerName                 // destination of requests

federationapi/queue/queue.go

@@ -27,12 +27,10 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
-	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
-	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -43,7 +41,6 @@ type OutgoingQueues struct {
 	db          storage.Database
 	process     *process.ProcessContext
 	disabled    bool
-	rsAPI       api.FederationRoomserverAPI
 	origin      spec.ServerName
 	client      fclient.FederationClient
 	statistics  *statistics.Statistics
@@ -90,7 +87,6 @@ func NewOutgoingQueues(
 	disabled bool,
 	origin spec.ServerName,
 	client fclient.FederationClient,
-	rsAPI api.FederationRoomserverAPI,
 	statistics *statistics.Statistics,
 	signing []*fclient.SigningIdentity,
 ) *OutgoingQueues {
@@ -98,7 +94,6 @@ func NewOutgoingQueues(
 		disabled:   disabled,
 		process:    process,
 		db:         db,
-		rsAPI:      rsAPI,
 		origin:     origin,
 		client:     client,
 		statistics: statistics,
@@ -162,7 +157,6 @@ func (oqs *OutgoingQueues) getQueue(destination spec.ServerName) *destinationQue
 			queues:      oqs,
 			db:          oqs.db,
 			process:     oqs.process,
-			rsAPI:       oqs.rsAPI,
 			origin:      oqs.origin,
 			destination: destination,
 			client:      oqs.client,
@@ -213,18 +207,6 @@ func (oqs *OutgoingQueues) SendEvent(
 		delete(destmap, local)
 	}
 
-	// Check if any of the destinations are prohibited by server ACLs.
-	for destination := range destmap {
-		if api.IsServerBannedFromRoom(
-			oqs.process.Context(),
-			oqs.rsAPI,
-			ev.RoomID(),
-			destination,
-		) {
-			delete(destmap, destination)
-		}
-	}
-
 	// If there are no remaining destinations then give up.
 	if len(destmap) == 0 {
 		return nil
@@ -303,24 +285,6 @@ func (oqs *OutgoingQueues) SendEDU(
 		delete(destmap, local)
 	}
 
-	// There is absolutely no guarantee that the EDU will have a room_id
-	// field, as it is not required by the spec. However, if it *does*
-	// (e.g. typing notifications) then we should try to make sure we don't
-	// bother sending them to servers that are prohibited by the server
-	// ACLs.
-	if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() {
-		for destination := range destmap {
-			if api.IsServerBannedFromRoom(
-				oqs.process.Context(),
-				oqs.rsAPI,
-				result.Str,
-				destination,
-			) {
-				delete(destmap, destination)
-			}
-		}
-	}
-
 	// If there are no remaining destinations then give up.
 	if len(destmap) == 0 {
 		return nil

federationapi/queue/queue_test.go

@@ -34,7 +34,6 @@ import (
 
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -65,15 +64,6 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase
 	}
 }
 
-type stubFederationRoomServerAPI struct {
-	rsapi.FederationRoomserverAPI
-}
-
-func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error {
-	res.Banned = false
-	return nil
-}
-
 type stubFederationClient struct {
 	fclient.FederationClient
 	shouldTxSucceed      bool
@@ -104,7 +94,7 @@ func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u
 
 func mustCreatePDU(t *testing.T) *types.HeaderedEvent {
 	t.Helper()
-	content := `{"type":"m.room.message"}`
+	content := `{"type":"m.room.message", "room_id":"!room:a"}`
 	ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV10).NewEventFromTrustedJSON([]byte(content), false)
 	if err != nil {
 		t.Fatalf("failed to create event: %v", err)
@@ -126,7 +116,6 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
 		txCount:              *atomic.NewUint32(0),
 		txRelayCount:         *atomic.NewUint32(0),
 	}
-	rs := &stubFederationRoomServerAPI{}
 
 	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline)
 	signingInfo := []*fclient.SigningIdentity{
@@ -136,7 +125,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
 			ServerName: "localhost",
 		},
 	}
-	queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, rs, &stats, signingInfo)
+	queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, &stats, signingInfo)
 
 	return db, fc, queues, processContext, close
 }

federationapi/routing/backfill.go

@@ -95,7 +95,7 @@ func Backfill(
 		}
 	}
 
-	// Query the roomserver.
+	// Query the Roomserver.
 	if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed")
 		return util.JSONResponse{
@@ -109,7 +109,7 @@ func Backfill(
 
 	var ev *types.HeaderedEvent
 	for _, ev = range res.Events {
-		if ev.RoomID() == roomID {
+		if ev.RoomID().String() == roomID {
 			evs = append(evs, ev.PDU)
 		}
 	}

federationapi/routing/eventauth.go

@@ -42,10 +42,10 @@ func GetEventAuth(
 		return *resErr
 	}
 
-	if event.RoomID() != roomID {
+	if event.RoomID().String() != roomID {
 		return util.JSONResponse{Code: http.StatusNotFound, JSON: spec.NotFound("event does not belong to this room")}
 	}
-	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID)
+	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String())
 	if resErr != nil {
 		return *resErr
 	}

federationapi/routing/events.go

@@ -35,10 +35,6 @@ func GetEvent(
 	eventID string,
 	origin spec.ServerName,
 ) util.JSONResponse {
-	err := allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID)
-	if err != nil {
-		return *err
-	}
 	// /_matrix/federation/v1/event/{eventId} doesn't have a roomID, we use an empty string,
 	// which results in `QueryEventsByID` to first get the event and use that to determine the roomID.
 	event, err := fetchEvent(ctx, rsAPI, "", eventID)
@@ -46,6 +42,11 @@ func GetEvent(
 		return *err
 	}
 
+	err = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String())
+	if err != nil {
+		return *err
+	}
+
 	return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.Transaction{
 		Origin:         origin,
 		OriginServerTS: spec.AsTimestamp(time.Now()),
@@ -62,8 +63,9 @@ func allowedToSeeEvent(
 	origin spec.ServerName,
 	rsAPI api.FederationRoomserverAPI,
 	eventID string,
+	roomID string,
 ) *util.JSONResponse {
-	allowed, err := rsAPI.QueryServerAllowedToSeeEvent(ctx, origin, eventID)
+	allowed, err := rsAPI.QueryServerAllowedToSeeEvent(ctx, origin, eventID, roomID)
 	if err != nil {
 		resErr := util.ErrorResponse(err)
 		return &resErr

federationapi/routing/invite.go

@@ -16,6 +16,7 @@ package routing
 
 import (
 	"context"
+	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -29,6 +30,73 @@ import (
 	"github.com/matrix-org/util"
 )
 
+// InviteV3 implements /_matrix/federation/v2/invite/{roomID}/{userID}
+func InviteV3(
+	httpReq *http.Request,
+	request *fclient.FederationRequest,
+	roomID spec.RoomID,
+	invitedUser spec.UserID,
+	cfg *config.FederationAPI,
+	rsAPI api.FederationRoomserverAPI,
+	keys gomatrixserverlib.JSONVerifier,
+) util.JSONResponse {
+	inviteReq := fclient.InviteV3Request{}
+	err := json.Unmarshal(request.Content(), &inviteReq)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON(err.Error()),
+		}
+	}
+	if !cfg.Matrix.IsLocalServerName(invitedUser.Domain()) {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("The invited user domain does not belong to this server"),
+		}
+	}
+
+	input := gomatrixserverlib.HandleInviteV3Input{
+		HandleInviteInput: gomatrixserverlib.HandleInviteInput{
+			RoomVersion:       inviteReq.RoomVersion(),
+			RoomID:            roomID,
+			InvitedUser:       invitedUser,
+			KeyID:             cfg.Matrix.KeyID,
+			PrivateKey:        cfg.Matrix.PrivateKey,
+			Verifier:          keys,
+			RoomQuerier:       rsAPI,
+			MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI},
+			StateQuerier:      rsAPI.StateQuerier(),
+			InviteEvent:       nil,
+			StrippedState:     inviteReq.InviteRoomState(),
+			UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+				return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID)
+			},
+		},
+		InviteProtoEvent: inviteReq.Event(),
+		GetOrCreateSenderID: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) {
+			// assign a roomNID, otherwise we can't create a private key for the user
+			_, nidErr := rsAPI.AssignRoomNID(ctx, roomID, gomatrixserverlib.RoomVersion(roomVersion))
+			if nidErr != nil {
+				return "", nil, nidErr
+			}
+			key, keyErr := rsAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID)
+			if keyErr != nil {
+				return "", nil, keyErr
+			}
+
+			return spec.SenderIDFromPseudoIDKey(key), key, nil
+		},
+	}
+	event, jsonErr := handleInviteV3(httpReq.Context(), input, rsAPI)
+	if jsonErr != nil {
+		return *jsonErr
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: fclient.RespInviteV2{Event: event.JSON()},
+	}
+}
+
 // InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID}
 func InviteV2(
 	httpReq *http.Request,
@@ -95,6 +163,9 @@ func InviteV2(
 			StateQuerier:      rsAPI.StateQuerier(),
 			InviteEvent:       inviteReq.Event(),
 			StrippedState:     inviteReq.InviteRoomState(),
+			UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+				return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID)
+			},
 		}
 		event, jsonErr := handleInvite(httpReq.Context(), input, rsAPI)
 		if jsonErr != nil {
@@ -185,6 +256,9 @@ func InviteV1(
 		StateQuerier:      rsAPI.StateQuerier(),
 		InviteEvent:       event,
 		StrippedState:     strippedState,
+		UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID)
+		},
 	}
 	event, jsonErr := handleInvite(httpReq.Context(), input, rsAPI)
 	if jsonErr != nil {
@@ -198,6 +272,15 @@ func InviteV1(
 
 func handleInvite(ctx context.Context, input gomatrixserverlib.HandleInviteInput, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) {
 	inviteEvent, err := gomatrixserverlib.HandleInvite(ctx, input)
+	return handleInviteResult(ctx, inviteEvent, err, rsAPI)
+}
+
+func handleInviteV3(ctx context.Context, input gomatrixserverlib.HandleInviteV3Input, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) {
+	inviteEvent, err := gomatrixserverlib.HandleInviteV3(ctx, input)
+	return handleInviteResult(ctx, inviteEvent, err, rsAPI)
+}
+
+func handleInviteResult(ctx context.Context, inviteEvent gomatrixserverlib.PDU, err error, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) {
 	switch e := err.(type) {
 	case nil:
 	case spec.InternalServerError:
@@ -239,4 +322,5 @@ func handleInvite(ctx context.Context, input gomatrixserverlib.HandleInviteInput
 		}
 	}
 	return inviteEvent, nil
+
 }

federationapi/routing/join.go

@@ -33,53 +33,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-type JoinRoomQuerier struct {
-	roomserver api.FederationRoomserverAPI
-}
-
-func (rq *JoinRoomQuerier) CurrentStateEvent(ctx context.Context, roomID spec.RoomID, eventType string, stateKey string) (gomatrixserverlib.PDU, error) {
-	return rq.roomserver.CurrentStateEvent(ctx, roomID, eventType, stateKey)
-}
-
-func (rq *JoinRoomQuerier) InvitePending(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (bool, error) {
-	return rq.roomserver.InvitePending(ctx, roomID, userID)
-}
-
-func (rq *JoinRoomQuerier) RestrictedRoomJoinInfo(ctx context.Context, roomID spec.RoomID, userID spec.UserID, localServerName spec.ServerName) (*gomatrixserverlib.RestrictedRoomJoinInfo, error) {
-	roomInfo, err := rq.roomserver.QueryRoomInfo(ctx, roomID)
-	if err != nil || roomInfo == nil || roomInfo.IsStub() {
-		return nil, err
-	}
-
-	req := api.QueryServerJoinedToRoomRequest{
-		ServerName: localServerName,
-		RoomID:     roomID.String(),
-	}
-	res := api.QueryServerJoinedToRoomResponse{}
-	if err = rq.roomserver.QueryServerJoinedToRoom(ctx, &req, &res); err != nil {
-		util.GetLogger(ctx).WithError(err).Error("rsAPI.QueryServerJoinedToRoom failed")
-		return nil, fmt.Errorf("InternalServerError: Failed to query room: %w", err)
-	}
-
-	userJoinedToRoom, err := rq.roomserver.UserJoinedToRoom(ctx, types.RoomNID(roomInfo.RoomNID), userID)
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("rsAPI.UserJoinedToRoom failed")
-		return nil, fmt.Errorf("InternalServerError: %w", err)
-	}
-
-	locallyJoinedUsers, err := rq.roomserver.LocallyJoinedUsers(ctx, roomInfo.RoomVersion, types.RoomNID(roomInfo.RoomNID))
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("rsAPI.GetLocallyJoinedUsers failed")
-		return nil, fmt.Errorf("InternalServerError: %w", err)
-	}
-
-	return &gomatrixserverlib.RestrictedRoomJoinInfo{
-		LocalServerInRoom: res.RoomExists && res.IsInRoom,
-		UserJoinedToRoom:  userJoinedToRoom,
-		JoinedUsers:       locallyJoinedUsers,
-	}, nil
-}
-
 // MakeJoin implements the /make_join API
 func MakeJoin(
 	httpReq *http.Request,
@@ -103,7 +56,7 @@ func MakeJoin(
 		RoomID:     roomID.String(),
 	}
 	res := api.QueryServerJoinedToRoomResponse{}
-	if err := rsAPI.QueryServerJoinedToRoom(httpReq.Context(), &req, &res); err != nil {
+	if err = rsAPI.QueryServerJoinedToRoom(httpReq.Context(), &req, &res); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryServerJoinedToRoom failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -112,26 +65,26 @@ func MakeJoin(
 	}
 
 	createJoinTemplate := func(proto *gomatrixserverlib.ProtoEvent) (gomatrixserverlib.PDU, []gomatrixserverlib.PDU, error) {
-		identity, err := cfg.Matrix.SigningIdentityFor(request.Destination())
+		identity, signErr := cfg.Matrix.SigningIdentityFor(request.Destination())
-		if err != nil {
+		if signErr != nil {
-			util.GetLogger(httpReq.Context()).WithError(err).Errorf("obtaining signing identity for %s failed", request.Destination())
+			util.GetLogger(httpReq.Context()).WithError(signErr).Errorf("obtaining signing identity for %s failed", request.Destination())
 			return nil, nil, spec.NotFound(fmt.Sprintf("Server name %q does not exist", request.Destination()))
 		}
 
 		queryRes := api.QueryLatestEventsAndStateResponse{
 			RoomVersion: roomVersion,
 		}
-		event, err := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, identity, time.Now(), rsAPI, &queryRes)
+		event, signErr := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, identity, time.Now(), rsAPI, &queryRes)
-		switch e := err.(type) {
+		switch e := signErr.(type) {
 		case nil:
 		case eventutil.ErrRoomNoExists:
-			util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed")
+			util.GetLogger(httpReq.Context()).WithError(signErr).Error("eventutil.BuildEvent failed")
 			return nil, nil, spec.NotFound("Room does not exist")
 		case gomatrixserverlib.BadJSONError:
-			util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed")
+			util.GetLogger(httpReq.Context()).WithError(signErr).Error("eventutil.BuildEvent failed")
 			return nil, nil, spec.BadJSON(e.Error())
 		default:
-			util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed")
+			util.GetLogger(httpReq.Context()).WithError(signErr).Error("eventutil.BuildEvent failed")
 			return nil, nil, spec.InternalServerError{}
 		}
 
@@ -142,13 +95,30 @@ func MakeJoin(
 		return event, stateEvents, nil
 	}
 
-	roomQuerier := JoinRoomQuerier{
+	roomQuerier := api.JoinRoomQuerier{
-		roomserver: rsAPI,
+		Roomserver: rsAPI,
+	}
+
+	senderIDPtr, err := rsAPI.QuerySenderIDForUser(httpReq.Context(), roomID, userID)
+	if err != nil {
+		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QuerySenderIDForUser failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	var senderID spec.SenderID
+	if senderIDPtr == nil {
+		senderID = spec.SenderID(userID.String())
+	} else {
+		senderID = *senderIDPtr
 	}
 
 	input := gomatrixserverlib.HandleMakeJoinInput{
 		Context:           httpReq.Context(),
 		UserID:            userID,
+		SenderID:          senderID,
 		RoomID:            roomID,
 		RoomVersion:       roomVersion,
 		RemoteVersions:    remoteVersions,
@@ -156,6 +126,9 @@ func MakeJoin(
 		LocalServerName:   cfg.Matrix.ServerName,
 		LocalServerInRoom: res.RoomExists && res.IsInRoom,
 		RoomQuerier:       &roomQuerier,
+		UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID)
+		},
 		BuildEventTemplate: createJoinTemplate,
 	}
 	response, internalErr := gomatrixserverlib.HandleMakeJoin(input)
@@ -217,9 +190,6 @@ func MakeJoin(
 }
 
 // SendJoin implements the /send_join API
-// The make-join send-join dance makes much more sense as a single
-// flow so the cyclomatic complexity is high:
-// nolint:gocyclo
 func SendJoin(
 	httpReq *http.Request,
 	request *fclient.FederationRequest,
@@ -250,6 +220,16 @@ func SendJoin(
 		PrivateKey:        cfg.Matrix.PrivateKey,
 		Verifier:          keys,
 		MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI},
+		UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID)
+		},
+		StoreSenderIDFromPublicID: func(ctx context.Context, senderID spec.SenderID, userIDRaw string, roomID spec.RoomID) error {
+			userID, userErr := spec.NewUserID(userIDRaw, true)
+			if userErr != nil {
+				return userErr
+			}
+			return rsAPI.StoreUserRoomPublicKey(ctx, senderID, *userID, roomID)
+		},
 	}
 	response, joinErr := gomatrixserverlib.HandleSendJoin(input)
 	switch e := joinErr.(type) {

federationapi/routing/leave.go

@@ -50,7 +50,7 @@ func MakeLeave(
 		RoomID:     roomID.String(),
 	}
 	res := api.QueryServerJoinedToRoomResponse{}
-	if err := rsAPI.QueryServerJoinedToRoom(httpReq.Context(), &req, &res); err != nil {
+	if err = rsAPI.QueryServerJoinedToRoom(httpReq.Context(), &req, &res); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryServerJoinedToRoom failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -59,24 +59,24 @@ func MakeLeave(
 	}
 
 	createLeaveTemplate := func(proto *gomatrixserverlib.ProtoEvent) (gomatrixserverlib.PDU, []gomatrixserverlib.PDU, error) {
-		identity, err := cfg.Matrix.SigningIdentityFor(request.Destination())
+		identity, signErr := cfg.Matrix.SigningIdentityFor(request.Destination())
-		if err != nil {
+		if signErr != nil {
-			util.GetLogger(httpReq.Context()).WithError(err).Errorf("obtaining signing identity for %s failed", request.Destination())
+			util.GetLogger(httpReq.Context()).WithError(signErr).Errorf("obtaining signing identity for %s failed", request.Destination())
 			return nil, nil, spec.NotFound(fmt.Sprintf("Server name %q does not exist", request.Destination()))
 		}
 
 		queryRes := api.QueryLatestEventsAndStateResponse{}
-		event, err := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, identity, time.Now(), rsAPI, &queryRes)
+		event, buildErr := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, identity, time.Now(), rsAPI, &queryRes)
-		switch e := err.(type) {
+		switch e := buildErr.(type) {
 		case nil:
 		case eventutil.ErrRoomNoExists:
-			util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed")
+			util.GetLogger(httpReq.Context()).WithError(buildErr).Error("eventutil.BuildEvent failed")
 			return nil, nil, spec.NotFound("Room does not exist")
 		case gomatrixserverlib.BadJSONError:
-			util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed")
+			util.GetLogger(httpReq.Context()).WithError(buildErr).Error("eventutil.BuildEvent failed")
 			return nil, nil, spec.BadJSON(e.Error())
 		default:
-			util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed")
+			util.GetLogger(httpReq.Context()).WithError(buildErr).Error("eventutil.BuildEvent failed")
 			return nil, nil, spec.InternalServerError{}
 		}
 
@@ -87,14 +87,33 @@ func MakeLeave(
 		return event, stateEvents, nil
 	}
 
+	senderID, err := rsAPI.QuerySenderIDForUser(httpReq.Context(), roomID, userID)
+	if err != nil {
+		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QuerySenderIDForUser failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	} else if senderID == nil {
+		util.GetLogger(httpReq.Context()).WithField("roomID", roomID).WithField("userID", userID).Error("rsAPI.QuerySenderIDForUser returned nil sender ID")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
 	input := gomatrixserverlib.HandleMakeLeaveInput{
 		UserID:             userID,
+		SenderID:           *senderID,
 		RoomID:             roomID,
 		RoomVersion:        roomVersion,
 		RequestOrigin:      request.Origin(),
 		LocalServerName:    cfg.Matrix.ServerName,
 		LocalServerInRoom:  res.RoomExists && res.IsInRoom,
 		BuildEventTemplate: createLeaveTemplate,
+		UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID)
+		},
 	}
 
 	response, internalErr := gomatrixserverlib.HandleMakeLeave(input)
@@ -168,13 +187,15 @@ func SendLeave(
 	verImpl, err := gomatrixserverlib.GetRoomVersion(roomVersion)
 	if err != nil {
 		return util.JSONResponse{
-			Code: http.StatusBadRequest,
+			Code: http.StatusInternalServerError,
-			JSON: spec.UnsupportedRoomVersion(err.Error()),
+			JSON: spec.UnsupportedRoomVersion(
+				fmt.Sprintf("QueryRoomVersionForRoom returned unknown version: %s", roomVersion),
+			),
 		}
 	}
 
-	// Decode the incomingEvent JSON from the request.
+	// Decode the event JSON from the request.
-	incomingEvent, err := verImpl.NewEventFromUntrustedJSON(request.Content())
+	event, err := verImpl.NewEventFromUntrustedJSON(request.Content())
 	switch err.(type) {
 	case gomatrixserverlib.BadJSONError:
 		return util.JSONResponse{

federationapi/routing/missingevents.go

@@ -87,7 +87,7 @@ func filterEvents(
 ) []*types.HeaderedEvent {
 	ref := events[:0]
 	for _, ev := range events {
-		if ev.RoomID() == roomID {
+		if ev.RoomID().String() == roomID {
 			ref = append(ref, ev)
 		}
 	}

federationapi/routing/profile.go

@@ -15,9 +15,11 @@
 package routing
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
 
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -52,6 +54,12 @@ func GetProfile(
 
 	profile, err := userAPI.QueryProfile(httpReq.Context(), userID)
 	if err != nil {
+		if errors.Is(err, appserviceAPI.ErrProfileNotExists) {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: spec.NotFound("The user does not exist or does not have a profile."),
+			}
+		}
 		util.GetLogger(httpReq.Context()).WithError(err).Error("userAPI.QueryProfile failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,

federationapi/routing/profile_test.go

@@ -26,7 +26,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
-	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -67,11 +66,8 @@ func TestHandleQueryProfile(t *testing.T) {
 		keyRing := serverKeyAPI.KeyRing()
 		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true)
 		userapi := fakeUserAPI{}
-		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+
-		if !ok {
+		routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
-			panic("This is a programming error.")
-		}
-		routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)

federationapi/routing/query.go

@@ -20,12 +20,14 @@ import (
 
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
+	log "github.com/sirupsen/logrus"
 )
 
 // RoomAliasToID converts the queried alias into a room ID and returns it
@@ -116,3 +118,65 @@ func RoomAliasToID(
 		JSON: resp,
 	}
 }
+
+// Query the immediate children of a room/space
+//
+// Implements /_matrix/federation/v1/hierarchy/{roomID}
+func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationRequest, roomIDStr string, rsAPI roomserverAPI.FederationRoomserverAPI) util.JSONResponse {
+	parsedRoomID, err := spec.NewRoomID(roomIDStr)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.InvalidParam("room is unknown/forbidden"),
+		}
+	}
+	roomID := *parsedRoomID
+
+	suggestedOnly := false // Defaults to false (spec-defined)
+	switch httpReq.URL.Query().Get("suggested_only") {
+	case "true":
+		suggestedOnly = true
+	case "false":
+	case "": // Empty string is returned when query param is not set
+	default:
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"),
+		}
+	}
+
+	walker := roomserverAPI.NewRoomHierarchyWalker(types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1)
+	discoveredRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1)
+
+	if err != nil {
+		switch err.(type) {
+		case roomserverAPI.ErrRoomUnknownOrNotAllowed:
+			util.GetLogger(httpReq.Context()).WithError(err).Debugln("room unknown/forbidden when handling SS room hierarchy request")
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: spec.NotFound("room is unknown/forbidden"),
+			}
+		default:
+			log.WithError(err).Errorf("failed to fetch next page of room hierarchy (SS API)")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+	}
+
+	if len(discoveredRooms) == 0 {
+		util.GetLogger(httpReq.Context()).Debugln("no rooms found when handling SS room hierarchy request")
+		return util.JSONResponse{
+			Code: 404,
+			JSON: spec.NotFound("room is unknown/forbidden"),
+		}
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: fclient.RoomHierarchyResponse{
+			Room:     discoveredRooms[0],
+			Children: discoveredRooms[1:],
+		},
+	}
+}

federationapi/routing/query_test.go

@@ -25,7 +25,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
-	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -65,11 +64,8 @@ func TestHandleQueryDirectory(t *testing.T) {
 		keyRing := serverKeyAPI.KeyRing()
 		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true)
 		userapi := fakeUserAPI{}
-		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+
-		if !ok {
+		routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
-			panic("This is a programming error.")
-		}
-		routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)

federationapi/routing/routing.go

@@ -78,6 +78,7 @@ func Setup(
 	v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
 	v1fedmux := fedMux.PathPrefix("/v1").Subrouter()
 	v2fedmux := fedMux.PathPrefix("/v2").Subrouter()
+	v3fedmux := fedMux.PathPrefix("/v3").Subrouter()
 
 	wakeup := &FederationWakeups{
 		FsAPI: fsAPI,
@@ -191,6 +192,37 @@ func Setup(
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
 
+	v3fedmux.Handle("/invite/{roomID}/{userID}", MakeFedAPI(
+		"federation_invite", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,
+		func(httpReq *http.Request, request *fclient.FederationRequest, vars map[string]string) util.JSONResponse {
+			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
+				return util.JSONResponse{
+					Code: http.StatusForbidden,
+					JSON: spec.Forbidden("Forbidden by server ACLs"),
+				}
+			}
+
+			userID, err := spec.NewUserID(vars["userID"], true)
+			if err != nil {
+				return util.JSONResponse{
+					Code: http.StatusBadRequest,
+					JSON: spec.InvalidParam("Invalid UserID"),
+				}
+			}
+			roomID, err := spec.NewRoomID(vars["roomID"])
+			if err != nil {
+				return util.JSONResponse{
+					Code: http.StatusBadRequest,
+					JSON: spec.InvalidParam("Invalid RoomID"),
+				}
+			}
+			return InviteV3(
+				httpReq, request, *roomID, *userID,
+				cfg, rsAPI, keys,
+			)
+		},
+	)).Methods(http.MethodPut, http.MethodOptions)
+
 	v1fedmux.Handle("/3pid/onbind", httputil.MakeExternalAPI("3pid_onbind",
 		func(req *http.Request) util.JSONResponse {
 			return CreateInvitesFrom3PIDInvites(req, rsAPI, cfg, federation, userAPI)
@@ -564,6 +596,13 @@ func Setup(
 			return GetOpenIDUserInfo(req, userAPI)
 		}),
 	).Methods(http.MethodGet)
+
+	v1fedmux.Handle("/hierarchy/{roomID}", MakeFedAPI(
+		"federation_room_hierarchy", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,
+		func(httpReq *http.Request, request *fclient.FederationRequest, vars map[string]string) util.JSONResponse {
+			return QueryRoomHierarchy(httpReq, request, vars["roomID"], rsAPI)
+		},
+	)).Methods(http.MethodGet)
 }
 
 func ErrorIfLocalServerNotInRoom(

federationapi/routing/send.go

@@ -34,7 +34,7 @@ import (
 )
 
 const (
-	// Event was passed to the roomserver
+	// Event was passed to the Roomserver
 	MetricsOutcomeOK = "ok"
 	// Event failed to be processed
 	MetricsOutcomeFail = "fail"

federationapi/routing/send_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
-	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -62,11 +61,8 @@ func TestHandleSend(t *testing.T) {
 		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, nil, nil, nil, true)
 		serverKeyAPI := &signing.YggdrasilKeys{}
 		keyRing := serverKeyAPI.KeyRing()
-		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+
-		if !ok {
+		routing.Setup(routers, cfg, nil, fedapi, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics)
-			panic("This is a programming error.")
-		}
-		routing.Setup(routers, cfg, nil, r, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)

federationapi/routing/state.go

@@ -113,10 +113,10 @@ func getState(
 		return nil, nil, resErr
 	}
 
-	if event.RoomID() != roomID {
+	if event.RoomID().String() != roomID {
 		return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: spec.NotFound("event does not belong to this room")}
 	}
-	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID)
+	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String())
 	if resErr != nil {
 		return nil, nil, resErr
 	}

federationapi/routing/threepid.go

@@ -140,22 +140,31 @@ func ExchangeThirdPartyInvite(
 		}
 	}
 
-	_, senderDomain, err := cfg.Matrix.SplitLocalID('@', proto.Sender)
+	validRoomID, err := spec.NewRoomID(roomID)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Invalid sender ID: " + err.Error()),
+			JSON: spec.BadJSON("Invalid room ID"),
 		}
 	}
+	userID, err := rsAPI.QueryUserIDForSender(httpReq.Context(), *validRoomID, spec.SenderID(proto.SenderID))
+	if err != nil || userID == nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON("Invalid sender ID"),
+		}
+	}
+	senderDomain := userID.Domain()
 
 	// Check that the state key is correct.
-	_, targetDomain, err := gomatrixserverlib.SplitID('@', *proto.StateKey)
+	targetUserID, err := rsAPI.QueryUserIDForSender(httpReq.Context(), *validRoomID, spec.SenderID(*proto.StateKey))
-	if err != nil {
+	if err != nil || targetUserID == nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: spec.BadJSON("The event's state key isn't a Matrix user ID"),
 		}
 	}
+	targetDomain := targetUserID.Domain()
 
 	// Check that the target user is from the requesting homeserver.
 	if targetDomain != request.Origin() {
@@ -223,7 +232,7 @@ func ExchangeThirdPartyInvite(
 		}
 	}
 
-	// Send the event to the roomserver
+	// Send the event to the Roomserver
 	if err = api.SendEvents(
 		httpReq.Context(), rsAPI,
 		api.KindNew,
@@ -271,7 +280,7 @@ func createInviteFrom3PIDInvite(
 	// Build the event
 	proto := &gomatrixserverlib.ProtoEvent{
 		Type:     "m.room.member",
-		Sender:   inv.Sender,
+		SenderID: inv.Sender,
 		RoomID:   inv.RoomID,
 		StateKey: &inv.MXID,
 	}
@@ -324,7 +333,7 @@ func buildMembershipEvent(
 		return nil, errors.New("expecting state tuples for event builder, got none")
 	}
 
-	// Ask the roomserver for information about this room
+	// Ask the Roomserver for information about this room
 	queryReq := api.QueryLatestEventsAndStateRequest{
 		RoomID:       protoEvent.RoomID,
 		StateToFetch: eventsNeeded.Tuples(),

federationapi/storage/postgres/notary_server_keys_metadata_table.go

@@ -151,7 +151,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn
 		}
 		results = append(results, sk)
 	}
-	return results, nil
+	return results, rows.Err()
 }
 
 func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error {

federationapi/storage/postgres/queue_json_table.go

@@ -109,5 +109,5 @@ func (s *queueJSONStatements) SelectQueueJSON(
 		}
 		blobs[nid] = blob
 	}
-	return blobs, err
+	return blobs, rows.Err()
 }

federationapi/storage/postgres/relay_servers_table.go

@@ -110,7 +110,7 @@ func (s *relayServersStatements) SelectRelayServers(
 		}
 		result = append(result, spec.ServerName(relayServer))
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *relayServersStatements) DeleteRelayServers(

federationapi/storage/postgres/server_key_table.go

@@ -94,12 +94,14 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed")
 	results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
-	for rows.Next() {
+
 	var serverName string
 	var keyID string
 	var key string
 	var validUntilTS int64
 	var expiredTS int64
+	var vk gomatrixserverlib.VerifyKey
+	for rows.Next() {
 		if err = rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil {
 			return nil, err
 		}
@@ -107,7 +109,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 			ServerName: spec.ServerName(serverName),
 			KeyID:      gomatrixserverlib.KeyID(keyID),
 		}
-		vk := gomatrixserverlib.VerifyKey{}
 		err = vk.Key.Decode(key)
 		if err != nil {
 			return nil, err

federationapi/storage/postgres/storage.go

@@ -36,7 +36,7 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool) (*Database, error) {
+func NewDatabase(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool) (*Database, error) {
 	var d Database
 	var err error
 	if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil {

federationapi/storage/sqlite3/joined_hosts_table.go

@@ -216,5 +216,5 @@ func joinedHostsFromStmt(
 		})
 	}
 
-	return result, nil
+	return result, rows.Err()
 }