Merge branch 'main' of github.com:matrix-org/dendrite into gh-pages

.github/workflows/dendrite.yml

@@ -6,10 +6,12 @@ on:
       - main
     paths:
       - '**.go'                 # only execute on changes to go files
+      - 'go.sum'                # or dependency updates
       - '.github/workflows/**'  # or workflow changes
   pull_request:
     paths:
       - '**.go'
+      - 'go.sum'                # or dependency updates
       - '.github/workflows/**'
   release:
     types: [published]
@@ -391,7 +393,7 @@ jobs:
         # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+          go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
       - name: Run actions/checkout@v3 for dendrite
         uses: actions/checkout@v3
         with:

.github/workflows/helm.yml

@@ -6,6 +6,7 @@ on:
       - main
     paths:
       - 'helm/**' # only execute if we have helm chart changes
+  workflow_dispatch:
 
 jobs:
   release:

.github/workflows/k8s.yml

@@ -84,6 +84,7 @@ jobs:
           kubectl get pods -A
           kubectl get services
           kubectl get ingress
+          kubectl logs -l app.kubernetes.io/name=dendrite
       - name: Run create account
         run: |
           podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name)

.github/workflows/schedules.yaml

@@ -65,10 +65,11 @@ jobs:
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
+          name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }})
           path: |
             /logs/results.tap
             /logs/**/*.log*
+            /logs/**/covdatafiles/**
 
   sytest-coverage:
     timeout-minutes: 5
@@ -85,16 +86,15 @@ jobs:
           cache: true
       - name: Download all artifacts
         uses: actions/download-artifact@v3
-      - name: Install gocovmerge
+      - name: Collect coverage
-        run: go install github.com/wadey/gocovmerge@latest
-      - name: Run gocovmerge
         run: |
-          find -name 'integrationcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > sytest.cov
+          go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov
-          go tool cover -func=sytest.cov
+          grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov
+          go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          files: ./sytest.cov
+          files: ./final.cov
           flags: sytest
           fail_ci_if_error: true
 
@@ -167,7 +167,7 @@ jobs:
           cat <<EOF > /tmp/posttest.sh
           #!/bin/bash
           mkdir -p /tmp/Complement/logs/\$2/\$1/
-          docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
+          docker cp \$1:/tmp/covdatafiles/. /tmp/Complement/logs/\$2/\$1/
           EOF
           
           chmod +x /tmp/posttest.sh
@@ -188,9 +188,9 @@ jobs:
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
+          name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }})
           path: |
-            /tmp/Complement/**/complementcover.log
+            /tmp/Complement/logs/**
 
   complement-coverage:
     timeout-minutes: 5
@@ -207,20 +207,19 @@ jobs:
           cache: true
       - name: Download all artifacts
         uses: actions/download-artifact@v3
-      - name: Install gocovmerge
+      - name: Collect coverage
-        run: go install github.com/wadey/gocovmerge@latest
-      - name: Run gocovmerge
         run: |
-          find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
+          go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov
-          go tool cover -func=complement.cov
+          grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
+          go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          files: ./complement.cov
+          files: ./final.cov
           flags: complement
           fail_ci_if_error: true
 
-  element_web:
+  element-web:
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -258,3 +257,42 @@ jobs:
         env:
           PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
           TMPDIR: ${{ runner.temp }}
+
+  element-web-pinecone:
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tecolicom/actions-use-apt-tools@v1
+        with:
+          # Our test suite includes some screenshot tests with unusual diacritics, which are
+          # supposed to be covered by STIXGeneral.
+          tools: fonts-stix
+      - uses: actions/checkout@v2
+        with:
+          repository: matrix-org/matrix-react-sdk
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+      - name: Fetch layered build
+        run: scripts/ci/layered.sh
+      - name: Copy config
+        run: cp element.io/develop/config.json config.json
+        working-directory: ./element-web
+      - name: Build
+        env:
+          CI_PACKAGE: true
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        run: yarn build
+        working-directory: ./element-web
+      - name: Edit Test Config
+        run: |
+          sed -i '/HOMESERVER/c\        HOMESERVER: "dendritePinecone",' cypress.config.ts
+      - name: "Run cypress tests"
+        uses: cypress-io/github-action@v4.1.1
+        with:
+          browser: chrome
+          start: npx serve -p 8080 ./element-web/webapp
+          wait-on: 'http://localhost:8080'
+        env:
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          TMPDIR: ${{ runner.temp }}

.gitignore

@@ -74,3 +74,4 @@ complement/
 docs/_site
 
 media_store/
+build

.golangci.yml

@@ -179,7 +179,6 @@ linters-settings:
 
 linters:
   enable:
-    - deadcode
     - errcheck
     - goconst
     - gocyclo
@@ -191,10 +190,8 @@ linters:
     - misspell # Check code comments, whereas misspell in CI checks *.md files
     - nakedret
     - staticcheck
-    - structcheck
     - unparam
     - unused
-    - varcheck
   enable-all: false
   disable:
     - bodyclose

CHANGES.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## Dendrite 0.12.0 (2023-03-13)
+
+### Features
+
+- The userapi and keyserver have been merged (no actions needed regarding the database)
+- The internal NATS JetStream server is now using logrus for logging (contributed by [dvob](https://github.com/dvob))
+- The roomserver database has been refactored to have separate interfaces when working with rooms and events. Also includes increased usage of the cache to avoid database round trips. (database is unchanged)
+- The pinecone demo now shuts down more cleanly
+- The Helm chart now has the ability to deploy a Grafana chart as well (contributed by [genofire](https://github.com/genofire))
+- Support for listening on unix sockets has been added (contributed by [cyberb](https://github.com/cyberb))
+- The internal NATS server was updated to v2.9.15
+- Initial support for `runtime/trace` has been added, to further track down long-running tasks
+
+### Fixes
+
+- The `session_id` is now correctly set when using SQLite
+- An issue where device keys could be removed if a device ID is reused has been fixed
+- A possible DoS issue related to relations has been fixed (reported by [sleroq](https://github.com/sleroq))
+- When backfilling events, errors are now ignored if we still could fetch events
+
+### Other
+
+- **⚠️ DEPRECATION: Polylith/HTTP API mode has been removed**
+- The default endpoint to report usages stats to has been updated
+
 ## Dendrite 0.11.1 (2023-02-10)
 
 **⚠️ DEPRECATION WARNING: This is the last release to have polylith and HTTP API mode. Future releases are monolith only.**

appservice/api/query.go

@@ -22,8 +22,6 @@ import (
 	"encoding/json"
 	"errors"
 
-	"github.com/matrix-org/gomatrixserverlib"
-
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
@@ -150,6 +148,10 @@ type ASLocationResponse struct {
 	Fields   json.RawMessage `json:"fields"`
 }
 
+// ErrProfileNotExists is returned when trying to lookup a user's profile that
+// doesn't exist locally.
+var ErrProfileNotExists = errors.New("no known profile for given user ID")
+
 // RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
 // TODO: Remove this, it's called from federationapi and clientapi but is a pure function
@@ -157,25 +159,11 @@ func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
 	asAPI AppServiceInternalAPI,
-	profileAPI userapi.ClientUserAPI,
+	profileAPI userapi.ProfileAPI,
 ) (*authtypes.Profile, error) {
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
-	if err != nil {
-		return nil, err
-	}
-
 	// Try to query the user from the local database
-	res := &userapi.QueryProfileResponse{}
+	profile, err := profileAPI.QueryProfile(ctx, userID)
-	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
+	if err == nil {
-	if err != nil {
-		return nil, err
-	}
-	profile := &authtypes.Profile{
-		Localpart:   localpart,
-		DisplayName: res.DisplayName,
-		AvatarURL:   res.AvatarURL,
-	}
-	if res.UserExists {
 		return profile, nil
 	}
 
@@ -188,19 +176,15 @@ func RetrieveUserProfile(
 
 	// If no user exists, return
 	if !userResp.UserIDExists {
-		return nil, errors.New("no known profile for given user ID")
+		return nil, ErrProfileNotExists
 	}
 
 	// Try to query the user from the local database again
-	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
+	profile, err = profileAPI.QueryProfile(ctx, userID)
 	if err != nil {
 		return nil, err
 	}
 
 	// profile should not be nil at this point
-	return &authtypes.Profile{
+	return profile, nil
-		Localpart:   localpart,
-		DisplayName: res.DisplayName,
-		AvatarURL:   res.AvatarURL,
-	}, nil
 }

appservice/appservice.go

@@ -16,20 +16,17 @@ package appservice
 
 import (
 	"context"
-	"crypto/tls"
-	"net/http"
 	"sync"
-	"time"
 
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 
-	"github.com/matrix-org/gomatrixserverlib"
-
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/appservice/consumers"
 	"github.com/matrix-org/dendrite/appservice/query"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
@@ -37,39 +34,31 @@ import (
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *base.BaseDendrite,
+	processContext *process.ProcessContext,
+	cfg *config.Dendrite,
+	natsInstance *jetstream.NATSInstance,
 	userAPI userapi.AppserviceUserAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceInternalAPI {
-	client := &http.Client{
+
-		Timeout: time.Second * 30,
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation,
-			},
-			Proxy: http.ProxyFromEnvironment,
-		},
-	}
 	// Create appserivce query API with an HTTP client that will be used for all
 	// outbound and inbound requests (inbound only for the internal API)
 	appserviceQueryAPI := &query.AppServiceQueryAPI{
-		HTTPClient:    client,
+		Cfg:           &cfg.AppServiceAPI,
-		Cfg:           &base.Cfg.AppServiceAPI,
 		ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{},
 		CacheMu:       sync.Mutex{},
 	}
 
-	if len(base.Cfg.Derived.ApplicationServices) == 0 {
+	if len(cfg.Derived.ApplicationServices) == 0 {
 		return appserviceQueryAPI
 	}
 
 	// Wrap application services in a type that relates the application service and
 	// a sync.Cond object that can be used to notify workers when there are new
 	// events to be sent out.
-	for _, appservice := range base.Cfg.Derived.ApplicationServices {
+	for _, appservice := range cfg.Derived.ApplicationServices {
 		// Create bot account for this AS if it doesn't already exist
-		if err := generateAppServiceAccount(userAPI, appservice, base.Cfg.Global.ServerName); err != nil {
+		if err := generateAppServiceAccount(userAPI, appservice, cfg.Global.ServerName); err != nil {
 			logrus.WithFields(logrus.Fields{
 				"appservice": appservice.ID,
 			}).WithError(err).Panicf("failed to generate bot account for appservice")
@@ -78,10 +67,10 @@ func NewInternalAPI(
 
 	// Only consume if we actually have ASes to track, else we'll just chew cycles needlessly.
 	// We can't add ASes at runtime so this is safe to do.
-	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	js, _ := natsInstance.Prepare(processContext, &cfg.Global.JetStream)
 	consumer := consumers.NewOutputRoomEventConsumer(
-		base.ProcessContext, &base.Cfg.AppServiceAPI,
+		processContext, &cfg.AppServiceAPI,
-		client, js, rsAPI,
+		js, rsAPI,
 	)
 	if err := consumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
@@ -96,7 +85,7 @@ func NewInternalAPI(
 func generateAppServiceAccount(
 	userAPI userapi.AppserviceUserAPI,
 	as config.ApplicationService,
-	serverName gomatrixserverlib.ServerName,
+	serverName spec.ServerName,
 ) error {
 	var accRes userapi.PerformAccountCreationResponse
 	err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{

appservice/appservice_test.go

@@ -3,19 +3,31 @@ package appservice_test
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"net"
 	"net/http"
 	"net/http/httptest"
+	"path"
 	"reflect"
 	"regexp"
 	"strings"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
 
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/appservice/consumers"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/test/testrig"
 )
@@ -104,34 +116,138 @@ func TestAppserviceInternalAPI(t *testing.T) {
 	}
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, closeBase := testrig.CreateBaseDendrite(t, dbType)
+		cfg, ctx, close := testrig.CreateConfig(t, dbType)
-		defer closeBase()
+		defer close()
 
 		// Create a dummy application service
-		base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
+		as := &config.ApplicationService{
-			{
+			ID:              "someID",
-				ID:              "someID",
+			URL:             srv.URL,
-				URL:             srv.URL,
+			ASToken:         "",
-				ASToken:         "",
+			HSToken:         "",
-				HSToken:         "",
+			SenderLocalpart: "senderLocalPart",
-				SenderLocalpart: "senderLocalPart",
+			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
-				NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+				"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
-					"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
+				"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
-					"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
-				},
-				Protocols: []string{existingProtocol},
 			},
+			Protocols: []string{existingProtocol},
 		}
+		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
 
+		t.Cleanup(func() {
+			ctx.ShutdownDendrite()
+			ctx.WaitForShutdown()
+		})
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		// Create required internal APIs
-		rsAPI := roomserver.NewInternalAPI(base)
+		natsInstance := jetstream.NATSInstance{}
-		usrAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
-		asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
+		rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+		asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 		runCases(t, asAPI)
 	})
 }
 
+func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) {
+
+	// Set expected results
+	existingProtocol := "irc"
+	wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}}
+
+	// create a dummy AS url, handling some cases
+	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.Contains(r.URL.Path, "location"):
+			// Check if we've got an existing protocol, if so, return a proper response.
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "user"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "protocol"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode(nil); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		default:
+			t.Logf("hit location: %s", r.URL.Path)
+		}
+	}))
+
+	tmpDir := t.TempDir()
+	socket := path.Join(tmpDir, "socket")
+	l, err := net.Listen("unix", socket)
+	assert.NoError(t, err)
+	_ = srv.Listener.Close()
+	srv.Listener = l
+	srv.Start()
+	defer srv.Close()
+
+	cfg, ctx, tearDown := testrig.CreateConfig(t, test.DBTypeSQLite)
+	defer tearDown()
+
+	// Create a dummy application service
+	as := &config.ApplicationService{
+		ID:              "someID",
+		URL:             fmt.Sprintf("unix://%s", socket),
+		ASToken:         "",
+		HSToken:         "",
+		SenderLocalpart: "senderLocalPart",
+		NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+			"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
+			"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
+		},
+		Protocols: []string{existingProtocol},
+	}
+	as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
+	cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+	t.Cleanup(func() {
+		ctx.ShutdownDendrite()
+		ctx.WaitForShutdown()
+	})
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	// Create required internal APIs
+	natsInstance := jetstream.NATSInstance{}
+	cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
+	rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+	asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
+
+	t.Run("UserIDExists", func(t *testing.T) {
+		testUserIDExists(t, asAPI, "@as-testing:test", true)
+		testUserIDExists(t, asAPI, "@as1-testing:test", false)
+	})
+
+}
+
 func testUserIDExists(t *testing.T, asAPI api.AppServiceInternalAPI, userID string, wantExists bool) {
 	ctx := context.Background()
 	userResp := &api.UserIDExistsResponse{}
@@ -201,3 +317,87 @@ func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, w
 		t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
 	}
 }
+
+// Tests that the roomserver consumer only receives one invite
+func TestRoomserverConsumerOneInvite(t *testing.T) {
+
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	// Invite Bob
+	room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
+		"membership": "invite",
+	}, test.WithStateKey(bob.ID))
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		natsInstance := &jetstream.NATSInstance{}
+
+		evChan := make(chan struct{})
+		// create a dummy AS url, handling the events
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var txn consumers.ApplicationServiceTransaction
+			err := json.NewDecoder(r.Body).Decode(&txn)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, ev := range txn.Events {
+				if ev.Type != spec.MRoomMember {
+					continue
+				}
+				// Usually we would check the event content for the membership, but since
+				// we only invited bob, this should be fine for this test.
+				if ev.StateKey != nil && *ev.StateKey == bob.ID {
+					evChan <- struct{}{}
+				}
+			}
+		}))
+		defer srv.Close()
+
+		as := &config.ApplicationService{
+			ID:              "someID",
+			URL:             srv.URL,
+			ASToken:         "",
+			HSToken:         "",
+			SenderLocalpart: "senderLocalPart",
+			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+				"users":   {{RegexpObject: regexp.MustCompile(bob.ID)}},
+				"aliases": {{RegexpObject: regexp.MustCompile(room.ID)}},
+			},
+		}
+		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
+
+		// Create a dummy application service
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		// Create required internal APIs
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		// start the consumer
+		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
+
+		// Create the room
+		if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+		var seenInvitesForBob int
+	waitLoop:
+		for {
+			select {
+			case <-time.After(time.Millisecond * 50): // wait for the AS to process the events
+				break waitLoop
+			case <-evChan:
+				seenInvitesForBob++
+				if seenInvitesForBob != 1 {
+					t.Fatalf("received unexpected invites: %d", seenInvitesForBob)
+				}
+			}
+		}
+		close(evChan)
+	})
+}

appservice/consumers/roomserver.go

@@ -26,21 +26,28 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/syncapi/synctypes"
 
 	log "github.com/sirupsen/logrus"
 )
 
+// ApplicationServiceTransaction is the transaction that is sent off to an
+// application service.
+type ApplicationServiceTransaction struct {
+	Events []synctypes.ClientEvent `json:"events"`
+}
+
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
 	ctx       context.Context
 	cfg       *config.AppServiceAPI
-	client    *http.Client
 	jetstream nats.JetStreamContext
 	topic     string
 	rsAPI     api.AppserviceRoomserverAPI
@@ -56,14 +63,12 @@ type appserviceState struct {
 func NewOutputRoomEventConsumer(
 	process *process.ProcessContext,
 	cfg *config.AppServiceAPI,
-	client *http.Client,
 	js nats.JetStreamContext,
 	rsAPI api.AppserviceRoomserverAPI,
 ) *OutputRoomEventConsumer {
 	return &OutputRoomEventConsumer{
 		ctx:       process.Context(),
 		cfg:       cfg,
-		client:    client,
 		jetstream: js,
 		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
 		rsAPI:     rsAPI,
@@ -140,12 +145,6 @@ func (s *OutputRoomEventConsumer) onMessage(
 				}
 			}
 
-		case api.OutputTypeNewInviteEvent:
-			if output.NewInviteEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewInviteEvent.Event, state.ApplicationService) {
-				continue
-			}
-			events = append(events, output.NewInviteEvent.Event)
-
 		default:
 			continue
 		}
@@ -180,8 +179,8 @@ func (s *OutputRoomEventConsumer) sendEvents(
 ) error {
 	// Create the transaction body.
 	transaction, err := json.Marshal(
-		gomatrixserverlib.ApplicationServiceTransaction{
+		ApplicationServiceTransaction{
-			Events: gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll),
+			Events: synctypes.HeaderedToClientEvents(events, synctypes.FormatAll),
 		},
 	)
 	if err != nil {
@@ -195,13 +194,13 @@ func (s *OutputRoomEventConsumer) sendEvents(
 
 	// Send the transaction to the appservice.
 	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
-	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.URL, txnID, url.QueryEscape(state.HSToken))
+	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.RequestUrl(), txnID, url.QueryEscape(state.HSToken))
 	req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
 	if err != nil {
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	resp, err := s.client.Do(req)
+	resp, err := state.HTTPClient.Do(req)
 	if err != nil {
 		return state.backoffAndPause(err)
 	}
@@ -212,7 +211,7 @@ func (s *OutputRoomEventConsumer) sendEvents(
 	case http.StatusOK:
 		state.backoff = 0
 	default:
-		return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice", resp.StatusCode))
+		return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice url %s", resp.StatusCode, address))
 	}
 	return nil
 }
@@ -242,7 +241,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 		return true
 	}
 
-	if event.Type() == gomatrixserverlib.MRoomMember && event.StateKey() != nil {
+	if event.Type() == spec.MRoomMember && event.StateKey() != nil {
 		if appservice.IsInterestedInUserID(*event.StateKey()) {
 			return true
 		}
@@ -288,7 +287,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 			switch {
 			case ev.StateKey == nil:
 				continue
-			case ev.Type != gomatrixserverlib.MRoomMember:
+			case ev.Type != spec.MRoomMember:
 				continue
 			}
 			var membership gomatrixserverlib.MemberContent
@@ -296,7 +295,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 			switch {
 			case err != nil:
 				continue
-			case membership.Membership == gomatrixserverlib.Join:
+			case membership.Membership == spec.Join:
 				if appservice.IsInterestedInUserID(*ev.StateKey) {
 					return true
 				}

appservice/query/query.go

@@ -25,10 +25,10 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/opentracing/opentracing-go"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
@@ -37,7 +37,6 @@ const userIDExistsPath = "/users/"
 
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
-	HTTPClient    *http.Client
 	Cfg           *config.AppServiceAPI
 	ProtocolCache map[string]api.ASProtocolResponse
 	CacheMu       sync.Mutex
@@ -50,14 +49,14 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 	request *api.RoomAliasExistsRequest,
 	response *api.RoomAliasExistsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ApplicationServiceRoomAlias")
+	trace, ctx := internal.StartRegion(ctx, "ApplicationServiceRoomAlias")
-	defer span.Finish()
+	defer trace.EndRegion()
 
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.URL + roomAliasExistsPath)
+			URL, err := url.Parse(appservice.RequestUrl() + roomAliasExistsPath)
 			if err != nil {
 				return err
 			}
@@ -73,7 +72,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			}
 			req = req.WithContext(ctx)
 
-			resp, err := a.HTTPClient.Do(req)
+			resp, err := appservice.HTTPClient.Do(req)
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
@@ -117,14 +116,14 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	request *api.UserIDExistsRequest,
 	response *api.UserIDExistsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ApplicationServiceUserID")
+	trace, ctx := internal.StartRegion(ctx, "ApplicationServiceUserID")
-	defer span.Finish()
+	defer trace.EndRegion()
 
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.URL + userIDExistsPath)
+			URL, err := url.Parse(appservice.RequestUrl() + userIDExistsPath)
 			if err != nil {
 				return err
 			}
@@ -137,7 +136,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
-			resp, err := a.HTTPClient.Do(req.WithContext(ctx))
+			resp, err := appservice.HTTPClient.Do(req.WithContext(ctx))
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
@@ -212,12 +211,12 @@ func (a *AppServiceQueryAPI) Locations(
 		var asLocations []api.ASLocationResponse
 		params.Set("access_token", as.HSToken)
 
-		url := as.URL + api.ASLocationPath
+		url := as.RequestUrl() + api.ASLocationPath
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
+		if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
 			log.WithError(err).Error("unable to get 'locations' from application service")
 			continue
 		}
@@ -247,12 +246,12 @@ func (a *AppServiceQueryAPI) User(
 		var asUsers []api.ASUserResponse
 		params.Set("access_token", as.HSToken)
 
-		url := as.URL + api.ASUserPath
+		url := as.RequestUrl() + api.ASUserPath
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
+		if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
 			log.WithError(err).Error("unable to get 'user' from application service")
 			continue
 		}
@@ -290,7 +289,7 @@ func (a *AppServiceQueryAPI) Protocols(
 		response := api.ASProtocolResponse{}
 		for _, as := range a.Cfg.Derived.ApplicationServices {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil {
 				log.WithError(err).Error("unable to get 'protocol' from application service")
 				continue
 			}
@@ -320,7 +319,7 @@ func (a *AppServiceQueryAPI) Protocols(
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		for _, p := range as.Protocols {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil {
 				log.WithError(err).Error("unable to get 'protocol' from application service")
 				continue
 			}

build/dendritejs-pinecone/main.go

@@ -29,11 +29,14 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/userapi"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -157,9 +160,8 @@ func startup() {
 	pManager.AddPeer("wss://pinecone.matrix.org/public")
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+	cfg.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: false})
 	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
-	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
 	cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
 	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
 	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
@@ -169,35 +171,37 @@ func startup() {
 	cfg.Global.TrustedIDServers = []string{}
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.PrivateKey = sk
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
 	cfg.ClientAPI.RegistrationDisabled = false
 	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 
 	if err := cfg.Derive(); err != nil {
 		logrus.Fatalf("Failed to derive values from config: %s", err)
 	}
-	base := base.NewBaseDendrite(cfg)
+	natsInstance := jetstream.NATSInstance{}
-	defer base.Close() // nolint: errcheck
+	processCtx := process.NewProcessContext()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 
-	rsAPI := roomserver.NewInternalAPI(base)
+	federation := conn.CreateFederationClient(cfg, pSessions)
-
-	federation := conn.CreateFederationClient(base, pSessions)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
 
 	asQuery := appservice.NewInternalAPI(
-		base, userAPI, rsAPI,
+		processCtx, cfg, &natsInstance, userAPI, rsAPI,
 	)
 	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, keyRing, true)
+	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
 	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
 
 	monolith := setup.Monolith{
-		Config:    base.Cfg,
+		Config:    cfg,
-		Client:    conn.CreateClient(base, pSessions),
+		Client:    conn.CreateClient(pSessions),
 		FedClient: federation,
 		KeyRing:   keyRing,
 
@@ -208,15 +212,15 @@ func startup() {
 		//ServerKeyAPI:        serverKeyAPI,
 		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation),
 	}
-	monolith.AddAllPublicRoutes(base)
+	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
 
 	p2pRouter := pSessions.Protocol("matrix").HTTP().Mux()
-	p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux)
+	p2pRouter.Handle(httputil.PublicFederationPathPrefix, routers.Federation)
-	p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux)
+	p2pRouter.Handle(httputil.PublicMediaPathPrefix, routers.Media)
 
 	// Expose the matrix APIs via fetch - for local traffic
 	go func() {

build/gobind-pinecone/monolith.go

@@ -30,8 +30,12 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/relay"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/process"
 	userapiAPI "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/pinecone/types"
 	"github.com/sirupsen/logrus"
 
@@ -137,9 +141,9 @@ func (m *DendriteMonolith) SetStaticPeer(uri string) {
 	}
 }
 
-func getServerKeyFromString(nodeID string) (gomatrixserverlib.ServerName, error) {
+func getServerKeyFromString(nodeID string) (spec.ServerName, error) {
-	var nodeKey gomatrixserverlib.ServerName
+	var nodeKey spec.ServerName
-	if userID, err := gomatrixserverlib.NewUserID(nodeID, false); err == nil {
+	if userID, err := spec.NewUserID(nodeID, false); err == nil {
 		hexKey, decodeErr := hex.DecodeString(string(userID.Domain()))
 		if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
 			return "", fmt.Errorf("UserID domain is not a valid ed25519 public key: %v", userID.Domain())
@@ -151,7 +155,7 @@ func getServerKeyFromString(nodeID string) (gomatrixserverlib.ServerName, error)
 		if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
 			return "", fmt.Errorf("Relay server uri is not a valid ed25519 public key: %v", nodeID)
 		} else {
-			nodeKey = gomatrixserverlib.ServerName(nodeID)
+			nodeKey = spec.ServerName(nodeID)
 		}
 	}
 
@@ -159,7 +163,7 @@ func getServerKeyFromString(nodeID string) (gomatrixserverlib.ServerName, error)
 }
 
 func (m *DendriteMonolith) SetRelayServers(nodeID string, uris string) {
-	relays := []gomatrixserverlib.ServerName{}
+	relays := []spec.ServerName{}
 	for _, uri := range strings.Split(uris, ",") {
 		uri = strings.TrimSpace(uri)
 		if len(uri) == 0 {
@@ -185,9 +189,9 @@ func (m *DendriteMonolith) SetRelayServers(nodeID string, uris string) {
 		m.p2pMonolith.RelayRetriever.SetRelayServers(relays)
 	} else {
 		relay.UpdateNodeRelayServers(
-			gomatrixserverlib.ServerName(nodeKey),
+			spec.ServerName(nodeKey),
 			relays,
-			m.p2pMonolith.BaseDendrite.Context(),
+			m.p2pMonolith.ProcessCtx.Context(),
 			m.p2pMonolith.GetFederationAPI(),
 		)
 	}
@@ -212,9 +216,9 @@ func (m *DendriteMonolith) GetRelayServers(nodeID string) string {
 			relaysString += string(relay)
 		}
 	} else {
-		request := api.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(nodeKey)}
+		request := api.P2PQueryRelayServersRequest{Server: spec.ServerName(nodeKey)}
 		response := api.P2PQueryRelayServersResponse{}
-		err := m.p2pMonolith.GetFederationAPI().P2PQueryRelayServers(m.p2pMonolith.BaseDendrite.Context(), &request, &response)
+		err := m.p2pMonolith.GetFederationAPI().P2PQueryRelayServers(m.p2pMonolith.ProcessCtx.Context(), &request, &response)
 		if err != nil {
 			logrus.Warnf("Failed obtaining list of this node's relay servers: %s", err.Error())
 			return ""
@@ -288,7 +292,7 @@ func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, err
 	pubkey := m.p2pMonolith.Router.PublicKey()
 	userID := userutil.MakeUserID(
 		localpart,
-		gomatrixserverlib.ServerName(hex.EncodeToString(pubkey[:])),
+		spec.ServerName(hex.EncodeToString(pubkey[:])),
 	)
 	userReq := &userapiAPI.PerformAccountCreationRequest{
 		AccountType: userapiAPI.AccountTypeUser,
@@ -339,17 +343,21 @@ func (m *DendriteMonolith) Start() {
 
 	prefix := hex.EncodeToString(pk)
 	cfg := monolith.GenerateDefaultConfig(sk, m.StorageDirectory, m.CacheDirectory, prefix)
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.InMemory = false
 	// NOTE : disabled for now since there is a 64 bit alignment panic on 32 bit systems
 	// This isn't actually fixed: https://github.com/blevesearch/zapx/pull/147
 	cfg.SyncAPI.Fulltext.Enabled = false
 
+	processCtx := process.NewProcessContext()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+
 	enableRelaying := false
 	enableMetrics := false
 	enableWebsockets := false
-	m.p2pMonolith.SetupDendrite(cfg, 65432, enableRelaying, enableMetrics, enableWebsockets)
+	m.p2pMonolith.SetupDendrite(processCtx, cfg, cm, routers, 65432, enableRelaying, enableMetrics, enableWebsockets)
 	m.p2pMonolith.StartMonolith()
 }
 

build/gobind-pinecone/monolith_test.go

@@ -18,7 +18,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 func TestMonolithStarts(t *testing.T) {
@@ -110,7 +110,7 @@ func TestParseServerKey(t *testing.T) {
 		name        string
 		serverKey   string
 		expectedErr bool
-		expectedKey gomatrixserverlib.ServerName
+		expectedKey spec.ServerName
 	}{
 		{
 			name:        "valid userid as key",

build/gobind-yggdrasil/monolith.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
@@ -19,15 +20,20 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 
 	_ "golang.org/x/mobile/bind"
@@ -129,7 +135,7 @@ func (m *DendriteMonolith) Start() {
 		Generate:       true,
 		SingleDatabase: true,
 	})
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory))
@@ -148,25 +154,71 @@ func (m *DendriteMonolith) Start() {
 		panic(err)
 	}
 
-	base := base.NewBaseDendrite(cfg)
+	configErrors := &config.ConfigErrors{}
-	base.ConfigureAdminEndpoints()
+	cfg.Verify(configErrors)
-	m.processContext = base.ProcessContext
+	if len(*configErrors) > 0 {
-	defer base.Close() // nolint: errcheck
+		for _, err := range *configErrors {
+			logrus.Errorf("Configuration error: %s", err)
+		}
+		logrus.Fatalf("Failed to start due to configuration errors")
+	}
 
-	federation := ygg.CreateFederationClient(base)
+	internal.SetupStdLogging()
+	internal.SetupHookLogging(cfg.Logging)
+	internal.SetupPprof()
+
+	logrus.Infof("Dendrite version %s", internal.VersionString())
+
+	if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled {
+		logrus.Warn("Open registration is enabled")
+	}
+
+	closer, err := cfg.SetupTracing()
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to start opentracing")
+	}
+	defer closer.Close()
+
+	if cfg.Global.Sentry.Enabled {
+		logrus.Info("Setting up Sentry for debugging...")
+		err = sentry.Init(sentry.ClientOptions{
+			Dsn:              cfg.Global.Sentry.DSN,
+			Environment:      cfg.Global.Sentry.Environment,
+			Debug:            true,
+			ServerName:       string(cfg.Global.ServerName),
+			Release:          "dendrite@" + internal.VersionString(),
+			AttachStacktrace: true,
+		})
+		if err != nil {
+			logrus.WithError(err).Panic("failed to start Sentry")
+		}
+	}
+	processCtx := process.NewProcessContext()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+	basepkg.ConfigureAdminEndpoints(processCtx, routers)
+	m.processContext = processCtx
+	defer func() {
+		processCtx.ShutdownDendrite()
+		processCtx.WaitForShutdown()
+	}() // nolint: errcheck
+
+	federation := ygg.CreateFederationClient(cfg)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	rsAPI := roomserver.NewInternalAPI(base)
+	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
+	natsInstance := jetstream.NATSInstance{}
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, keyRing, true,
+		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
-	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
 
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
@@ -174,8 +226,8 @@ func (m *DendriteMonolith) Start() {
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{
-		Config:    base.Cfg,
+		Config:    cfg,
-		Client:    ygg.CreateClient(base),
+		Client:    ygg.CreateClient(),
 		FedClient: federation,
 		KeyRing:   keyRing,
 
@@ -187,17 +239,17 @@ func (m *DendriteMonolith) Start() {
 			ygg, fsAPI, federation,
 		),
 	}
-	monolith.AddAllPublicRoutes(base)
+	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
 
 	httpRouter := mux.NewRouter()
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
-	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin)
-	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin)
 
 	yggRouter := mux.NewRouter()
-	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
+	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(routers.Federation)
-	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
 
 	// Build both ends of a HTTP multiplex.
 	m.httpServer = &http.Server{

build/scripts/Complement.Dockerfile

@@ -1,6 +1,6 @@
 #syntax=docker/dockerfile:1.2
 
-FROM golang:1.18-stretch as build
+FROM golang:1.20-bullseye as build
 RUN apt-get update && apt-get install -y sqlite3
 WORKDIR /build
 
@@ -17,7 +17,7 @@ RUN --mount=target=. \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
     CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
-    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
     cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite

build/scripts/ComplementPostgres.Dockerfile

@@ -1,19 +1,19 @@
 #syntax=docker/dockerfile:1.2
 
-FROM golang:1.18-stretch as build
+FROM golang:1.20-bullseye as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 
 # No password when connecting over localhost
-RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/9.6/main/pg_hba.conf && \
+RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/13/main/pg_hba.conf && \
     # Bump up max conns for moar concurrency
-    sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf
+    sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/13/main/postgresql.conf
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
     #!/bin/bash -eu \n\
     pg_lsclusters \n\
-    pg_ctlcluster 9.6 main start \n\
+    pg_ctlcluster 13 main start \n\
     \n\
     until pg_isready \n\
     do \n\
@@ -35,7 +35,7 @@ RUN --mount=target=. \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
     CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
-    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
     cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite

build/scripts/complement-cmd.sh

@@ -2,14 +2,15 @@
 
 # This script is intended to be used inside a docker container for Complement
 
+export GOCOVERDIR=/tmp/covdatafiles
+mkdir -p "${GOCOVERDIR}"
 if [[ "${COVER}" -eq 1 ]]; then
   echo "Running with coverage"
   exec /dendrite/dendrite-cover \
     --really-enable-open-registration \
     --tls-cert server.crt \
     --tls-key server.key \
-    --config dendrite.yaml \
+    --config dendrite.yaml
-    --test.coverprofile=complementcover.log
 else
   echo "Not running with coverage"
   exec /dendrite/dendrite \

clientapi/admin_test.go

@@ -4,15 +4,22 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"testing"
+	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/tidwall/gjson"
 
@@ -29,54 +36,30 @@ func TestAdminResetPassword(t *testing.T) {
 
 	ctx := context.Background()
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
-		defer baseClose()
+		defer close()
-
+		natsInstance := jetstream.NATSInstance{}
 		// add a vhost
-		base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
+		cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
+			SigningIdentity: fclient.SigningIdentity{ServerName: "vh1"},
 		})
 
-		rsAPI := roomserver.NewInternalAPI(base)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		// Needed for changing the password/login
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
-		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil)
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
 		// Create the users in the userapi and login
-		accessTokens := map[*test.User]string{
+		accessTokens := map[*test.User]userDevice{
-			aliceAdmin: "",
+			aliceAdmin: {},
-			bob:        "",
+			bob:        {},
-			vhUser:     "",
+			vhUser:     {},
-		}
-		for u := range accessTokens {
-			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
-			userRes := &uapi.PerformAccountCreationResponse{}
-			password := util.RandomString(8)
-			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
-				AccountType: u.AccountType,
-				Localpart:   localpart,
-				ServerName:  serverName,
-				Password:    password,
-			}, userRes); err != nil {
-				t.Errorf("failed to create account: %s", err)
-			}
-
-			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
-				"type": authtypes.LoginTypePassword,
-				"identifier": map[string]interface{}{
-					"type": "m.id.user",
-					"user": u.ID,
-				},
-				"password": password,
-			}))
-			rec := httptest.NewRecorder()
-			base.PublicClientAPIMux.ServeHTTP(rec, req)
-			if rec.Code != http.StatusOK {
-				t.Fatalf("failed to login: %s", rec.Body.String())
-			}
-			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
 		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
 
 		testCases := []struct {
 			name           string
@@ -120,11 +103,11 @@ func TestAdminResetPassword(t *testing.T) {
 				}
 
 				if tc.withHeader {
-					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser])
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken)
 				}
 
 				rec := httptest.NewRecorder()
-				base.DendriteAdminMux.ServeHTTP(rec, req)
+				routers.DendriteAdmin.ServeHTTP(rec, req)
 				t.Logf("%s", rec.Body.String())
 				if tc.wantOK && rec.Code != http.StatusOK {
 					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
@@ -140,23 +123,26 @@ func TestPurgeRoom(t *testing.T) {
 	room := test.NewRoom(t, aliceAdmin, test.RoomPreset(test.PresetTrustedPrivateChat))
 
 	// Invite Bob
-	room.CreateAndInsert(t, aliceAdmin, gomatrixserverlib.MRoomMember, map[string]interface{}{
+	room.CreateAndInsert(t, aliceAdmin, spec.MRoomMember, map[string]interface{}{
 		"membership": "invite",
 	}, test.WithStateKey(bob.ID))
 
 	ctx := context.Background()
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
-		defer baseClose()
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
 
-		fedClient := base.CreateFederationClient()
+		routers := httputil.NewRouters()
-		rsAPI := roomserver.NewInternalAPI(base)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		// this starts the JetStream consumers
-		syncapi.AddPublicRoutes(base, userAPI, rsAPI)
+		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
-		federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true)
+		federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(nil, nil)
 
 		// Create the room
@@ -165,40 +151,13 @@ func TestPurgeRoom(t *testing.T) {
 		}
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
-		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil)
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
 		// Create the users in the userapi and login
-		accessTokens := map[*test.User]string{
+		accessTokens := map[*test.User]userDevice{
-			aliceAdmin: "",
+			aliceAdmin: {},
-		}
-		for u := range accessTokens {
-			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
-			userRes := &uapi.PerformAccountCreationResponse{}
-			password := util.RandomString(8)
-			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
-				AccountType: u.AccountType,
-				Localpart:   localpart,
-				ServerName:  serverName,
-				Password:    password,
-			}, userRes); err != nil {
-				t.Errorf("failed to create account: %s", err)
-			}
-
-			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
-				"type": authtypes.LoginTypePassword,
-				"identifier": map[string]interface{}{
-					"type": "m.id.user",
-					"user": u.ID,
-				},
-				"password": password,
-			}))
-			rec := httptest.NewRecorder()
-			base.PublicClientAPIMux.ServeHTTP(rec, req)
-			if rec.Code != http.StatusOK {
-				t.Fatalf("failed to login: %s", rec.Body.String())
-			}
-			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
 		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
 
 		testCases := []struct {
 			name   string
@@ -215,10 +174,10 @@ func TestPurgeRoom(t *testing.T) {
 			t.Run(tc.name, func(t *testing.T) {
 				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/purgeRoom/"+tc.roomID)
 
-				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin])
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
 
 				rec := httptest.NewRecorder()
-				base.DendriteAdminMux.ServeHTTP(rec, req)
+				routers.DendriteAdmin.ServeHTTP(rec, req)
 				t.Logf("%s", rec.Body.String())
 				if tc.wantOK && rec.Code != http.StatusOK {
 					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
@@ -228,3 +187,256 @@ func TestPurgeRoom(t *testing.T) {
 
 	})
 }
+
+func TestAdminEvacuateRoom(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, aliceAdmin)
+
+	// Join Bob
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// this starts the JetStream consumers
+		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
+		rsAPI.SetFederationAPI(fsAPI, nil)
+
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+
+		testCases := []struct {
+			name         string
+			roomID       string
+			wantOK       bool
+			wantAffected []string
+		}{
+			{name: "Can evacuate existing room", wantOK: true, roomID: room.ID, wantAffected: []string{aliceAdmin.ID, bob.ID}},
+			{name: "Can not evacuate non-existent room", wantOK: false, roomID: "!doesnotexist:localhost", wantAffected: []string{}},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateRoom/"+tc.roomID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
+
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+
+				affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array()
+				affected := make([]string, 0, len(affectedArr))
+				for _, x := range affectedArr {
+					affected = append(affected, x.Str)
+				}
+				if !reflect.DeepEqual(affected, tc.wantAffected) {
+					t.Fatalf("expected affected %#v, but got %#v", tc.wantAffected, affected)
+				}
+			})
+		}
+
+		// Wait for the FS API to have consumed every message
+		js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		timeout := time.After(time.Second)
+		for {
+			select {
+			case <-timeout:
+				t.Fatalf("FS API didn't process all events in time")
+			default:
+			}
+			info, err := js.ConsumerInfo(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), cfg.Global.JetStream.Durable("FederationAPIRoomServerConsumer")+"Pull")
+			if err != nil {
+				time.Sleep(time.Millisecond * 10)
+				continue
+			}
+			if info.NumPending == 0 && info.NumAckPending == 0 {
+				break
+			}
+		}
+	})
+}
+
+func TestAdminEvacuateUser(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, aliceAdmin)
+	room2 := test.NewRoom(t, aliceAdmin)
+
+	// Join Bob
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+	room2.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// this starts the JetStream consumers
+		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true)
+		rsAPI.SetFederationAPI(fsAPI, nil)
+
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room2.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+
+		testCases := []struct {
+			name              string
+			userID            string
+			wantOK            bool
+			wantAffectedRooms []string
+		}{
+			{name: "Can evacuate existing user", wantOK: true, userID: bob.ID, wantAffectedRooms: []string{room.ID, room2.ID}},
+			{name: "invalid userID is rejected", wantOK: false, userID: "!notauserid:test", wantAffectedRooms: []string{}},
+			{name: "Can not evacuate user from different server", wantOK: false, userID: "@doesnotexist:localhost", wantAffectedRooms: []string{}},
+			{name: "Can not evacuate non-existent user", wantOK: false, userID: "@doesnotexist:test", wantAffectedRooms: []string{}},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateUser/"+tc.userID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
+
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+
+				affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array()
+				affected := make([]string, 0, len(affectedArr))
+				for _, x := range affectedArr {
+					affected = append(affected, x.Str)
+				}
+				if !reflect.DeepEqual(affected, tc.wantAffectedRooms) {
+					t.Fatalf("expected affected %#v, but got %#v", tc.wantAffectedRooms, affected)
+				}
+
+			})
+		}
+		// Wait for the FS API to have consumed every message
+		js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		timeout := time.After(time.Second)
+		for {
+			select {
+			case <-timeout:
+				t.Fatalf("FS API didn't process all events in time")
+			default:
+			}
+			info, err := js.ConsumerInfo(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), cfg.Global.JetStream.Durable("FederationAPIRoomServerConsumer")+"Pull")
+			if err != nil {
+				time.Sleep(time.Millisecond * 10)
+				continue
+			}
+			if info.NumPending == 0 && info.NumAckPending == 0 {
+				break
+			}
+		}
+	})
+}
+
+func TestAdminMarkAsStale(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+
+		testCases := []struct {
+			name   string
+			userID string
+			wantOK bool
+		}{
+			{name: "local user is not allowed", userID: aliceAdmin.ID},
+			{name: "invalid userID", userID: "!notvalid:test"},
+			{name: "remote user is allowed", userID: "@alice:localhost", wantOK: true},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/refreshDevices/"+tc.userID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
+
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}

clientapi/api/api.go

@@ -14,10 +14,10 @@
 
 package api
 
-import "github.com/matrix-org/gomatrixserverlib"
+import "github.com/matrix-org/gomatrixserverlib/fclient"
 
 // ExtraPublicRoomsProvider provides a way to inject extra published rooms into /publicRooms requests.
 type ExtraPublicRoomsProvider interface {
 	// Rooms returns the extra rooms. This is called on-demand by clients, so cache appropriately.
-	Rooms() []gomatrixserverlib.PublicRoom
+	Rooms() []fclient.PublicRoom
 }

clientapi/auth/authtypes/threepid.go

@@ -16,6 +16,8 @@ package authtypes
 
 // ThreePID represents a third-party identifier
 type ThreePID struct {
-	Address string `json:"address"`
+	Address     string `json:"address"`
-	Medium  string `json:"medium"`
+	Medium      string `json:"medium"`
+	AddedAt     int64  `json:"added_at"`
+	ValidatedAt int64  `json:"validated_at"`
 }

clientapi/auth/login_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -68,7 +68,7 @@ func TestLoginFromJSONReader(t *testing.T) {
 			var userAPI fakeUserInternalAPI
 			cfg := &config.ClientAPI{
 				Matrix: &config.Global{
-					SigningIdentity: gomatrixserverlib.SigningIdentity{
+					SigningIdentity: fclient.SigningIdentity{
 						ServerName: serverName,
 					},
 				},
@@ -148,7 +148,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 			var userAPI fakeUserInternalAPI
 			cfg := &config.ClientAPI{
 				Matrix: &config.Global{
-					SigningIdentity: gomatrixserverlib.SigningIdentity{
+					SigningIdentity: fclient.SigningIdentity{
 						ServerName: serverName,
 					},
 				},

clientapi/auth/user_interactive_test.go

@@ -8,13 +8,14 @@ import (
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
 var (
 	ctx        = context.Background()
-	serverName = gomatrixserverlib.ServerName("example.com")
+	serverName = spec.ServerName("example.com")
 	// space separated localpart+password -> account
 	lookup = make(map[string]*api.Account)
 	device = &api.Device{
@@ -47,7 +48,7 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a
 func setup() *UserInteractive {
 	cfg := &config.ClientAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: serverName,
 			},
 		},

clientapi/clientapi.go

@@ -15,8 +15,11 @@
 package clientapi
 
 import (
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/api"
@@ -25,41 +28,41 @@ import (
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 )
 
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
 func AddPublicRoutes(
-	base *base.BaseDendrite,
+	processContext *process.ProcessContext,
-	federation *gomatrixserverlib.FederationClient,
+	routers httputil.Routers,
+	cfg *config.Dendrite,
+	natsInstance *jetstream.NATSInstance,
+	federation *fclient.FederationClient,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 	transactionsCache *transactions.Cache,
 	fsAPI federationAPI.ClientFederationAPI,
 	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.QuerySearchProfilesAPI,
-	extRoomsProvider api.ExtraPublicRoomsProvider,
+	extRoomsProvider api.ExtraPublicRoomsProvider, enableMetrics bool,
 ) {
-	cfg := &base.Cfg.ClientAPI
+	js, natsClient := natsInstance.Prepare(processContext, &cfg.Global.JetStream)
-	mscCfg := &base.Cfg.MSCs
-	js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	syncProducer := &producers.SyncAPIProducer{
 		JetStream:              js,
-		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+		TopicReceiptEvent:      cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent),
-		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		TopicSendToDeviceEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
-		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicTypingEvent:       cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent),
-		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		TopicPresenceEvent:     cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent),
 		UserAPI:                userAPI,
-		ServerName:             cfg.Matrix.ServerName,
+		ServerName:             cfg.Global.ServerName,
 	}
 
 	routing.Setup(
-		base,
+		routers,
 		cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
 		syncProducer, transactionsCache, fsAPI,
-		extRoomsProvider, mscCfg, natsClient,
+		extRoomsProvider, natsClient, enableMetrics,
 	)
 }

clientapi/producers/syncapi.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 
@@ -37,13 +38,13 @@ type SyncAPIProducer struct {
 	TopicTypingEvent       string
 	TopicPresenceEvent     string
 	JetStream              nats.JetStreamContext
-	ServerName             gomatrixserverlib.ServerName
+	ServerName             spec.ServerName
 	UserAPI                userapi.ClientUserAPI
 }
 
 func (p *SyncAPIProducer) SendReceipt(
 	ctx context.Context,
-	userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp,
+	userID, roomID, eventID, receiptType string, timestamp spec.Timestamp,
 ) error {
 	m := &nats.Msg{
 		Subject: p.TopicReceiptEvent,
@@ -154,7 +155,7 @@ func (p *SyncAPIProducer) SendPresence(
 		m.Header.Set("status_msg", *statusMsg)
 	}
 
-	m.Header.Set("last_active_ts", strconv.Itoa(int(gomatrixserverlib.AsTimestamp(time.Now()))))
+	m.Header.Set("last_active_ts", strconv.Itoa(int(spec.AsTimestamp(time.Now()))))
 
 	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
 	return err

clientapi/routing/admin.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	"github.com/sirupsen/logrus"
@@ -22,23 +23,16 @@ import (
 	"github.com/matrix-org/dendrite/userapi/api"
 )
 
-func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	roomID, ok := vars["roomID"]
-	if !ok {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting room ID."),
-		}
-	}
 	res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
 	if err := rsAPI.PerformAdminEvacuateRoom(
 		req.Context(),
 		&roomserverAPI.PerformAdminEvacuateRoomRequest{
-			RoomID: roomID,
+			RoomID: vars["roomID"],
 		},
 		res,
 	); err != nil {
@@ -55,18 +49,13 @@ func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Dev
 	}
 }
 
-func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	userID, ok := vars["userID"]
+	userID := vars["userID"]
-	if !ok {
+
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting user ID."),
-		}
-	}
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		return util.MessageResponse(http.StatusBadRequest, err.Error())
@@ -103,13 +92,8 @@ func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	roomID, ok := vars["roomID"]
+	roomID := vars["roomID"]
-	if !ok {
+
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting room ID."),
-		}
-	}
 	res := &roomserverAPI.PerformAdminPurgeRoomResponse{}
 	if err := rsAPI.PerformAdminPurgeRoom(
 		context.Background(),
@@ -279,7 +263,7 @@ func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *api.De
 		&roomserverAPI.PerformAdminDownloadStateRequest{
 			UserID:     device.UserID,
 			RoomID:     roomID,
-			ServerName: gomatrixserverlib.ServerName(serverName),
+			ServerName: spec.ServerName(serverName),
 		},
 		res,
 	); err != nil {

clientapi/routing/aliases.go

@@ -22,6 +22,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/util"
 )
@@ -31,7 +32,7 @@ func GetAliases(
 	req *http.Request, rsAPI api.ClientRoomserverAPI, device *userapi.Device, roomID string,
 ) util.JSONResponse {
 	stateTuple := gomatrixserverlib.StateKeyTuple{
-		EventType: gomatrixserverlib.MRoomHistoryVisibility,
+		EventType: spec.MRoomHistoryVisibility,
 		StateKey:  "",
 	}
 	stateReq := &api.QueryCurrentStateRequest{
@@ -53,7 +54,7 @@ func GetAliases(
 			return util.ErrorResponse(fmt.Errorf("historyVisEvent.HistoryVisibility: %w", err))
 		}
 	}
-	if visibility != gomatrixserverlib.WorldReadable {
+	if visibility != spec.WorldReadable {
 		queryReq := api.QueryMembershipForUserRequest{
 			RoomID: roomID,
 			UserID: device.UserID,

clientapi/routing/auth_fallback_test.go

@@ -10,30 +10,28 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/test/testrig"
 )
 
 func Test_AuthFallback(t *testing.T) {
-	base, _, _ := testrig.Base(nil)
+	cfg := config.Dendrite{}
-	defer base.Close()
+	cfg.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
-
 	for _, useHCaptcha := range []bool{false, true} {
 		for _, recaptchaEnabled := range []bool{false, true} {
 			for _, wantErr := range []bool{false, true} {
 				t.Run(fmt.Sprintf("useHCaptcha(%v) - recaptchaEnabled(%v) - wantErr(%v)", useHCaptcha, recaptchaEnabled, wantErr), func(t *testing.T) {
 					// Set the defaults for each test
-					base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
+					cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
-					base.Cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled
+					cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled
-					base.Cfg.ClientAPI.RecaptchaPublicKey = "pub"
+					cfg.ClientAPI.RecaptchaPublicKey = "pub"
-					base.Cfg.ClientAPI.RecaptchaPrivateKey = "priv"
+					cfg.ClientAPI.RecaptchaPrivateKey = "priv"
 					if useHCaptcha {
-						base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify"
+						cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify"
-						base.Cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js"
+						cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js"
-						base.Cfg.ClientAPI.RecaptchaFormField = "h-captcha-response"
+						cfg.ClientAPI.RecaptchaFormField = "h-captcha-response"
-						base.Cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha"
+						cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha"
 					}
 					cfgErrs := &config.ConfigErrors{}
-					base.Cfg.ClientAPI.Verify(cfgErrs)
+					cfg.ClientAPI.Verify(cfgErrs)
 					if len(*cfgErrs) > 0 {
 						t.Fatalf("(hCaptcha=%v) unexpected config errors: %s", useHCaptcha, cfgErrs.Error())
 					}
@@ -41,7 +39,7 @@ func Test_AuthFallback(t *testing.T) {
 					req := httptest.NewRequest(http.MethodGet, "/?session=1337", nil)
 					rec := httptest.NewRecorder()
 
-					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 					if !recaptchaEnabled {
 						if rec.Code != http.StatusBadRequest {
 							t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
@@ -50,8 +48,8 @@ func Test_AuthFallback(t *testing.T) {
 							t.Fatalf("unexpected response body: %s", rec.Body.String())
 						}
 					} else {
-						if !strings.Contains(rec.Body.String(), base.Cfg.ClientAPI.RecaptchaSitekeyClass) {
+						if !strings.Contains(rec.Body.String(), cfg.ClientAPI.RecaptchaSitekeyClass) {
-							t.Fatalf("body does not contain %s: %s", base.Cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String())
+							t.Fatalf("body does not contain %s: %s", cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String())
 						}
 					}
 
@@ -64,14 +62,14 @@ func Test_AuthFallback(t *testing.T) {
 					}))
 					defer srv.Close() // nolint: errcheck
 
-					base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
+					cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
 
 					// check the result after sending the captcha
 					req = httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
 					req.Form = url.Values{}
-					req.Form.Add(base.Cfg.ClientAPI.RecaptchaFormField, "someRandomValue")
+					req.Form.Add(cfg.ClientAPI.RecaptchaFormField, "someRandomValue")
 					rec = httptest.NewRecorder()
-					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 					if recaptchaEnabled {
 						if !wantErr {
 							if rec.Code != http.StatusOK {
@@ -105,7 +103,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("unknown fallbacks are handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, "DoesNotExist", &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, "DoesNotExist", &cfg.ClientAPI)
 		if rec.Code != http.StatusNotImplemented {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusNotImplemented)
 		}
@@ -114,7 +112,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("unknown methods are handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodDelete, "/?session=1337", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 		if rec.Code != http.StatusMethodNotAllowed {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusMethodNotAllowed)
 		}
@@ -123,7 +121,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodGet, "/", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 		if rec.Code != http.StatusBadRequest {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
 		}
@@ -132,7 +130,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodGet, "/", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 		if rec.Code != http.StatusBadRequest {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
 		}
@@ -141,7 +139,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("missing 'response' is handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 		if rec.Code != http.StatusBadRequest {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
 		}

clientapi/routing/capabilities.go

@@ -17,26 +17,21 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/roomserver/version"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
-
 	"github.com/matrix-org/util"
 )
 
 // GetCapabilities returns information about the server's supported feature set
 // and other relevant capabilities to an authenticated user.
-func GetCapabilities(
+func GetCapabilities() util.JSONResponse {
-	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
+	versionsMap := map[gomatrixserverlib.RoomVersion]string{}
-) util.JSONResponse {
+	for v, desc := range version.SupportedRoomVersions() {
-	roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{}
+		if desc.Stable {
-	roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{}
+			versionsMap[v] = "stable"
-	if err := rsAPI.QueryRoomVersionCapabilities(
+		} else {
-		req.Context(),
+			versionsMap[v] = "unstable"
-		&roomVersionsQueryReq,
+		}
-		&roomVersionsQueryRes,
-	); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryRoomVersionCapabilities failed")
-		return jsonerror.InternalServerError()
 	}
 
 	response := map[string]interface{}{
@@ -44,7 +39,10 @@ func GetCapabilities(
 			"m.change_password": map[string]bool{
 				"enabled": true,
 			},
-			"m.room_versions": roomVersionsQueryRes,
+			"m.room_versions": map[string]interface{}{
+				"default":   version.DefaultRoomVersion(),
+				"available": versionsMap,
+			},
 		},
 	}
 

clientapi/routing/createroom.go

@@ -26,6 +26,8 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
@@ -233,7 +235,7 @@ func createRoom(
 	createContent["room_version"] = roomVersion
 	powerLevelContent := eventutil.InitialPowerLevelsContent(userID)
 	joinRuleContent := gomatrixserverlib.JoinRuleContent{
-		JoinRule: gomatrixserverlib.Invite,
+		JoinRule: spec.Invite,
 	}
 	historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{
 		HistoryVisibility: historyVisibilityShared,
@@ -253,40 +255,40 @@ func createRoom(
 
 	switch r.Preset {
 	case presetPrivateChat:
-		joinRuleContent.JoinRule = gomatrixserverlib.Invite
+		joinRuleContent.JoinRule = spec.Invite
 		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
 	case presetTrustedPrivateChat:
-		joinRuleContent.JoinRule = gomatrixserverlib.Invite
+		joinRuleContent.JoinRule = spec.Invite
 		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
 		for _, invitee := range r.Invite {
 			powerLevelContent.Users[invitee] = 100
 		}
 	case presetPublicChat:
-		joinRuleContent.JoinRule = gomatrixserverlib.Public
+		joinRuleContent.JoinRule = spec.Public
 		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
 	}
 
 	createEvent := fledglingEvent{
-		Type:    gomatrixserverlib.MRoomCreate,
+		Type:    spec.MRoomCreate,
 		Content: createContent,
 	}
 	powerLevelEvent := fledglingEvent{
-		Type:    gomatrixserverlib.MRoomPowerLevels,
+		Type:    spec.MRoomPowerLevels,
 		Content: powerLevelContent,
 	}
 	joinRuleEvent := fledglingEvent{
-		Type:    gomatrixserverlib.MRoomJoinRules,
+		Type:    spec.MRoomJoinRules,
 		Content: joinRuleContent,
 	}
 	historyVisibilityEvent := fledglingEvent{
-		Type:    gomatrixserverlib.MRoomHistoryVisibility,
+		Type:    spec.MRoomHistoryVisibility,
 		Content: historyVisibilityContent,
 	}
 	membershipEvent := fledglingEvent{
-		Type:     gomatrixserverlib.MRoomMember,
+		Type:     spec.MRoomMember,
 		StateKey: userID,
 		Content: gomatrixserverlib.MemberContent{
-			Membership:  gomatrixserverlib.Join,
+			Membership:  spec.Join,
 			DisplayName: profile.DisplayName,
 			AvatarURL:   profile.AvatarURL,
 		},
@@ -299,7 +301,7 @@ func createRoom(
 
 	if r.Name != "" {
 		nameEvent = &fledglingEvent{
-			Type: gomatrixserverlib.MRoomName,
+			Type: spec.MRoomName,
 			Content: eventutil.NameContent{
 				Name: r.Name,
 			},
@@ -308,7 +310,7 @@ func createRoom(
 
 	if r.Topic != "" {
 		topicEvent = &fledglingEvent{
-			Type: gomatrixserverlib.MRoomTopic,
+			Type: spec.MRoomTopic,
 			Content: eventutil.TopicContent{
 				Topic: r.Topic,
 			},
@@ -317,7 +319,7 @@ func createRoom(
 
 	if r.GuestCanJoin {
 		guestAccessEvent = &fledglingEvent{
-			Type: gomatrixserverlib.MRoomGuestAccess,
+			Type: spec.MRoomGuestAccess,
 			Content: eventutil.GuestAccessContent{
 				GuestAccess: "can_join",
 			},
@@ -347,7 +349,7 @@ func createRoom(
 		}
 
 		aliasEvent = &fledglingEvent{
-			Type: gomatrixserverlib.MRoomCanonicalAlias,
+			Type: spec.MRoomCanonicalAlias,
 			Content: eventutil.CanonicalAlias{
 				Alias: roomAlias,
 			},
@@ -362,25 +364,25 @@ func createRoom(
 		}
 
 		switch r.InitialState[i].Type {
-		case gomatrixserverlib.MRoomCreate:
+		case spec.MRoomCreate:
 			continue
 
-		case gomatrixserverlib.MRoomPowerLevels:
+		case spec.MRoomPowerLevels:
 			powerLevelEvent = r.InitialState[i]
 
-		case gomatrixserverlib.MRoomJoinRules:
+		case spec.MRoomJoinRules:
 			joinRuleEvent = r.InitialState[i]
 
-		case gomatrixserverlib.MRoomHistoryVisibility:
+		case spec.MRoomHistoryVisibility:
 			historyVisibilityEvent = r.InitialState[i]
 
-		case gomatrixserverlib.MRoomGuestAccess:
+		case spec.MRoomGuestAccess:
 			guestAccessEvent = &r.InitialState[i]
 
-		case gomatrixserverlib.MRoomName:
+		case spec.MRoomName:
 			nameEvent = &r.InitialState[i]
 
-		case gomatrixserverlib.MRoomTopic:
+		case spec.MRoomTopic:
 			topicEvent = &r.InitialState[i]
 
 		default:
@@ -448,7 +450,7 @@ func createRoom(
 			builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
 		}
 		var ev *gomatrixserverlib.Event
-		ev, err = buildEvent(&builder, userDomain, &authEvents, cfg, evTime, roomVersion)
+		ev, err = builder.AddAuthEventsAndBuild(userDomain, &authEvents, evTime, roomVersion, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey)
 		if err != nil {
 			util.GetLogger(ctx).WithError(err).Error("buildEvent failed")
 			return jsonerror.InternalServerError()
@@ -510,30 +512,30 @@ func createRoom(
 	// If this is a direct message then we should invite the participants.
 	if len(r.Invite) > 0 {
 		// Build some stripped state for the invite.
-		var globalStrippedState []gomatrixserverlib.InviteV2StrippedState
+		var globalStrippedState []fclient.InviteV2StrippedState
 		for _, event := range builtEvents {
 			// Chosen events from the spec:
 			// https://spec.matrix.org/v1.3/client-server-api/#stripped-state
 			switch event.Type() {
-			case gomatrixserverlib.MRoomCreate:
+			case spec.MRoomCreate:
 				fallthrough
-			case gomatrixserverlib.MRoomName:
+			case spec.MRoomName:
 				fallthrough
-			case gomatrixserverlib.MRoomAvatar:
+			case spec.MRoomAvatar:
 				fallthrough
-			case gomatrixserverlib.MRoomTopic:
+			case spec.MRoomTopic:
 				fallthrough
-			case gomatrixserverlib.MRoomCanonicalAlias:
+			case spec.MRoomCanonicalAlias:
 				fallthrough
-			case gomatrixserverlib.MRoomEncryption:
+			case spec.MRoomEncryption:
 				fallthrough
-			case gomatrixserverlib.MRoomMember:
+			case spec.MRoomMember:
 				fallthrough
-			case gomatrixserverlib.MRoomJoinRules:
+			case spec.MRoomJoinRules:
 				ev := event.Event
 				globalStrippedState = append(
 					globalStrippedState,
-					gomatrixserverlib.NewInviteV2StrippedState(ev),
+					fclient.NewInviteV2StrippedState(ev),
 				)
 			}
 		}
@@ -542,7 +544,7 @@ func createRoom(
 		for _, invitee := range r.Invite {
 			// Build the invite event.
 			inviteEvent, err := buildMembershipEvent(
-				ctx, invitee, "", profileAPI, device, gomatrixserverlib.Invite,
+				ctx, invitee, "", profileAPI, device, spec.Invite,
 				roomID, r.IsDirect, cfg, evTime, rsAPI, asAPI,
 			)
 			if err != nil {
@@ -551,7 +553,7 @@ func createRoom(
 			}
 			inviteStrippedState := append(
 				globalStrippedState,
-				gomatrixserverlib.NewInviteV2StrippedState(inviteEvent.Event),
+				fclient.NewInviteV2StrippedState(inviteEvent.Event),
 			)
 			// Send the invite event to the roomserver.
 			var inviteRes roomserverAPI.PerformInviteResponse
@@ -599,31 +601,3 @@ func createRoom(
 		JSON: response,
 	}
 }
-
-// buildEvent fills out auth_events for the builder then builds the event
-func buildEvent(
-	builder *gomatrixserverlib.EventBuilder,
-	serverName gomatrixserverlib.ServerName,
-	provider gomatrixserverlib.AuthEventProvider,
-	cfg *config.ClientAPI,
-	evTime time.Time,
-	roomVersion gomatrixserverlib.RoomVersion,
-) (*gomatrixserverlib.Event, error) {
-	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
-	if err != nil {
-		return nil, err
-	}
-	refs, err := eventsNeeded.AuthEventReferences(provider)
-	if err != nil {
-		return nil, err
-	}
-	builder.AuthEvents = refs
-	event, err := builder.Build(
-		evTime, serverName, cfg.Matrix.KeyID,
-		cfg.Matrix.PrivateKey, roomVersion,
-	)
-	if err != nil {
-		return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err)
-	}
-	return event, nil
-}

clientapi/routing/deactivate.go

@@ -33,7 +33,7 @@ func Deactivate(
 		return *errRes
 	}
 
-	localpart, _, err := gomatrixserverlib.SplitID('@', login.Username())
+	localpart, serverName, err := gomatrixserverlib.SplitID('@', login.Username())
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
 		return jsonerror.InternalServerError()
@@ -41,7 +41,8 @@ func Deactivate(
 
 	var res api.PerformAccountDeactivationResponse
 	err = accountAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{
-		Localpart: localpart,
+		Localpart:  localpart,
+		ServerName: serverName,
 	}, &res)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformAccountDeactivation failed")

clientapi/routing/device.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"encoding/json"
 	"io"
 	"net"
 	"net/http"
@@ -146,12 +147,6 @@ func UpdateDeviceByID(
 			JSON: jsonerror.Forbidden("device does not exist"),
 		}
 	}
-	if performRes.Forbidden {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("device not owned by current user"),
-		}
-	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -189,7 +184,7 @@ func DeleteDeviceById(
 		if dev != deviceID {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("session & device mismatch"),
+				JSON: jsonerror.Forbidden("session and device mismatch"),
 			}
 		}
 	}
@@ -242,16 +237,37 @@ func DeleteDeviceById(
 
 // DeleteDevices handles POST requests to /delete_devices
 func DeleteDevices(
-	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
+	req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.ClientUserAPI, device *api.Device,
 ) util.JSONResponse {
 	ctx := req.Context()
-	payload := devicesDeleteJSON{}
 
-	if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil {
+	bodyBytes, err := io.ReadAll(req.Body)
-		return *resErr
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
+		}
+	}
+	defer req.Body.Close() // nolint:errcheck
+
+	// initiate UIA
+	login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, device)
+	if errRes != nil {
+		return *errRes
 	}
 
-	defer req.Body.Close() // nolint: errcheck
+	if login.Username() != device.UserID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("unable to delete devices for other user"),
+		}
+	}
+
+	payload := devicesDeleteJSON{}
+	if err = json.Unmarshal(bodyBytes, &payload); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("unable to unmarshal device deletion request")
+		return jsonerror.InternalServerError()
+	}
 
 	var res api.PerformDeviceDeletionResponse
 	if err := userAPI.PerformDeviceDeletion(ctx, &api.PerformDeviceDeletionRequest{

clientapi/routing/directory.go

@@ -19,6 +19,8 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -34,7 +36,7 @@ type roomDirectoryResponse struct {
 	Servers []string `json:"servers"`
 }
 
-func (r *roomDirectoryResponse) fillServers(servers []gomatrixserverlib.ServerName) {
+func (r *roomDirectoryResponse) fillServers(servers []spec.ServerName) {
 	r.Servers = make([]string, len(servers))
 	for i, s := range servers {
 		r.Servers[i] = string(s)
@@ -45,7 +47,7 @@ func (r *roomDirectoryResponse) fillServers(servers []gomatrixserverlib.ServerNa
 func DirectoryRoom(
 	req *http.Request,
 	roomAlias string,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	fedSenderAPI federationAPI.ClientFederationAPI,
@@ -252,7 +254,7 @@ func GetVisibility(
 
 	var v roomVisibility
 	if len(res.RoomIDs) == 1 {
-		v.Visibility = gomatrixserverlib.Public
+		v.Visibility = spec.Public
 	} else {
 		v.Visibility = "private"
 	}
@@ -277,7 +279,7 @@ func SetVisibility(
 	queryEventsReq := roomserverAPI.QueryLatestEventsAndStateRequest{
 		RoomID: roomID,
 		StateToFetch: []gomatrixserverlib.StateKeyTuple{{
-			EventType: gomatrixserverlib.MRoomPowerLevels,
+			EventType: spec.MRoomPowerLevels,
 			StateKey:  "",
 		}},
 	}
@@ -290,7 +292,7 @@ func SetVisibility(
 
 	// NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event
 	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
-	if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomCanonicalAlias, true) {
+	if power.UserLevel(dev.UserID) < power.EventLevel(spec.MRoomCanonicalAlias, true) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),

clientapi/routing/directory_public.go

@@ -23,7 +23,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/api"
@@ -35,7 +36,7 @@ import (
 
 var (
 	cacheMu          sync.Mutex
-	publicRoomsCache []gomatrixserverlib.PublicRoom
+	publicRoomsCache []fclient.PublicRoom
 )
 
 type PublicRoomReq struct {
@@ -56,7 +57,7 @@ type filter struct {
 func GetPostPublicRooms(
 	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var request PublicRoomReq
@@ -71,7 +72,7 @@ func GetPostPublicRooms(
 		}
 	}
 
-	serverName := gomatrixserverlib.ServerName(request.Server)
+	serverName := spec.ServerName(request.Server)
 	if serverName != "" && !cfg.Matrix.IsLocalServerName(serverName) {
 		res, err := federation.GetPublicRoomsFiltered(
 			req.Context(), cfg.Matrix.ServerName, serverName,
@@ -102,10 +103,10 @@ func GetPostPublicRooms(
 
 func publicRooms(
 	ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
-) (*gomatrixserverlib.RespPublicRooms, error) {
+) (*fclient.RespPublicRooms, error) {
 
-	response := gomatrixserverlib.RespPublicRooms{
+	response := fclient.RespPublicRooms{
-		Chunk: []gomatrixserverlib.PublicRoom{},
+		Chunk: []fclient.PublicRoom{},
 	}
 	var limit int64
 	var offset int64
@@ -122,7 +123,7 @@ func publicRooms(
 	}
 	err = nil
 
-	var rooms []gomatrixserverlib.PublicRoom
+	var rooms []fclient.PublicRoom
 	if request.Since == "" {
 		rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider, request)
 	} else {
@@ -146,14 +147,14 @@ func publicRooms(
 	return &response, err
 }
 
-func filterRooms(rooms []gomatrixserverlib.PublicRoom, searchTerm string) []gomatrixserverlib.PublicRoom {
+func filterRooms(rooms []fclient.PublicRoom, searchTerm string) []fclient.PublicRoom {
 	if searchTerm == "" {
 		return rooms
 	}
 
 	normalizedTerm := strings.ToLower(searchTerm)
 
-	result := make([]gomatrixserverlib.PublicRoom, 0)
+	result := make([]fclient.PublicRoom, 0)
 	for _, room := range rooms {
 		if strings.Contains(strings.ToLower(room.Name), normalizedTerm) ||
 			strings.Contains(strings.ToLower(room.Topic), normalizedTerm) ||
@@ -214,7 +215,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 //	 limit=3&since=6  => G     (prev='3', next='')
 //
 //	A value of '-1' for prev/next indicates no position.
-func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
+func sliceInto(slice []fclient.PublicRoom, since int64, limit int64) (subset []fclient.PublicRoom, prev, next int) {
 	prev = -1
 	next = -1
 
@@ -241,10 +242,10 @@ func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) (
 func refreshPublicRoomCache(
 	ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
 	request PublicRoomReq,
-) []gomatrixserverlib.PublicRoom {
+) []fclient.PublicRoom {
 	cacheMu.Lock()
 	defer cacheMu.Unlock()
-	var extraRooms []gomatrixserverlib.PublicRoom
+	var extraRooms []fclient.PublicRoom
 	if extRoomsProvider != nil {
 		extraRooms = extRoomsProvider.Rooms()
 	}
@@ -269,7 +270,7 @@ func refreshPublicRoomCache(
 		util.GetLogger(ctx).WithError(err).Error("PopulatePublicRooms failed")
 		return publicRoomsCache
 	}
-	publicRoomsCache = []gomatrixserverlib.PublicRoom{}
+	publicRoomsCache = []fclient.PublicRoom{}
 	publicRoomsCache = append(publicRoomsCache, pubRooms...)
 	publicRoomsCache = append(publicRoomsCache, extraRooms...)
 	publicRoomsCache = dedupeAndShuffle(publicRoomsCache)
@@ -281,16 +282,16 @@ func refreshPublicRoomCache(
 	return publicRoomsCache
 }
 
-func getPublicRoomsFromCache() []gomatrixserverlib.PublicRoom {
+func getPublicRoomsFromCache() []fclient.PublicRoom {
 	cacheMu.Lock()
 	defer cacheMu.Unlock()
 	return publicRoomsCache
 }
 
-func dedupeAndShuffle(in []gomatrixserverlib.PublicRoom) []gomatrixserverlib.PublicRoom {
+func dedupeAndShuffle(in []fclient.PublicRoom) []fclient.PublicRoom {
 	// de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers
 	// are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit)
-	var publicRooms []gomatrixserverlib.PublicRoom
+	var publicRooms []fclient.PublicRoom
 	haveRoomIDs := make(map[string]bool)
 	rand.Shuffle(len(in), func(i, j int) {
 		in[i], in[j] = in[j], in[i]

clientapi/routing/directory_public_test.go

@@ -4,17 +4,17 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 )
 
-func pubRoom(name string) gomatrixserverlib.PublicRoom {
+func pubRoom(name string) fclient.PublicRoom {
-	return gomatrixserverlib.PublicRoom{
+	return fclient.PublicRoom{
 		Name: name,
 	}
 }
 
 func TestSliceInto(t *testing.T) {
-	slice := []gomatrixserverlib.PublicRoom{
+	slice := []fclient.PublicRoom{
 		pubRoom("a"), pubRoom("b"), pubRoom("c"), pubRoom("d"), pubRoom("e"), pubRoom("f"), pubRoom("g"),
 	}
 	limit := int64(3)
@@ -22,7 +22,7 @@ func TestSliceInto(t *testing.T) {
 		since      int64
 		wantPrev   int
 		wantNext   int
-		wantSubset []gomatrixserverlib.PublicRoom
+		wantSubset []fclient.PublicRoom
 	}{
 		{
 			since:      0,

clientapi/routing/joinroom.go

@@ -18,11 +18,12 @@ import (
 	"net/http"
 	"time"
 
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -48,7 +49,7 @@ func JoinRoomByIDOrAlias(
 		for _, serverName := range serverNames {
 			joinReq.ServerNames = append(
 				joinReq.ServerNames,
-				gomatrixserverlib.ServerName(serverName),
+				spec.ServerName(serverName),
 			)
 		}
 	}
@@ -61,21 +62,19 @@ func JoinRoomByIDOrAlias(
 	// Work out our localpart for the client profile request.
 
 	// Request our profile content to populate the request content with.
-	res := &api.QueryProfileResponse{}
+	profile, err := profileAPI.QueryProfile(req.Context(), device.UserID)
-	err := profileAPI.QueryProfile(req.Context(), &api.QueryProfileRequest{UserID: device.UserID}, res)
-	if err != nil || !res.UserExists {
-		if !res.UserExists {
-			util.GetLogger(req.Context()).Error("Unable to query user profile, no profile found.")
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: jsonerror.Unknown("Unable to query user profile, no profile found."),
-			}
-		}
 
-		util.GetLogger(req.Context()).WithError(err).Error("UserProfileAPI.QueryProfile failed")
+	switch err {
-	} else {
+	case nil:
-		joinReq.Content["displayname"] = res.DisplayName
+		joinReq.Content["displayname"] = profile.DisplayName
-		joinReq.Content["avatar_url"] = res.AvatarURL
+		joinReq.Content["avatar_url"] = profile.AvatarURL
+	case appserviceAPI.ErrProfileNotExists:
+		util.GetLogger(req.Context()).Error("Unable to query user profile, no profile found.")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.Unknown("Unable to query user profile, no profile found."),
+		}
+	default:
 	}
 
 	// Ask the roomserver to perform the join.

clientapi/routing/joinroom_test.go

@@ -7,6 +7,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/appservice"
@@ -24,12 +27,15 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 
 	ctx := context.Background()
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
-		defer baseClose()
+		defer close()
 
-		rsAPI := roomserver.NewInternalAPI(base)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
-		asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+		natsInstance := jetstream.NATSInstance{}
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
 
 		// Create the users in the userapi
@@ -61,7 +67,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 			RoomAliasName: "alias",
 			Invite:        []string{bob.ID},
 			GuestCanJoin:  false,
-		}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
+		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
 		crResp, ok := resp.JSON.(createRoomResponse)
 		if !ok {
 			t.Fatalf("response is not a createRoomResponse: %+v", resp)
@@ -76,7 +82,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 			Preset:       presetPublicChat,
 			Invite:       []string{charlie.ID},
 			GuestCanJoin: true,
-		}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
+		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
 		crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse)
 		if !ok {
 			t.Fatalf("response is not a createRoomResponse: %+v", resp)

clientapi/routing/login_test.go

@@ -7,11 +7,17 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/test"
@@ -28,20 +34,24 @@ func TestLogin(t *testing.T) {
 
 	ctx := context.Background()
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
-		defer baseClose()
+		defer close()
-		base.Cfg.ClientAPI.RateLimiting.Enabled = false
+		cfg.ClientAPI.RateLimiting.Enabled = false
+		natsInstance := jetstream.NATSInstance{}
 		// add a vhost
-		base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
+		cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
+			SigningIdentity: fclient.SigningIdentity{ServerName: "vh1"},
 		})
 
-		rsAPI := roomserver.NewInternalAPI(base)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		routers := httputil.NewRouters()
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		// Needed for /login
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
-		Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, &base.Cfg.MSCs, nil)
+		Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics)
 
 		// Create password
 		password := util.RandomString(8)
@@ -114,7 +124,7 @@ func TestLogin(t *testing.T) {
 					"password": password,
 				}))
 				rec := httptest.NewRecorder()
-				base.PublicClientAPIMux.ServeHTTP(rec, req)
+				routers.Client.ServeHTTP(rec, req)
 				if tc.wantOK && rec.Code != http.StatusOK {
 					t.Fatalf("failed to login: %s", rec.Body.String())
 				}

clientapi/routing/membership.go

@@ -16,11 +16,13 @@ package routing
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -31,76 +33,56 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/util"
 )
 
-var errMissingUserID = errors.New("'user_id' must be supplied")
-
 func SendBan(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
-	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
 		return *reqErr
 	}
 
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing user_id"),
+		}
+	}
+
 	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
 
-	plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
+	pl, errRes := getPowerlevels(req, rsAPI, roomID)
-		EventType: gomatrixserverlib.MRoomPowerLevels,
+	if errRes != nil {
-		StateKey:  "",
+		return *errRes
-	})
-	if plEvent == nil {
-		return util.JSONResponse{
-			Code: 403,
-			JSON: jsonerror.Forbidden("You don't have permission to ban this user, no power_levels event in this room."),
-		}
-	}
-	pl, err := plEvent.PowerLevels()
-	if err != nil {
-		return util.JSONResponse{
-			Code: 403,
-			JSON: jsonerror.Forbidden("You don't have permission to ban this user, the power_levels event for this room is malformed so auth checks cannot be performed."),
-		}
 	}
 	allowedToBan := pl.UserLevel(device.UserID) >= pl.Ban
 	if !allowedToBan {
 		return util.JSONResponse{
-			Code: 403,
+			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("You don't have permission to ban this user, power level too low."),
 		}
 	}
 
-	return sendMembership(req.Context(), profileAPI, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+	return sendMembership(req.Context(), profileAPI, device, roomID, spec.Ban, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI)
 }
 
 func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time,
-	roomVer gomatrixserverlib.RoomVersion,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI) util.JSONResponse {
 
 	event, err := buildMembershipEvent(
 		ctx, targetUserID, reason, profileAPI, device, membership,
 		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
-	if err == errMissingUserID {
+	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
-		}
-	} else if err == eventutil.ErrRoomNoExists {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(err.Error()),
-		}
-	} else if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
 		return jsonerror.InternalServerError()
 	}
@@ -109,7 +91,7 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic
 	if err = roomserverAPI.SendEvents(
 		ctx, rsAPI,
 		roomserverAPI.KindNew,
-		[]*gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)},
+		[]*gomatrixserverlib.HeaderedEvent{event},
 		device.UserDomain(),
 		serverName,
 		serverName,
@@ -131,13 +113,65 @@ func SendKick(
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
-	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
 		return *reqErr
 	}
 	if body.UserID == "" {
 		return util.JSONResponse{
-			Code: 400,
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing user_id"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+
+	pl, errRes := getPowerlevels(req, rsAPI, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+	allowedToKick := pl.UserLevel(device.UserID) >= pl.Kick
+	if !allowedToKick {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You don't have permission to kick this user, power level too low."),
+		}
+	}
+
+	var queryRes roomserverAPI.QueryMembershipForUserResponse
+	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+		RoomID: roomID,
+		UserID: body.UserID,
+	}, &queryRes)
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	// kick is only valid if the user is not currently banned or left (that is, they are joined or invited)
+	if queryRes.Membership != spec.Join && queryRes.Membership != spec.Invite {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Unknown("cannot /kick banned or left users"),
+		}
+	}
+	// TODO: should we be using SendLeave instead?
+	return sendMembership(req.Context(), profileAPI, device, roomID, spec.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI)
+}
+
+func SendUnban(
+	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
+	roomID string, cfg *config.ClientAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
+) util.JSONResponse {
+	body, evTime, reqErr := extractRequestData(req)
+	if reqErr != nil {
+		return *reqErr
+	}
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
 			JSON: jsonerror.BadJSON("missing user_id"),
 		}
 	}
@@ -155,56 +189,16 @@ func SendKick(
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	// kick is only valid if the user is not currently banned or left (that is, they are joined or invited)
-	if queryRes.Membership != "join" && queryRes.Membership != "invite" {
-		return util.JSONResponse{
-			Code: 403,
-			JSON: jsonerror.Unknown("cannot /kick banned or left users"),
-		}
-	}
-	// TODO: should we be using SendLeave instead?
-	return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
-}
 
-func SendUnban(
-	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
-	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
-) util.JSONResponse {
-	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
-	if reqErr != nil {
-		return *reqErr
-	}
-	if body.UserID == "" {
-		return util.JSONResponse{
-			Code: 400,
-			JSON: jsonerror.BadJSON("missing user_id"),
-		}
-	}
-
-	var queryRes roomserverAPI.QueryMembershipForUserResponse
-	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
-		RoomID: roomID,
-		UserID: body.UserID,
-	}, &queryRes)
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-	if !queryRes.RoomExists {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("room does not exist"),
-		}
-	}
 	// unban is only valid if the user is currently banned
-	if queryRes.Membership != "ban" {
+	if queryRes.Membership != spec.Ban {
 		return util.JSONResponse{
-			Code: 400,
+			Code: http.StatusBadRequest,
 			JSON: jsonerror.Unknown("can only /unban users that are banned"),
 		}
 	}
 	// TODO: should we be using SendLeave instead?
-	return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+	return sendMembership(req.Context(), profileAPI, device, roomID, spec.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI)
 }
 
 func SendInvite(
@@ -212,7 +206,7 @@ func SendInvite(
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
-	body, evTime, _, reqErr := extractRequestData(req, roomID, rsAPI)
+	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
 		return *reqErr
 	}
@@ -234,6 +228,18 @@ func SendInvite(
 		}
 	}
 
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing user_id"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+
 	// We already received the return value, so no need to check for an error here.
 	response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime)
 	return response
@@ -250,20 +256,10 @@ func sendInvite(
 	asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
 ) (util.JSONResponse, error) {
 	event, err := buildMembershipEvent(
-		ctx, userID, reason, profileAPI, device, "invite",
+		ctx, userID, reason, profileAPI, device, spec.Invite,
 		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
-	if err == errMissingUserID {
+	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
-		}, err
-	} else if err == eventutil.ErrRoomNoExists {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(err.Error()),
-		}, err
-	} else if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
 		return jsonerror.InternalServerError(), err
 	}
@@ -357,19 +353,7 @@ func loadProfile(
 	return profile, err
 }
 
-func extractRequestData(req *http.Request, roomID string, rsAPI roomserverAPI.ClientRoomserverAPI) (
+func extractRequestData(req *http.Request) (body *threepid.MembershipRequest, evTime time.Time, resErr *util.JSONResponse) {
-	body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse,
-) {
-	verReq := roomserverAPI.QueryRoomVersionForRoomRequest{RoomID: roomID}
-	verRes := roomserverAPI.QueryRoomVersionForRoomResponse{}
-	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
-		resErr = &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
-		}
-		return
-	}
-	roomVer = verRes.RoomVersion
 
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		resErr = reqErr
@@ -432,34 +416,17 @@ func checkAndProcessThreepid(
 }
 
 func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID, roomID string) *util.JSONResponse {
-	tuple := gomatrixserverlib.StateKeyTuple{
+	var membershipRes roomserverAPI.QueryMembershipForUserResponse
-		EventType: gomatrixserverlib.MRoomMember,
+	err := rsAPI.QueryMembershipForUser(ctx, &roomserverAPI.QueryMembershipForUserRequest{
-		StateKey:  userID,
+		RoomID: roomID,
-	}
+		UserID: userID,
-	var membershipRes roomserverAPI.QueryCurrentStateResponse
-	err := rsAPI.QueryCurrentState(ctx, &roomserverAPI.QueryCurrentStateRequest{
-		RoomID:      roomID,
-		StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
 	}, &membershipRes)
 	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("QueryCurrentState: could not query membership for user")
+		util.GetLogger(ctx).WithError(err).Error("QueryMembershipForUser: could not query membership for user")
 		e := jsonerror.InternalServerError()
 		return &e
 	}
-	ev := membershipRes.StateEvents[tuple]
+	if !membershipRes.IsInRoom {
-	if ev == nil {
-		return &util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("user does not belong to room"),
-		}
-	}
-	membership, err := ev.Membership()
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("Member event isn't valid")
-		e := jsonerror.InternalServerError()
-		return &e
-	}
-	if membership != gomatrixserverlib.Join {
 		return &util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("user does not belong to room"),
@@ -511,3 +478,24 @@ func SendForget(
 		JSON: struct{}{},
 	}
 }
+
+func getPowerlevels(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, roomID string) (*gomatrixserverlib.PowerLevelContent, *util.JSONResponse) {
+	plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
+		EventType: spec.MRoomPowerLevels,
+		StateKey:  "",
+	})
+	if plEvent == nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You don't have permission to perform this action, no power_levels event in this room."),
+		}
+	}
+	pl, err := plEvent.PowerLevels()
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You don't have permission to perform this action, the power_levels event for this room is malformed so auth checks cannot be performed."),
+		}
+	}
+	return pl, nil
+}

clientapi/routing/peekroom.go

@@ -20,7 +20,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -49,7 +49,7 @@ func PeekRoomByIDOrAlias(
 		for _, serverName := range serverNames {
 			peekReq.ServerNames = append(
 				peekReq.ServerNames,
-				gomatrixserverlib.ServerName(serverName),
+				spec.ServerName(serverName),
 			)
 		}
 	}

clientapi/routing/presence.go

@@ -27,7 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
@@ -123,7 +123,7 @@ func GetPresence(
 		}
 	}
 
-	p := types.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(lastActive)}
+	p := types.PresenceInternal{LastActiveTS: spec.Timestamp(lastActive)}
 	currentlyActive := p.CurrentlyActive()
 	return util.JSONResponse{
 		Code: http.StatusOK,

clientapi/routing/profile.go

@@ -20,6 +20,8 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@@ -36,14 +38,14 @@ import (
 
 // GetProfile implements GET /profile/{userID}
 func GetProfile(
-	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
 	if err != nil {
-		if err == eventutil.ErrProfileNoExists {
+		if err == appserviceAPI.ErrProfileNotExists {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
 				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
@@ -56,7 +58,7 @@ func GetProfile(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: eventutil.ProfileResponse{
+		JSON: eventutil.UserProfile{
 			AvatarURL:   profile.AvatarURL,
 			DisplayName: profile.DisplayName,
 		},
@@ -65,34 +67,28 @@ func GetProfile(
 
 // GetAvatarURL implements GET /profile/{userID}/avatar_url
 func GetAvatarURL(
-	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 ) util.JSONResponse {
-	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
+	profile := GetProfile(req, profileAPI, cfg, userID, asAPI, federation)
-	if err != nil {
+	p, ok := profile.JSON.(eventutil.UserProfile)
-		if err == eventutil.ErrProfileNoExists {
+	// not a profile response, so most likely an error, return that
-			return util.JSONResponse{
+	if !ok {
-				Code: http.StatusNotFound,
+		return profile
-				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
-			}
-		}
-
-		util.GetLogger(req.Context()).WithError(err).Error("getProfile failed")
-		return jsonerror.InternalServerError()
 	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: eventutil.AvatarURL{
+		JSON: eventutil.UserProfile{
-			AvatarURL: profile.AvatarURL,
+			AvatarURL: p.AvatarURL,
 		},
 	}
 }
 
 // SetAvatarURL implements PUT /profile/{userID}/avatar_url
 func SetAvatarURL(
-	req *http.Request, profileAPI userapi.ClientUserAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI,
 	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -102,7 +98,7 @@ func SetAvatarURL(
 		}
 	}
 
-	var r eventutil.AvatarURL
+	var r eventutil.UserProfile
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
@@ -134,24 +130,20 @@ func SetAvatarURL(
 		}
 	}
 
-	setRes := &userapi.PerformSetAvatarURLResponse{}
+	profile, changed, err := profileAPI.SetAvatarURL(req.Context(), localpart, domain, r.AvatarURL)
-	if err = profileAPI.SetAvatarURL(req.Context(), &userapi.PerformSetAvatarURLRequest{
+	if err != nil {
-		Localpart:  localpart,
-		ServerName: domain,
-		AvatarURL:  r.AvatarURL,
-	}, setRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetAvatarURL failed")
 		return jsonerror.InternalServerError()
 	}
 	// No need to build new membership events, since nothing changed
-	if !setRes.Changed {
+	if !changed {
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: struct{}{},
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, setRes.Profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
 	if err != nil {
 		return response
 	}
@@ -164,34 +156,28 @@ func SetAvatarURL(
 
 // GetDisplayName implements GET /profile/{userID}/displayname
 func GetDisplayName(
-	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 ) util.JSONResponse {
-	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
+	profile := GetProfile(req, profileAPI, cfg, userID, asAPI, federation)
-	if err != nil {
+	p, ok := profile.JSON.(eventutil.UserProfile)
-		if err == eventutil.ErrProfileNoExists {
+	// not a profile response, so most likely an error, return that
-			return util.JSONResponse{
+	if !ok {
-				Code: http.StatusNotFound,
+		return profile
-				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
-			}
-		}
-
-		util.GetLogger(req.Context()).WithError(err).Error("getProfile failed")
-		return jsonerror.InternalServerError()
 	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: eventutil.DisplayName{
+		JSON: eventutil.UserProfile{
-			DisplayName: profile.DisplayName,
+			DisplayName: p.DisplayName,
 		},
 	}
 }
 
 // SetDisplayName implements PUT /profile/{userID}/displayname
 func SetDisplayName(
-	req *http.Request, profileAPI userapi.ClientUserAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI,
 	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -201,7 +187,7 @@ func SetDisplayName(
 		}
 	}
 
-	var r eventutil.DisplayName
+	var r eventutil.UserProfile
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
@@ -233,25 +219,20 @@ func SetDisplayName(
 		}
 	}
 
-	profileRes := &userapi.PerformUpdateDisplayNameResponse{}
+	profile, changed, err := profileAPI.SetDisplayName(req.Context(), localpart, domain, r.DisplayName)
-	err = profileAPI.SetDisplayName(req.Context(), &userapi.PerformUpdateDisplayNameRequest{
-		Localpart:   localpart,
-		ServerName:  domain,
-		DisplayName: r.DisplayName,
-	}, profileRes)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetDisplayName failed")
 		return jsonerror.InternalServerError()
 	}
 	// No need to build new membership events, since nothing changed
-	if !profileRes.Changed {
+	if !changed {
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: struct{}{},
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, profileRes.Profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
 	if err != nil {
 		return response
 	}
@@ -308,12 +289,12 @@ func updateProfile(
 // getProfile gets the full profile of a user by querying the database or a
 // remote homeserver.
 // Returns an error when something goes wrong or specifically
-// eventutil.ErrProfileNoExists when the profile doesn't exist.
+// eventutil.ErrProfileNotExists when the profile doesn't exist.
 func getProfile(
-	ctx context.Context, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	ctx context.Context, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 ) (*authtypes.Profile, error) {
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
@@ -325,7 +306,7 @@ func getProfile(
 		if fedErr != nil {
 			if x, ok := fedErr.(gomatrix.HTTPError); ok {
 				if x.Code == http.StatusNotFound {
-					return nil, eventutil.ErrProfileNoExists
+					return nil, appserviceAPI.ErrProfileNotExists
 				}
 			}
 
@@ -371,7 +352,7 @@ func buildMembershipEvents(
 		}
 
 		content := gomatrixserverlib.MemberContent{
-			Membership: gomatrixserverlib.Join,
+			Membership: spec.Join,
 		}
 
 		content.DisplayName = newProfile.DisplayName

clientapi/routing/pushrules.go

@@ -31,7 +31,7 @@ func errorResponse(ctx context.Context, err error, msg string, args ...interface
 }
 
 func GetAllPushRules(ctx context.Context, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
+	ruleSets, err := userAPI.QueryPushRules(ctx, device.UserID)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRulesJSON failed")
 	}
@@ -42,7 +42,7 @@ func GetAllPushRules(ctx context.Context, device *userapi.Device, userAPI userap
 }
 
 func GetPushRulesByScope(ctx context.Context, scope string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
+	ruleSets, err := userAPI.QueryPushRules(ctx, device.UserID)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRulesJSON failed")
 	}
@@ -57,7 +57,7 @@ func GetPushRulesByScope(ctx context.Context, scope string, device *userapi.Devi
 }
 
 func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
+	ruleSets, err := userAPI.QueryPushRules(ctx, device.UserID)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRules failed")
 	}
@@ -66,7 +66,8 @@ func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi
 		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rule set"), "pushRuleSetByScope failed")
 	}
 	rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind))
-	if rulesPtr == nil {
+	// Even if rulesPtr is not nil, there may not be any rules for this kind
+	if rulesPtr == nil || (rulesPtr != nil && len(*rulesPtr) == 0) {
 		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rules kind"), "pushRuleSetKindPointer failed")
 	}
 	return util.JSONResponse{
@@ -76,7 +77,7 @@ func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi
 }
 
 func GetPushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
+	ruleSets, err := userAPI.QueryPushRules(ctx, device.UserID)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRules failed")
 	}
@@ -101,7 +102,10 @@ func GetPushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device
 func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID, beforeRuleID string, body io.Reader, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	var newRule pushrules.Rule
 	if err := json.NewDecoder(body).Decode(&newRule); err != nil {
-		return errorResponse(ctx, err, "JSON Decode failed")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
 	}
 	newRule.RuleID = ruleID
 
@@ -110,7 +114,7 @@ func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID,
 		return errorResponse(ctx, jsonerror.InvalidArgumentValue(errs[0].Error()), "rule sanity check failed: %v", errs)
 	}
 
-	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
+	ruleSets, err := userAPI.QueryPushRules(ctx, device.UserID)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRules failed")
 	}
@@ -120,6 +124,7 @@ func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID,
 	}
 	rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind))
 	if rulesPtr == nil {
+		// while this should be impossible (ValidateRule would already return an error), better keep it around
 		return errorResponse(ctx, jsonerror.InvalidArgumentValue("invalid push rules kind"), "pushRuleSetKindPointer failed")
 	}
 	i := pushRuleIndexByID(*rulesPtr, ruleID)
@@ -144,7 +149,7 @@ func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID,
 		}
 
 		// Add new rule.
-		i, err := findPushRuleInsertionIndex(*rulesPtr, afterRuleID, beforeRuleID)
+		i, err = findPushRuleInsertionIndex(*rulesPtr, afterRuleID, beforeRuleID)
 		if err != nil {
 			return errorResponse(ctx, err, "findPushRuleInsertionIndex failed")
 		}
@@ -153,7 +158,7 @@ func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID,
 		util.GetLogger(ctx).WithField("after", afterRuleID).WithField("before", beforeRuleID).Infof("Added new push rule at %d", i)
 	}
 
-	if err := putPushRules(ctx, device.UserID, ruleSets, userAPI); err != nil {
+	if err = userAPI.PerformPushRulesPut(ctx, device.UserID, ruleSets); err != nil {
 		return errorResponse(ctx, err, "putPushRules failed")
 	}
 
@@ -161,7 +166,7 @@ func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID,
 }
 
 func DeletePushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
-	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
+	ruleSets, err := userAPI.QueryPushRules(ctx, device.UserID)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRules failed")
 	}
@@ -180,7 +185,7 @@ func DeletePushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, dev
 
 	*rulesPtr = append((*rulesPtr)[:i], (*rulesPtr)[i+1:]...)
 
-	if err := putPushRules(ctx, device.UserID, ruleSets, userAPI); err != nil {
+	if err = userAPI.PerformPushRulesPut(ctx, device.UserID, ruleSets); err != nil {
 		return errorResponse(ctx, err, "putPushRules failed")
 	}
 
@@ -192,7 +197,7 @@ func GetPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
 	if err != nil {
 		return errorResponse(ctx, err, "pushRuleAttrGetter failed")
 	}
-	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
+	ruleSets, err := userAPI.QueryPushRules(ctx, device.UserID)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRules failed")
 	}
@@ -238,7 +243,7 @@ func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
 		return errorResponse(ctx, err, "pushRuleAttrSetter failed")
 	}
 
-	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
+	ruleSets, err := userAPI.QueryPushRules(ctx, device.UserID)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRules failed")
 	}
@@ -258,7 +263,7 @@ func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
 	if !reflect.DeepEqual(attrGet((*rulesPtr)[i]), attrGet(&newPartialRule)) {
 		attrSet((*rulesPtr)[i], &newPartialRule)
 
-		if err := putPushRules(ctx, device.UserID, ruleSets, userAPI); err != nil {
+		if err = userAPI.PerformPushRulesPut(ctx, device.UserID, ruleSets); err != nil {
 			return errorResponse(ctx, err, "putPushRules failed")
 		}
 	}
@@ -266,28 +271,6 @@ func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
 	return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
 }
 
-func queryPushRules(ctx context.Context, userID string, userAPI userapi.ClientUserAPI) (*pushrules.AccountRuleSets, error) {
-	var res userapi.QueryPushRulesResponse
-	if err := userAPI.QueryPushRules(ctx, &userapi.QueryPushRulesRequest{UserID: userID}, &res); err != nil {
-		util.GetLogger(ctx).WithError(err).Error("userAPI.QueryPushRules failed")
-		return nil, err
-	}
-	return res.RuleSets, nil
-}
-
-func putPushRules(ctx context.Context, userID string, ruleSets *pushrules.AccountRuleSets, userAPI userapi.ClientUserAPI) error {
-	req := userapi.PerformPushRulesPutRequest{
-		UserID:   userID,
-		RuleSets: ruleSets,
-	}
-	var res struct{}
-	if err := userAPI.PerformPushRulesPut(ctx, &req, &res); err != nil {
-		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformPushRulesPut failed")
-		return err
-	}
-	return nil
-}
-
 func pushRuleSetByScope(ruleSets *pushrules.AccountRuleSets, scope pushrules.Scope) *pushrules.RuleSet {
 	switch scope {
 	case pushrules.GlobalScope:

clientapi/routing/receipt.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/userapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -31,7 +31,7 @@ import (
 )
 
 func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
-	timestamp := gomatrixserverlib.AsTimestamp(time.Now())
+	timestamp := spec.AsTimestamp(time.Now())
 	logrus.WithFields(logrus.Fields{
 		"roomID":      roomID,
 		"receiptType": receiptType,

clientapi/routing/redaction.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -77,7 +78,7 @@ func SendRedaction(
 	allowedToRedact := ev.Sender() == device.UserID
 	if !allowedToRedact {
 		plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
-			EventType: gomatrixserverlib.MRoomPowerLevels,
+			EventType: spec.MRoomPowerLevels,
 			StateKey:  "",
 		})
 		if plEvent == nil {
@@ -114,7 +115,7 @@ func SendRedaction(
 	builder := gomatrixserverlib.EventBuilder{
 		Sender:  device.UserID,
 		RoomID:  roomID,
-		Type:    gomatrixserverlib.MRoomRedaction,
+		Type:    spec.MRoomRedaction,
 		Redacts: eventID,
 	}
 	err := builder.SetContent(r)

clientapi/routing/register.go

@@ -37,6 +37,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/gomatrixserverlib/tokens"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
@@ -206,10 +207,10 @@ var (
 // previous parameters with the ones supplied. This mean you cannot "build up" request params.
 type registerRequest struct {
 	// registration parameters
-	Password   string                       `json:"password"`
+	Password   string          `json:"password"`
-	Username   string                       `json:"username"`
+	Username   string          `json:"username"`
-	ServerName gomatrixserverlib.ServerName `json:"-"`
+	ServerName spec.ServerName `json:"-"`
-	Admin      bool                         `json:"admin"`
+	Admin      bool            `json:"admin"`
 	// user-interactive auth params
 	Auth authDict `json:"auth"`
 
@@ -478,7 +479,7 @@ func Register(
 	}
 
 	var r registerRequest
-	host := gomatrixserverlib.ServerName(req.Host)
+	host := spec.ServerName(req.Host)
 	if v := cfg.Matrix.VirtualHostForHTTPHost(host); v != nil {
 		r.ServerName = v.ServerName
 	} else {
@@ -824,7 +825,7 @@ func checkAndCompleteFlow(
 func completeRegistration(
 	ctx context.Context,
 	userAPI userapi.ClientUserAPI,
-	username string, serverName gomatrixserverlib.ServerName, displayName string,
+	username string, serverName spec.ServerName, displayName string,
 	password, appserviceID, ipAddr, userAgent, sessionID string,
 	inhibitLogin eventutil.WeakBoolean,
 	deviceDisplayName, deviceID *string,
@@ -888,13 +889,7 @@ func completeRegistration(
 	}
 
 	if displayName != "" {
-		nameReq := userapi.PerformUpdateDisplayNameRequest{
+		_, _, err = userAPI.SetDisplayName(ctx, username, serverName, displayName)
-			Localpart:   username,
-			ServerName:  serverName,
-			DisplayName: displayName,
-		}
-		var nameRes userapi.PerformUpdateDisplayNameResponse
-		err = userAPI.SetDisplayName(ctx, &nameReq, &nameRes)
 		if err != nil {
 			return util.JSONResponse{
 				Code: http.StatusInternalServerError,
@@ -1000,7 +995,7 @@ func RegisterAvailable(
 	// Squash username to all lowercase letters
 	username = strings.ToLower(username)
 	domain := cfg.Matrix.ServerName
-	host := gomatrixserverlib.ServerName(req.Host)
+	host := spec.ServerName(req.Host)
 	if v := cfg.Matrix.VirtualHostForHTTPHost(host); v != nil {
 		domain = v.ServerName
 	}

clientapi/routing/register_test.go

@@ -30,8 +30,11 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi"
@@ -404,11 +407,15 @@ func Test_register(t *testing.T) {
 	}
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
-		defer baseClose()
+		defer close()
 
-		rsAPI := roomserver.NewInternalAPI(base)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		natsInstance := jetstream.NATSInstance{}
+
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
@@ -430,16 +437,16 @@ func Test_register(t *testing.T) {
 						}
 					}))
 					defer srv.Close()
-					base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
+					cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
 				}
 
-				if err := base.Cfg.Derive(); err != nil {
+				if err := cfg.Derive(); err != nil {
 					t.Fatalf("failed to derive config: %s", err)
 				}
 
-				base.Cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha
+				cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha
-				base.Cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled
+				cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled
-				base.Cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled
+				cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled
 
 				if tc.kind == "" {
 					tc.kind = "user"
@@ -467,15 +474,15 @@ func Test_register(t *testing.T) {
 
 				req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/?kind=%s", tc.kind), body)
 
-				resp := Register(req, userAPI, &base.Cfg.ClientAPI)
+				resp := Register(req, userAPI, &cfg.ClientAPI)
 				t.Logf("Resp: %+v", resp)
 
 				// The first request should return a userInteractiveResponse
 				switch r := resp.JSON.(type) {
 				case userInteractiveResponse:
 					// Check that the flows are the ones we configured
-					if !reflect.DeepEqual(r.Flows, base.Cfg.Derived.Registration.Flows) {
+					if !reflect.DeepEqual(r.Flows, cfg.Derived.Registration.Flows) {
-						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, base.Cfg.Derived.Registration.Flows)
+						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows)
 					}
 				case *jsonerror.MatrixError:
 					if !reflect.DeepEqual(tc.wantResponse, resp) {
@@ -531,7 +538,7 @@ func Test_register(t *testing.T) {
 
 				req = httptest.NewRequest(http.MethodPost, "/", body)
 
-				resp = Register(req, userAPI, &base.Cfg.ClientAPI)
+				resp = Register(req, userAPI, &cfg.ClientAPI)
 
 				switch resp.JSON.(type) {
 				case *jsonerror.MatrixError:
@@ -574,16 +581,19 @@ func Test_register(t *testing.T) {
 
 func TestRegisterUserWithDisplayName(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
-		defer baseClose()
+		defer close()
-		base.Cfg.Global.ServerName = "server"
+		cfg.Global.ServerName = "server"
 
-		rsAPI := roomserver.NewInternalAPI(base)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		natsInstance := jetstream.NATSInstance{}
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 		deviceName, deviceID := "deviceName", "deviceID"
 		expectedDisplayName := "DisplayName"
 		response := completeRegistration(
-			base.Context(),
+			processCtx.Context(),
 			userAPI,
 			"user",
 			"server",
@@ -601,24 +611,25 @@ func TestRegisterUserWithDisplayName(t *testing.T) {
 
 		assert.Equal(t, http.StatusOK, response.Code)
 
-		req := api.QueryProfileRequest{UserID: "@user:server"}
+		profile, err := userAPI.QueryProfile(processCtx.Context(), "@user:server")
-		var res api.QueryProfileResponse
-		err := userAPI.QueryProfile(base.Context(), &req, &res)
 		assert.NoError(t, err)
-		assert.Equal(t, expectedDisplayName, res.DisplayName)
+		assert.Equal(t, expectedDisplayName, profile.DisplayName)
 	})
 }
 
 func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
-		defer baseClose()
+		defer close()
-		base.Cfg.Global.ServerName = "server"
+		natsInstance := jetstream.NATSInstance{}
+		cfg.Global.ServerName = "server"
 		sharedSecret := "dendritetest"
-		base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret
+		cfg.ClientAPI.RegistrationSharedSecret = sharedSecret
 
-		rsAPI := roomserver.NewInternalAPI(base)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		expectedDisplayName := "rabbit"
 		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
@@ -642,17 +653,15 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 		ssrr := httptest.NewRequest(http.MethodPost, "/", body)
 
 		response := handleSharedSecretRegistration(
-			&base.Cfg.ClientAPI,
+			&cfg.ClientAPI,
 			userAPI,
 			r,
 			ssrr,
 		)
 		assert.Equal(t, http.StatusOK, response.Code)
 
-		profilReq := api.QueryProfileRequest{UserID: "@alice:server"}
+		profile, err := userAPI.QueryProfile(processCtx.Context(), "@alice:server")
-		var profileRes api.QueryProfileResponse
-		err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes)
 		assert.NoError(t, err)
-		assert.Equal(t, expectedDisplayName, profileRes.DisplayName)
+		assert.Equal(t, expectedDisplayName, profile.DisplayName)
 	})
 }

clientapi/routing/routing.go

@@ -18,12 +18,12 @@ import (
 	"context"
 	"net/http"
 	"strings"
-	"sync"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/setup/base"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	"github.com/prometheus/client_golang/prometheus"
@@ -50,25 +50,27 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	base *base.BaseDendrite,
+	routers httputil.Routers,
-	cfg *config.ClientAPI,
+	dendriteCfg *config.Dendrite,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.QuerySearchProfilesAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
 	federationSender federationAPI.ClientFederationAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
-	mscCfg *config.MSCs, natsClient *nats.Conn,
+	natsClient *nats.Conn, enableMetrics bool,
 ) {
-	publicAPIMux := base.PublicClientAPIMux
+	cfg := &dendriteCfg.ClientAPI
-	wkMux := base.PublicWellKnownAPIMux
+	mscCfg := &dendriteCfg.MSCs
-	synapseAdminRouter := base.SynapseAdminMux
+	publicAPIMux := routers.Client
-	dendriteAdminRouter := base.DendriteAdminMux
+	wkMux := routers.WellKnown
+	synapseAdminRouter := routers.SynapseAdmin
+	dendriteAdminRouter := routers.DendriteAdmin
 
-	if base.EnableMetrics {
+	if enableMetrics {
 		prometheus.MustRegister(amtRegUsers, sendEventDuration)
 	}
 
@@ -154,15 +156,15 @@ func Setup(
 
 	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
 		httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateRoom(req, cfg, device, rsAPI)
+			return AdminEvacuateRoom(req, rsAPI)
 		}),
-	).Methods(http.MethodGet, http.MethodOptions)
+	).Methods(http.MethodPost, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}",
 		httputil.MakeAdminAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateUser(req, cfg, device, rsAPI)
+			return AdminEvacuateUser(req, cfg, rsAPI)
 		}),
-	).Methods(http.MethodGet, http.MethodOptions)
+	).Methods(http.MethodPost, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/purgeRoom/{roomID}",
 		httputil.MakeAdminAPI("admin_purge_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -197,18 +199,13 @@ func Setup(
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
-		var serverNotificationSender *userapi.Device
+		serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
-		var err error
+		if err != nil {
-		notificationSenderOnce := &sync.Once{}
+			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+		}
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
 			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-				notificationSenderOnce.Do(func() {
-					serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
-					if err != nil {
-						logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
-					}
-				})
 				// not specced, but ensure we're rate limiting requests to this endpoint
 				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
@@ -230,12 +227,6 @@ func Setup(
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice",
 			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-				notificationSenderOnce.Do(func() {
-					serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
-					if err != nil {
-						logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
-					}
-				})
 				// not specced, but ensure we're rate limiting requests to this endpoint
 				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
@@ -266,7 +257,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/join/{roomIDOrAlias}",
-		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		httputil.MakeAuthAPI(spec.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
@@ -282,7 +273,7 @@ func Setup(
 
 	if mscCfg.Enabled("msc2753") {
 		v3mux.Handle("/peek/{roomIDOrAlias}",
-			httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			httputil.MakeAuthAPI(spec.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
 				}
@@ -302,7 +293,7 @@ func Setup(
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodGet, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/join",
-		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		httputil.MakeAuthAPI(spec.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
@@ -656,7 +647,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	v3mux.Handle("/auth/{authType}/fallback/web",
-		httputil.MakeHTMLAPI("auth_fallback", base.EnableMetrics, func(w http.ResponseWriter, req *http.Request) {
+		httputil.MakeHTMLAPI("auth_fallback", enableMetrics, func(w http.ResponseWriter, req *http.Request) {
 			vars := mux.Vars(req)
 			AuthFallback(w, req, vars["authType"], cfg)
 		}),
@@ -860,6 +851,8 @@ func Setup(
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
 	// PUT requests, so we need to allow this method
 
+	threePIDClient := base.CreateClient(dendriteCfg, nil) // TODO: Move this somewhere else, e.g. pass in as parameter
+
 	v3mux.Handle("/account/3pid",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetAssociated3PIDs(req, userAPI, device)
@@ -868,11 +861,11 @@ func Setup(
 
 	v3mux.Handle("/account/3pid",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return CheckAndSave3PIDAssociation(req, userAPI, device, cfg)
+			return CheckAndSave3PIDAssociation(req, userAPI, device, cfg, threePIDClient)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	unstableMux.Handle("/account/3pid/delete",
+	v3mux.Handle("/account/3pid/delete",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return Forget3PID(req, userAPI)
 		}),
@@ -880,7 +873,7 @@ func Setup(
 
 	v3mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken",
 		httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse {
-			return RequestEmailToken(req, userAPI, cfg)
+			return RequestEmailToken(req, userAPI, cfg, threePIDClient)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -1114,7 +1107,7 @@ func Setup(
 
 	v3mux.Handle("/delete_devices",
 		httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return DeleteDevices(req, userAPI, device)
+			return DeleteDevices(req, userInteractiveAuth, userAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -1193,7 +1186,7 @@ func Setup(
 			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
-			return GetCapabilities(req, rsAPI)
+			return GetCapabilities()
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -1405,7 +1398,7 @@ func Setup(
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
-		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		httputil.MakeAuthAPI(spec.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}

clientapi/routing/sendevent.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
@@ -117,7 +118,7 @@ func SendEvent(
 	// If we're sending a membership update, make sure to strip the authorised
 	// via key if it is present, otherwise other servers won't be able to auth
 	// the event if the room is set to the "restricted" join rule.
-	if eventType == gomatrixserverlib.MRoomMember {
+	if eventType == spec.MRoomMember {
 		delete(r, "join_authorised_via_users_server")
 	}
 
@@ -136,7 +137,7 @@ func SendEvent(
 	timeToGenerateEvent := time.Since(startedGeneratingEvent)
 
 	// validate that the aliases exists
-	if eventType == gomatrixserverlib.MRoomCanonicalAlias && stateKey != nil && *stateKey == "" {
+	if eventType == spec.MRoomCanonicalAlias && stateKey != nil && *stateKey == "" {
 		aliasReq := api.AliasEvent{}
 		if err = json.Unmarshal(e.Content(), &aliasReq); err != nil {
 			return util.ErrorResponse(fmt.Errorf("unable to parse alias event: %w", err))

clientapi/routing/sendtyping.go

@@ -15,12 +15,13 @@ package routing
 import (
 	"net/http"
 
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/util"
 )
 
 type typingContentJSON struct {

clientapi/routing/server_notices.go

@@ -295,30 +295,28 @@ func getSenderDevice(
 	}
 
 	// Set the avatarurl for the user
-	avatarRes := &userapi.PerformSetAvatarURLResponse{}
+	profile, avatarChanged, err := userAPI.SetAvatarURL(ctx,
-	if err = userAPI.SetAvatarURL(ctx, &userapi.PerformSetAvatarURLRequest{
+		cfg.Matrix.ServerNotices.LocalPart,
-		Localpart:  cfg.Matrix.ServerNotices.LocalPart,
+		cfg.Matrix.ServerName,
-		ServerName: cfg.Matrix.ServerName,
+		cfg.Matrix.ServerNotices.AvatarURL,
-		AvatarURL:  cfg.Matrix.ServerNotices.AvatarURL,
+	)
-	}, avatarRes); err != nil {
+	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.SetAvatarURL failed")
 		return nil, err
 	}
 
-	profile := avatarRes.Profile
-
 	// Set the displayname for the user
-	displayNameRes := &userapi.PerformUpdateDisplayNameResponse{}
+	_, displayNameChanged, err := userAPI.SetDisplayName(ctx,
-	if err = userAPI.SetDisplayName(ctx, &userapi.PerformUpdateDisplayNameRequest{
+		cfg.Matrix.ServerNotices.LocalPart,
-		Localpart:   cfg.Matrix.ServerNotices.LocalPart,
+		cfg.Matrix.ServerName,
-		ServerName:  cfg.Matrix.ServerName,
+		cfg.Matrix.ServerNotices.DisplayName,
-		DisplayName: cfg.Matrix.ServerNotices.DisplayName,
+	)
-	}, displayNameRes); err != nil {
+	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.SetDisplayName failed")
 		return nil, err
 	}
 
-	if displayNameRes.Changed {
+	if displayNameChanged {
 		profile.DisplayName = cfg.Matrix.ServerNotices.DisplayName
 	}
 
@@ -334,7 +332,7 @@ func getSenderDevice(
 	// We've got an existing account, return the first device of it
 	if len(deviceRes.Devices) > 0 {
 		// If there were changes to the profile, create a new membership event
-		if displayNameRes.Changed || avatarRes.Changed {
+		if displayNameChanged || avatarChanged {
 			_, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, cfg, time.Now())
 			if err != nil {
 				return nil, err

clientapi/routing/state.go

@@ -22,14 +22,16 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/synctypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	log "github.com/sirupsen/logrus"
 )
 
 type stateEventInStateResp struct {
-	gomatrixserverlib.ClientEvent
+	synctypes.ClientEvent
 	PrevContent   json.RawMessage `json:"prev_content,omitempty"`
 	ReplacesState string          `json:"replaces_state,omitempty"`
 }
@@ -67,7 +69,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 	// that marks the room as world-readable. If we don't then we assume that
 	// the room is not world-readable.
 	for _, ev := range stateRes.StateEvents {
-		if ev.Type() == gomatrixserverlib.MRoomHistoryVisibility {
+		if ev.Type() == spec.MRoomHistoryVisibility {
 			content := map[string]string{}
 			if err := json.Unmarshal(ev.Content(), &content); err != nil {
 				util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for history visibility failed")
@@ -122,7 +124,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		"state_at_event": !wantLatestState,
 	}).Info("Fetching all state")
 
-	stateEvents := []gomatrixserverlib.ClientEvent{}
+	stateEvents := []synctypes.ClientEvent{}
 	if wantLatestState {
 		// If we are happy to use the latest state, either because the user is
 		// still in the room, or because the room is world-readable, then just
@@ -131,7 +133,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		for _, ev := range stateRes.StateEvents {
 			stateEvents = append(
 				stateEvents,
-				gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll),
+				synctypes.HeaderedToClientEvent(ev, synctypes.FormatAll),
 			)
 		}
 	} else {
@@ -150,7 +152,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		for _, ev := range stateAfterRes.StateEvents {
 			stateEvents = append(
 				stateEvents,
-				gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll),
+				synctypes.HeaderedToClientEvent(ev, synctypes.FormatAll),
 			)
 		}
 	}
@@ -184,9 +186,9 @@ func OnIncomingStateTypeRequest(
 			StateKey:  stateKey,
 		},
 	}
-	if evType != gomatrixserverlib.MRoomHistoryVisibility && stateKey != "" {
+	if evType != spec.MRoomHistoryVisibility && stateKey != "" {
 		stateToFetch = append(stateToFetch, gomatrixserverlib.StateKeyTuple{
-			EventType: gomatrixserverlib.MRoomHistoryVisibility,
+			EventType: spec.MRoomHistoryVisibility,
 			StateKey:  "",
 		})
 	}
@@ -207,7 +209,7 @@ func OnIncomingStateTypeRequest(
 	// that marks the room as world-readable. If we don't then we assume that
 	// the room is not world-readable.
 	for _, ev := range stateRes.StateEvents {
-		if ev.Type() == gomatrixserverlib.MRoomHistoryVisibility {
+		if ev.Type() == spec.MRoomHistoryVisibility {
 			content := map[string]string{}
 			if err := json.Unmarshal(ev.Content(), &content); err != nil {
 				util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for history visibility failed")
@@ -241,7 +243,7 @@ func OnIncomingStateTypeRequest(
 		}
 		// If the user has never been in the room then stop at this point.
 		// We won't tell the user about a room they have never joined.
-		if !membershipRes.HasBeenInRoom || membershipRes.Membership == gomatrixserverlib.Ban {
+		if !membershipRes.HasBeenInRoom || membershipRes.Membership == spec.Ban {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
@@ -309,7 +311,7 @@ func OnIncomingStateTypeRequest(
 	}
 
 	stateEvent := stateEventInStateResp{
-		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(event, gomatrixserverlib.FormatAll),
+		ClientEvent: synctypes.HeaderedToClientEvent(event, synctypes.FormatAll),
 	}
 
 	var res interface{}

clientapi/routing/threepid.go

@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	userdb "github.com/matrix-org/dendrite/userapi/storage"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -33,7 +34,7 @@ type reqTokenResponse struct {
 	SID string `json:"sid"`
 }
 
-type threePIDsResponse struct {
+type ThreePIDsResponse struct {
 	ThreePIDs []authtypes.ThreePID `json:"threepids"`
 }
 
@@ -41,7 +42,7 @@ type threePIDsResponse struct {
 //
 //	POST /account/3pid/email/requestToken
 //	POST /register/email/requestToken
-func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI) util.JSONResponse {
+func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI, client *fclient.Client) util.JSONResponse {
 	var body threepid.EmailAssociationRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr
@@ -72,7 +73,7 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *co
 		}
 	}
 
-	resp.SID, err = threepid.CreateSession(req.Context(), body, cfg)
+	resp.SID, err = threepid.CreateSession(req.Context(), body, cfg, client)
 	if err == threepid.ErrNotTrusted {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -92,7 +93,7 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *co
 // CheckAndSave3PIDAssociation implements POST /account/3pid
 func CheckAndSave3PIDAssociation(
 	req *http.Request, threePIDAPI api.ClientUserAPI, device *api.Device,
-	cfg *config.ClientAPI,
+	cfg *config.ClientAPI, client *fclient.Client,
 ) util.JSONResponse {
 	var body threepid.EmailAssociationCheckRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
@@ -100,7 +101,7 @@ func CheckAndSave3PIDAssociation(
 	}
 
 	// Check if the association has been validated
-	verified, address, medium, err := threepid.CheckAssociation(req.Context(), body.Creds, cfg)
+	verified, address, medium, err := threepid.CheckAssociation(req.Context(), body.Creds, cfg, client)
 	if err == threepid.ErrNotTrusted {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -123,13 +124,8 @@ func CheckAndSave3PIDAssociation(
 
 	if body.Bind {
 		// Publish the association on the identity server if requested
-		err = threepid.PublishAssociation(body.Creds, device.UserID, cfg)
+		err = threepid.PublishAssociation(req.Context(), body.Creds, device.UserID, cfg, client)
-		if err == threepid.ErrNotTrusted {
+		if err != nil {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.NotTrusted(body.Creds.IDServer),
-			}
-		} else if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("threepid.PublishAssociation failed")
 			return jsonerror.InternalServerError()
 		}
@@ -180,7 +176,7 @@ func GetAssociated3PIDs(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: threePIDsResponse{res.ThreePIDs},
+		JSON: ThreePIDsResponse{res.ThreePIDs},
 	}
 }
 
@@ -191,7 +187,10 @@ func Forget3PID(req *http.Request, threepidAPI api.ClientUserAPI) util.JSONRespo
 		return *reqErr
 	}
 
-	if err := threepidAPI.PerformForgetThreePID(req.Context(), &api.PerformForgetThreePIDRequest{}, &struct{}{}); err != nil {
+	if err := threepidAPI.PerformForgetThreePID(req.Context(), &api.PerformForgetThreePIDRequest{
+		ThreePID: body.Address,
+		Medium:   body.Medium,
+	}, &struct{}{}); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.PerformForgetThreePID failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/userdirectory.go

@@ -26,6 +26,8 @@ import (
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
@@ -41,8 +43,8 @@ func SearchUserDirectory(
 	provider userapi.QuerySearchProfilesAPI,
 	searchString string,
 	limit int,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
-	localServerName gomatrixserverlib.ServerName,
+	localServerName spec.ServerName,
 ) util.JSONResponse {
 	if limit < 10 {
 		limit = 10

clientapi/threepid/invites.go

@@ -30,6 +30,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 // MembershipRequest represents the body of an incoming POST request
@@ -209,24 +210,17 @@ func queryIDServerStoreInvite(
 	body *MembershipRequest, roomID string,
 ) (*idServerStoreInviteResponse, error) {
 	// Retrieve the sender's profile to get their display name
-	localpart, serverName, err := gomatrixserverlib.SplitID('@', device.UserID)
+	_, serverName, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		return nil, err
 	}
 
 	var profile *authtypes.Profile
 	if cfg.Matrix.IsLocalServerName(serverName) {
-		res := &userapi.QueryProfileResponse{}
+		profile, err = userAPI.QueryProfile(ctx, device.UserID)
-		err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res)
 		if err != nil {
 			return nil, err
 		}
-		profile = &authtypes.Profile{
-			Localpart:   localpart,
-			DisplayName: res.DisplayName,
-			AvatarURL:   res.AvatarURL,
-		}
-
 	} else {
 		profile = &authtypes.Profile{}
 	}
@@ -285,7 +279,7 @@ func queryIDServerPubKey(ctx context.Context, idServerName string, keyID string)
 	}
 
 	var pubKeyRes struct {
-		PublicKey gomatrixserverlib.Base64Bytes `json:"public_key"`
+		PublicKey spec.Base64Bytes `json:"public_key"`
 	}
 
 	if resp.StatusCode != http.StatusOK {

clientapi/threepid/threepid.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 )
 
 // EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken
@@ -37,7 +38,7 @@ type EmailAssociationRequest struct {
 
 // EmailAssociationCheckRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-account-3pid
 type EmailAssociationCheckRequest struct {
-	Creds Credentials `json:"threePidCreds"`
+	Creds Credentials `json:"three_pid_creds"`
 	Bind  bool        `json:"bind"`
 }
 
@@ -48,12 +49,16 @@ type Credentials struct {
 	Secret   string `json:"client_secret"`
 }
 
+type SID struct {
+	SID string `json:"sid"`
+}
+
 // CreateSession creates a session on an identity server.
 // Returns the session's ID.
 // Returns an error if there was a problem sending the request or decoding the
 // response, or if the identity server responded with a non-OK status.
 func CreateSession(
-	ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI,
+	ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI, client *fclient.Client,
 ) (string, error) {
 	if err := isTrusted(req.IDServer, cfg); err != nil {
 		return "", err
@@ -73,8 +78,7 @@ func CreateSession(
 	}
 	request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 
-	client := http.Client{}
+	resp, err := client.DoHTTPRequest(ctx, request)
-	resp, err := client.Do(request.WithContext(ctx))
 	if err != nil {
 		return "", err
 	}
@@ -85,14 +89,20 @@ func CreateSession(
 	}
 
 	// Extract the SID from the response and return it
-	var sid struct {
+	var sid SID
-		SID string `json:"sid"`
-	}
 	err = json.NewDecoder(resp.Body).Decode(&sid)
 
 	return sid.SID, err
 }
 
+type GetValidatedResponse struct {
+	Medium      string `json:"medium"`
+	ValidatedAt int64  `json:"validated_at"`
+	Address     string `json:"address"`
+	ErrCode     string `json:"errcode"`
+	Error       string `json:"error"`
+}
+
 // CheckAssociation checks the status of an ongoing association validation on an
 // identity server.
 // Returns a boolean set to true if the association has been validated, false if not.
@@ -102,6 +112,7 @@ func CreateSession(
 // response, or if the identity server responded with a non-OK status.
 func CheckAssociation(
 	ctx context.Context, creds Credentials, cfg *config.ClientAPI,
+	client *fclient.Client,
 ) (bool, string, string, error) {
 	if err := isTrusted(creds.IDServer, cfg); err != nil {
 		return false, "", "", err
@@ -112,19 +123,12 @@ func CheckAssociation(
 	if err != nil {
 		return false, "", "", err
 	}
-	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	resp, err := client.DoHTTPRequest(ctx, req)
 	if err != nil {
 		return false, "", "", err
 	}
 
-	var respBody struct {
+	var respBody GetValidatedResponse
-		Medium      string `json:"medium"`
-		ValidatedAt int64  `json:"validated_at"`
-		Address     string `json:"address"`
-		ErrCode     string `json:"errcode"`
-		Error       string `json:"error"`
-	}
-
 	if err = json.NewDecoder(resp.Body).Decode(&respBody); err != nil {
 		return false, "", "", err
 	}
@@ -142,7 +146,7 @@ func CheckAssociation(
 // identifier and a Matrix ID.
 // Returns an error if there was a problem sending the request or decoding the
 // response, or if the identity server responded with a non-OK status.
-func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI) error {
+func PublishAssociation(ctx context.Context, creds Credentials, userID string, cfg *config.ClientAPI, client *fclient.Client) error {
 	if err := isTrusted(creds.IDServer, cfg); err != nil {
 		return err
 	}
@@ -160,8 +164,7 @@ func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI)
 	}
 	request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 
-	client := http.Client{}
+	resp, err := client.DoHTTPRequest(ctx, request)
-	resp, err := client.Do(request)
 	if err != nil {
 		return err
 	}

clientapi/userutil/userutil.go

@@ -19,13 +19,14 @@ import (
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 // ParseUsernameParam extracts localpart from usernameParam.
 // usernameParam can either be a user ID or just the localpart/username.
 // If serverName is passed, it is verified against the domain obtained from usernameParam (if present)
 // Returns error in case of invalid usernameParam.
-func ParseUsernameParam(usernameParam string, cfg *config.Global) (string, gomatrixserverlib.ServerName, error) {
+func ParseUsernameParam(usernameParam string, cfg *config.Global) (string, spec.ServerName, error) {
 	localpart := usernameParam
 
 	if strings.HasPrefix(usernameParam, "@") {
@@ -45,6 +46,6 @@ func ParseUsernameParam(usernameParam string, cfg *config.Global) (string, gomat
 }
 
 // MakeUserID generates user ID from localpart & server name
-func MakeUserID(localpart string, server gomatrixserverlib.ServerName) string {
+func MakeUserID(localpart string, server spec.ServerName) string {
 	return fmt.Sprintf("@%s:%s", localpart, string(server))
 }

clientapi/userutil/userutil_test.go

@@ -16,21 +16,22 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 var (
-	localpart                                      = "somelocalpart"
+	localpart                         = "somelocalpart"
-	serverName        gomatrixserverlib.ServerName = "someservername"
+	serverName        spec.ServerName = "someservername"
-	invalidServerName gomatrixserverlib.ServerName = "invalidservername"
+	invalidServerName spec.ServerName = "invalidservername"
-	goodUserID                                     = "@" + localpart + ":" + string(serverName)
+	goodUserID                        = "@" + localpart + ":" + string(serverName)
-	badUserID                                      = "@bad:user:name@noservername:"
+	badUserID                         = "@bad:user:name@noservername:"
 )
 
 // TestGoodUserID checks that correct localpart is returned for a valid user ID.
 func TestGoodUserID(t *testing.T) {
 	cfg := &config.Global{
-		SigningIdentity: gomatrixserverlib.SigningIdentity{
+		SigningIdentity: fclient.SigningIdentity{
 			ServerName: serverName,
 		},
 	}
@@ -49,7 +50,7 @@ func TestGoodUserID(t *testing.T) {
 // TestWithLocalpartOnly checks that localpart is returned when usernameParam contains only localpart.
 func TestWithLocalpartOnly(t *testing.T) {
 	cfg := &config.Global{
-		SigningIdentity: gomatrixserverlib.SigningIdentity{
+		SigningIdentity: fclient.SigningIdentity{
 			ServerName: serverName,
 		},
 	}
@@ -68,7 +69,7 @@ func TestWithLocalpartOnly(t *testing.T) {
 // TestIncorrectDomain checks for error when there's server name mismatch.
 func TestIncorrectDomain(t *testing.T) {
 	cfg := &config.Global{
-		SigningIdentity: gomatrixserverlib.SigningIdentity{
+		SigningIdentity: fclient.SigningIdentity{
 			ServerName: invalidServerName,
 		},
 	}
@@ -83,7 +84,7 @@ func TestIncorrectDomain(t *testing.T) {
 // TestBadUserID checks that ParseUsernameParam fails for invalid user ID
 func TestBadUserID(t *testing.T) {
 	cfg := &config.Global{
-		SigningIdentity: gomatrixserverlib.SigningIdentity{
+		SigningIdentity: fclient.SigningIdentity{
 			ServerName: serverName,
 		},
 	}

cmd/dendrite-demo-pinecone/conn/client.go

@@ -21,8 +21,8 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"nhooyr.io/websocket"
 
 	pineconeRouter "github.com/matrix-org/pinecone/router"
@@ -90,18 +90,18 @@ func createTransport(s *pineconeSessions.Sessions) *http.Transport {
 }
 
 func CreateClient(
-	base *base.BaseDendrite, s *pineconeSessions.Sessions,
+	s *pineconeSessions.Sessions,
-) *gomatrixserverlib.Client {
+) *fclient.Client {
-	return gomatrixserverlib.NewClient(
+	return fclient.NewClient(
-		gomatrixserverlib.WithTransport(createTransport(s)),
+		fclient.WithTransport(createTransport(s)),
 	)
 }
 
 func CreateFederationClient(
-	base *base.BaseDendrite, s *pineconeSessions.Sessions,
+	cfg *config.Dendrite, s *pineconeSessions.Sessions,
-) *gomatrixserverlib.FederationClient {
+) *fclient.FederationClient {
-	return gomatrixserverlib.NewFederationClient(
+	return fclient.NewFederationClient(
-		base.Cfg.Global.SigningIdentities(),
+		cfg.Global.SigningIdentities(),
-		gomatrixserverlib.WithTransport(createTransport(s)),
+		fclient.WithTransport(createTransport(s)),
 	)
 }

cmd/dendrite-demo-pinecone/defaults/defaults.go

@@ -14,8 +14,8 @@
 
 package defaults
 
-import "github.com/matrix-org/gomatrixserverlib"
+import "github.com/matrix-org/gomatrixserverlib/spec"
 
-var DefaultServerNames = map[gomatrixserverlib.ServerName]struct{}{
+var DefaultServerNames = map[spec.ServerName]struct{}{
 	"3bf0258d23c60952639cc4c69c71d1508a7d43a0475d9000ff900a1848411ec7": {},
 }

cmd/dendrite-demo-pinecone/main.go

@@ -27,9 +27,13 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/monolith"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 
 	pineconeRouter "github.com/matrix-org/pinecone/router"
@@ -74,7 +78,7 @@ func main() {
 		cfg = monolith.GenerateDefaultConfig(sk, *instanceDir, *instanceDir, *instanceName)
 	}
 
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
 	p2pMonolith := monolith.P2PMonolith{}
@@ -87,9 +91,13 @@ func main() {
 		}
 	}
 
+	processCtx := process.NewProcessContext()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+
 	enableMetrics := true
 	enableWebsockets := true
-	p2pMonolith.SetupDendrite(cfg, *instancePort, *instanceRelayingEnabled, enableMetrics, enableWebsockets)
+	p2pMonolith.SetupDendrite(processCtx, cfg, cm, routers, *instancePort, *instanceRelayingEnabled, enableMetrics, enableWebsockets)
 	p2pMonolith.StartMonolith()
 	p2pMonolith.WaitForShutdown()
 

cmd/dendrite-demo-pinecone/monolith/monolith.go

@@ -23,6 +23,7 @@ import (
 	"net"
 	"net/http"
 	"path/filepath"
+	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -37,7 +38,9 @@ import (
 	"github.com/matrix-org/dendrite/federationapi"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/producers"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/relayapi"
 	relayAPI "github.com/matrix-org/dendrite/relayapi/api"
 	"github.com/matrix-org/dendrite/roomserver"
@@ -45,9 +48,10 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/userapi"
 	userAPI "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 
 	pineconeConnections "github.com/matrix-org/pinecone/connections"
@@ -60,13 +64,13 @@ import (
 const SessionProtocol = "matrix"
 
 type P2PMonolith struct {
-	BaseDendrite   *base.BaseDendrite
 	Sessions       *pineconeSessions.Sessions
 	Multicast      *pineconeMulticast.Multicast
 	ConnManager    *pineconeConnections.ConnectionManager
 	Router         *pineconeRouter.Router
 	EventChannel   chan pineconeEvents.Event
 	RelayRetriever relay.RelayServerRetriever
+	ProcessCtx     *process.ProcessContext
 
 	dendrite           setup.Monolith
 	port               int
@@ -76,6 +80,7 @@ type P2PMonolith struct {
 	listener           net.Listener
 	httpListenAddr     string
 	stopHandlingEvents chan bool
+	httpServerMu       sync.Mutex
 }
 
 func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir string, dbPrefix string) *config.Dendrite {
@@ -120,53 +125,52 @@ func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) {
 	p.ConnManager = pineconeConnections.NewConnectionManager(p.Router, nil)
 }
 
-func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) {
+func (p *P2PMonolith) SetupDendrite(
-	if enableMetrics {
+	processCtx *process.ProcessContext, cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers,
-		p.BaseDendrite = base.NewBaseDendrite(cfg)
+	port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) {
-	} else {
-		p.BaseDendrite = base.NewBaseDendrite(cfg, base.DisableMetrics)
-	}
-	p.port = port
-	p.BaseDendrite.ConfigureAdminEndpoints()
 
-	federation := conn.CreateFederationClient(p.BaseDendrite, p.Sessions)
+	p.port = port
+	base.ConfigureAdminEndpoints(processCtx, routers)
+
+	federation := conn.CreateFederationClient(cfg, p.Sessions)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	rsComponent := roomserver.NewInternalAPI(p.BaseDendrite)
+	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, enableMetrics)
-	rsAPI := rsComponent
+	natsInstance := jetstream.NATSInstance{}
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, enableMetrics)
 	fsAPI := federationapi.NewInternalAPI(
-		p.BaseDendrite, federation, rsAPI, p.BaseDendrite.Caches, keyRing, true,
+		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
-	userAPI := userapi.NewInternalAPI(p.BaseDendrite, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
 
-	asAPI := appservice.NewInternalAPI(p.BaseDendrite, userAPI, rsAPI)
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
-	rsComponent.SetFederationAPI(fsAPI, keyRing)
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	userProvider := users.NewPineconeUserProvider(p.Router, p.Sessions, userAPI, federation)
 	roomProvider := rooms.NewPineconeRoomProvider(p.Router, p.Sessions, fsAPI, federation)
 
-	js, _ := p.BaseDendrite.NATS.Prepare(p.BaseDendrite.ProcessContext, &p.BaseDendrite.Cfg.Global.JetStream)
+	js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
 	producer := &producers.SyncAPIProducer{
 		JetStream:              js,
-		TopicReceiptEvent:      p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+		TopicReceiptEvent:      cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent),
-		TopicSendToDeviceEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		TopicSendToDeviceEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
-		TopicTypingEvent:       p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicTypingEvent:       cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent),
-		TopicPresenceEvent:     p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		TopicPresenceEvent:     cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent),
-		TopicDeviceListUpdate:  p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
+		TopicDeviceListUpdate:  cfg.Global.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
-		TopicSigningKeyUpdate:  p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.InputSigningKeyUpdate),
+		TopicSigningKeyUpdate:  cfg.Global.JetStream.Prefixed(jetstream.InputSigningKeyUpdate),
-		Config:                 &p.BaseDendrite.Cfg.FederationAPI,
+		Config:                 &cfg.FederationAPI,
 		UserAPI:                userAPI,
 	}
-	relayAPI := relayapi.NewRelayInternalAPI(p.BaseDendrite, federation, rsAPI, keyRing, producer, enableRelaying)
+	relayAPI := relayapi.NewRelayInternalAPI(cfg, cm, federation, rsAPI, keyRing, producer, enableRelaying, caches)
 	logrus.Infof("Relaying enabled: %v", relayAPI.RelayingEnabled())
 
 	p.dendrite = setup.Monolith{
-		Config:    p.BaseDendrite.Cfg,
+		Config:    cfg,
-		Client:    conn.CreateClient(p.BaseDendrite, p.Sessions),
+		Client:    conn.CreateClient(p.Sessions),
 		FedClient: federation,
 		KeyRing:   keyRing,
 
@@ -178,9 +182,10 @@ func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelayi
 		ExtPublicRoomsProvider:   roomProvider,
 		ExtUserDirectoryProvider: userProvider,
 	}
-	p.dendrite.AddAllPublicRoutes(p.BaseDendrite)
+	p.ProcessCtx = processCtx
+	p.dendrite.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, enableMetrics)
 
-	p.setupHttpServers(userProvider, enableWebsockets)
+	p.setupHttpServers(userProvider, routers, enableWebsockets)
 }
 
 func (p *P2PMonolith) GetFederationAPI() federationAPI.FederationInternalAPI {
@@ -202,20 +207,22 @@ func (p *P2PMonolith) StartMonolith() {
 
 func (p *P2PMonolith) Stop() {
 	logrus.Info("Stopping monolith")
-	_ = p.BaseDendrite.Close()
+	p.ProcessCtx.ShutdownDendrite()
 	p.WaitForShutdown()
 	logrus.Info("Stopped monolith")
 }
 
 func (p *P2PMonolith) WaitForShutdown() {
-	p.BaseDendrite.WaitForShutdown()
+	base.WaitForShutdown(p.ProcessCtx)
 	p.closeAllResources()
 }
 
 func (p *P2PMonolith) closeAllResources() {
 	logrus.Info("Closing monolith resources")
+	p.httpServerMu.Lock()
 	if p.httpServer != nil {
 		_ = p.httpServer.Shutdown(context.Background())
+		p.httpServerMu.Unlock()
 	}
 
 	select {
@@ -245,12 +252,12 @@ func (p *P2PMonolith) Addr() string {
 	return p.httpListenAddr
 }
 
-func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, enableWebsockets bool) {
+func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, routers httputil.Routers, enableWebsockets bool) {
 	p.httpMux = mux.NewRouter().SkipClean(true).UseEncodedPath()
-	p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(p.BaseDendrite.PublicClientAPIMux)
+	p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client)
-	p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux)
+	p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
-	p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(p.BaseDendrite.DendriteAdminMux)
+	p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin)
-	p.httpMux.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(p.BaseDendrite.SynapseAdminMux)
+	p.httpMux.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin)
 
 	if enableWebsockets {
 		wsUpgrader := websocket.Upgrader{
@@ -283,8 +290,8 @@ func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider,
 
 	p.pineconeMux = mux.NewRouter().SkipClean(true).UseEncodedPath()
 	p.pineconeMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
-	p.pineconeMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(p.BaseDendrite.PublicFederationAPIMux)
+	p.pineconeMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(routers.Federation)
-	p.pineconeMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux)
+	p.pineconeMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
 
 	pHTTP := p.Sessions.Protocol(SessionProtocol).HTTP()
 	pHTTP.Mux().Handle(users.PublicURL, p.pineconeMux)
@@ -294,6 +301,7 @@ func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider,
 
 func (p *P2PMonolith) startHTTPServers() {
 	go func() {
+		p.httpServerMu.Lock()
 		// Build both ends of a HTTP multiplex.
 		p.httpServer = &http.Server{
 			Addr:         ":0",
@@ -306,7 +314,7 @@ func (p *P2PMonolith) startHTTPServers() {
 			},
 			Handler: p.pineconeMux,
 		}
-
+		p.httpServerMu.Unlock()
 		pubkey := p.Router.PublicKey()
 		pubkeyString := hex.EncodeToString(pubkey[:])
 		logrus.Info("Listening on ", pubkeyString)
@@ -339,7 +347,7 @@ func (p *P2PMonolith) startEventHandler() {
 	eLog := logrus.WithField("pinecone", "events")
 	p.RelayRetriever = relay.NewRelayServerRetriever(
 		context.Background(),
-		gomatrixserverlib.ServerName(p.Router.PublicKey().String()),
+		spec.ServerName(p.Router.PublicKey().String()),
 		p.dendrite.FederationAPI,
 		p.dendrite.RelayAPI,
 		stopRelayServerSync,
@@ -365,10 +373,10 @@ func (p *P2PMonolith) startEventHandler() {
 					// eLog.Info("Broadcast received from: ", e.PeerID)
 
 					req := &federationAPI.PerformWakeupServersRequest{
-						ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
+						ServerNames: []spec.ServerName{spec.ServerName(e.PeerID)},
 					}
 					res := &federationAPI.PerformWakeupServersResponse{}
-					if err := p.dendrite.FederationAPI.PerformWakeupServers(p.BaseDendrite.Context(), req, res); err != nil {
+					if err := p.dendrite.FederationAPI.PerformWakeupServers(p.ProcessCtx.Context(), req, res); err != nil {
 						eLog.WithError(err).Error("Failed to wakeup destination", e.PeerID)
 					}
 				}

cmd/dendrite-demo-pinecone/relay/retriever.go

@@ -21,7 +21,7 @@ import (
 
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 	"go.uber.org/atomic"
 )
@@ -32,10 +32,10 @@ const (
 
 type RelayServerRetriever struct {
 	ctx                 context.Context
-	serverName          gomatrixserverlib.ServerName
+	serverName          spec.ServerName
 	federationAPI       federationAPI.FederationInternalAPI
 	relayAPI            relayServerAPI.RelayInternalAPI
-	relayServersQueried map[gomatrixserverlib.ServerName]bool
+	relayServersQueried map[spec.ServerName]bool
 	queriedServersMutex sync.Mutex
 	running             atomic.Bool
 	quit                chan bool
@@ -43,7 +43,7 @@ type RelayServerRetriever struct {
 
 func NewRelayServerRetriever(
 	ctx context.Context,
-	serverName gomatrixserverlib.ServerName,
+	serverName spec.ServerName,
 	federationAPI federationAPI.FederationInternalAPI,
 	relayAPI relayServerAPI.RelayInternalAPI,
 	quit chan bool,
@@ -53,14 +53,14 @@ func NewRelayServerRetriever(
 		serverName:          serverName,
 		federationAPI:       federationAPI,
 		relayAPI:            relayAPI,
-		relayServersQueried: make(map[gomatrixserverlib.ServerName]bool),
+		relayServersQueried: make(map[spec.ServerName]bool),
 		running:             *atomic.NewBool(false),
 		quit:                quit,
 	}
 }
 
 func (r *RelayServerRetriever) InitializeRelayServers(eLog *logrus.Entry) {
-	request := federationAPI.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(r.serverName)}
+	request := federationAPI.P2PQueryRelayServersRequest{Server: spec.ServerName(r.serverName)}
 	response := federationAPI.P2PQueryRelayServersResponse{}
 	err := r.federationAPI.P2PQueryRelayServers(r.ctx, &request, &response)
 	if err != nil {
@@ -76,13 +76,13 @@ func (r *RelayServerRetriever) InitializeRelayServers(eLog *logrus.Entry) {
 	eLog.Infof("Registered relay servers: %v", response.RelayServers)
 }
 
-func (r *RelayServerRetriever) SetRelayServers(servers []gomatrixserverlib.ServerName) {
+func (r *RelayServerRetriever) SetRelayServers(servers []spec.ServerName) {
 	UpdateNodeRelayServers(r.serverName, servers, r.ctx, r.federationAPI)
 
 	// Replace list of servers to sync with and mark them all as unsynced.
 	r.queriedServersMutex.Lock()
 	defer r.queriedServersMutex.Unlock()
-	r.relayServersQueried = make(map[gomatrixserverlib.ServerName]bool)
+	r.relayServersQueried = make(map[spec.ServerName]bool)
 	for _, server := range servers {
 		r.relayServersQueried[server] = false
 	}
@@ -90,10 +90,10 @@ func (r *RelayServerRetriever) SetRelayServers(servers []gomatrixserverlib.Serve
 	r.StartSync()
 }
 
-func (r *RelayServerRetriever) GetRelayServers() []gomatrixserverlib.ServerName {
+func (r *RelayServerRetriever) GetRelayServers() []spec.ServerName {
 	r.queriedServersMutex.Lock()
 	defer r.queriedServersMutex.Unlock()
-	relayServers := []gomatrixserverlib.ServerName{}
+	relayServers := []spec.ServerName{}
 	for server := range r.relayServersQueried {
 		relayServers = append(relayServers, server)
 	}
@@ -101,11 +101,11 @@ func (r *RelayServerRetriever) GetRelayServers() []gomatrixserverlib.ServerName
 	return relayServers
 }
 
-func (r *RelayServerRetriever) GetQueriedServerStatus() map[gomatrixserverlib.ServerName]bool {
+func (r *RelayServerRetriever) GetQueriedServerStatus() map[spec.ServerName]bool {
 	r.queriedServersMutex.Lock()
 	defer r.queriedServersMutex.Unlock()
 
-	result := map[gomatrixserverlib.ServerName]bool{}
+	result := map[spec.ServerName]bool{}
 	for server, queried := range r.relayServersQueried {
 		result[server] = queried
 	}
@@ -128,7 +128,7 @@ func (r *RelayServerRetriever) SyncRelayServers(stop <-chan bool) {
 
 	t := time.NewTimer(relayServerRetryInterval)
 	for {
-		relayServersToQuery := []gomatrixserverlib.ServerName{}
+		relayServersToQuery := []spec.ServerName{}
 		func() {
 			r.queriedServersMutex.Lock()
 			defer r.queriedServersMutex.Unlock()
@@ -158,10 +158,10 @@ func (r *RelayServerRetriever) SyncRelayServers(stop <-chan bool) {
 	}
 }
 
-func (r *RelayServerRetriever) queryRelayServers(relayServers []gomatrixserverlib.ServerName) {
+func (r *RelayServerRetriever) queryRelayServers(relayServers []spec.ServerName) {
 	logrus.Info("Querying relay servers for any available transactions")
 	for _, server := range relayServers {
-		userID, err := gomatrixserverlib.NewUserID("@user:"+string(r.serverName), false)
+		userID, err := spec.NewUserID("@user:"+string(r.serverName), false)
 		if err != nil {
 			return
 		}
@@ -187,8 +187,8 @@ func (r *RelayServerRetriever) queryRelayServers(relayServers []gomatrixserverli
 }
 
 func UpdateNodeRelayServers(
-	node gomatrixserverlib.ServerName,
+	node spec.ServerName,
-	relays []gomatrixserverlib.ServerName,
+	relays []spec.ServerName,
 	ctx context.Context,
 	fedAPI federationAPI.FederationInternalAPI,
 ) {
@@ -201,7 +201,7 @@ func UpdateNodeRelayServers(
 	}
 
 	// Remove old, non-matching relays
-	var serversToRemove []gomatrixserverlib.ServerName
+	var serversToRemove []spec.ServerName
 	for _, existingServer := range response.RelayServers {
 		shouldRemove := true
 		for _, newServer := range relays {

cmd/dendrite-demo-pinecone/relay/retriever_test.go

@@ -21,13 +21,13 @@ import (
 
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"gotest.tools/v3/poll"
 )
 
-var testRelayServers = []gomatrixserverlib.ServerName{"relay1", "relay2"}
+var testRelayServers = []spec.ServerName{"relay1", "relay2"}
 
 type FakeFedAPI struct {
 	federationAPI.FederationInternalAPI
@@ -48,8 +48,8 @@ type FakeRelayAPI struct {
 
 func (r *FakeRelayAPI) PerformRelayServerSync(
 	ctx context.Context,
-	userID gomatrixserverlib.UserID,
+	userID spec.UserID,
-	relayServer gomatrixserverlib.ServerName,
+	relayServer spec.ServerName,
 ) error {
 	return nil
 }

cmd/dendrite-demo-pinecone/rooms/rooms.go

@@ -21,7 +21,8 @@ import (
 
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults"
 	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 
 	pineconeRouter "github.com/matrix-org/pinecone/router"
@@ -32,14 +33,14 @@ type PineconeRoomProvider struct {
 	r         *pineconeRouter.Router
 	s         *pineconeSessions.Sessions
 	fedSender api.FederationInternalAPI
-	fedClient *gomatrixserverlib.FederationClient
+	fedClient *fclient.FederationClient
 }
 
 func NewPineconeRoomProvider(
 	r *pineconeRouter.Router,
 	s *pineconeSessions.Sessions,
 	fedSender api.FederationInternalAPI,
-	fedClient *gomatrixserverlib.FederationClient,
+	fedClient *fclient.FederationClient,
 ) *PineconeRoomProvider {
 	p := &PineconeRoomProvider{
 		r:         r,
@@ -50,31 +51,31 @@ func NewPineconeRoomProvider(
 	return p
 }
 
-func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
+func (p *PineconeRoomProvider) Rooms() []fclient.PublicRoom {
-	list := map[gomatrixserverlib.ServerName]struct{}{}
+	list := map[spec.ServerName]struct{}{}
 	for k := range defaults.DefaultServerNames {
 		list[k] = struct{}{}
 	}
 	for _, k := range p.r.Peers() {
-		list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
+		list[spec.ServerName(k.PublicKey)] = struct{}{}
 	}
 	return bulkFetchPublicRoomsFromServers(
 		context.Background(), p.fedClient,
-		gomatrixserverlib.ServerName(p.r.PublicKey().String()), list,
+		spec.ServerName(p.r.PublicKey().String()), list,
 	)
 }
 
 // bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
 // Returns a list of public rooms.
 func bulkFetchPublicRoomsFromServers(
-	ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
+	ctx context.Context, fedClient *fclient.FederationClient,
-	origin gomatrixserverlib.ServerName,
+	origin spec.ServerName,
-	homeservers map[gomatrixserverlib.ServerName]struct{},
+	homeservers map[spec.ServerName]struct{},
-) (publicRooms []gomatrixserverlib.PublicRoom) {
+) (publicRooms []fclient.PublicRoom) {
 	limit := 200
 	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
 	// goroutines send rooms to this channel
-	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
+	roomCh := make(chan fclient.PublicRoom, int(limit))
 	// signalling channel to tell goroutines to stop sending rooms and quit
 	done := make(chan bool)
 	// signalling to say when we can close the room channel
@@ -83,7 +84,7 @@ func bulkFetchPublicRoomsFromServers(
 	// concurrently query for public rooms
 	reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5)
 	for hs := range homeservers {
-		go func(homeserverDomain gomatrixserverlib.ServerName) {
+		go func(homeserverDomain spec.ServerName) {
 			defer wg.Done()
 			util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
 			fres, err := fedClient.GetPublicRooms(reqctx, origin, homeserverDomain, int(limit), "", false, "")

cmd/dendrite-demo-pinecone/users/users.go

@@ -27,7 +27,8 @@ import (
 	clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 
 	pineconeRouter "github.com/matrix-org/pinecone/router"
@@ -38,7 +39,7 @@ type PineconeUserProvider struct {
 	r         *pineconeRouter.Router
 	s         *pineconeSessions.Sessions
 	userAPI   userapi.QuerySearchProfilesAPI
-	fedClient *gomatrixserverlib.FederationClient
+	fedClient *fclient.FederationClient
 }
 
 const PublicURL = "/_matrix/p2p/profiles"
@@ -47,7 +48,7 @@ func NewPineconeUserProvider(
 	r *pineconeRouter.Router,
 	s *pineconeSessions.Sessions,
 	userAPI userapi.QuerySearchProfilesAPI,
-	fedClient *gomatrixserverlib.FederationClient,
+	fedClient *fclient.FederationClient,
 ) *PineconeUserProvider {
 	p := &PineconeUserProvider{
 		r:         r,
@@ -79,12 +80,12 @@ func (p *PineconeUserProvider) FederatedUserProfiles(w http.ResponseWriter, r *h
 }
 
 func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error {
-	list := map[gomatrixserverlib.ServerName]struct{}{}
+	list := map[spec.ServerName]struct{}{}
 	for k := range defaults.DefaultServerNames {
 		list[k] = struct{}{}
 	}
 	for _, k := range p.r.Peers() {
-		list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
+		list[spec.ServerName(k.PublicKey)] = struct{}{}
 	}
 	res.Profiles = bulkFetchUserDirectoriesFromServers(context.Background(), req, p.fedClient, list)
 	return nil
@@ -94,8 +95,8 @@ func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *use
 // Returns a list of user profiles.
 func bulkFetchUserDirectoriesFromServers(
 	ctx context.Context, req *userapi.QuerySearchProfilesRequest,
-	fedClient *gomatrixserverlib.FederationClient,
+	fedClient *fclient.FederationClient,
-	homeservers map[gomatrixserverlib.ServerName]struct{},
+	homeservers map[spec.ServerName]struct{},
 ) (profiles []authtypes.Profile) {
 	jsonBody, err := json.Marshal(req)
 	if err != nil {
@@ -114,7 +115,7 @@ func bulkFetchUserDirectoriesFromServers(
 	// concurrently query for public rooms
 	reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5)
 	for hs := range homeservers {
-		go func(homeserverDomain gomatrixserverlib.ServerName) {
+		go func(homeserverDomain spec.ServerName) {
 			defer wg.Done()
 			util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for users")
 

cmd/dendrite-demo-yggdrasil/main.go

@@ -27,7 +27,13 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
@@ -41,7 +47,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/test"
@@ -57,6 +63,7 @@ var (
 	instanceDir    = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
 )
 
+// nolint: gocyclo
 func main() {
 	flag.Parse()
 	internal.SetupPprof()
@@ -139,40 +146,86 @@ func main() {
 		}
 	}
 
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
-	base := base.NewBaseDendrite(cfg)
+	configErrors := &config.ConfigErrors{}
-	base.ConfigureAdminEndpoints()
+	cfg.Verify(configErrors)
-	defer base.Close() // nolint: errcheck
+	if len(*configErrors) > 0 {
+		for _, err := range *configErrors {
+			logrus.Errorf("Configuration error: %s", err)
+		}
+		logrus.Fatalf("Failed to start due to configuration errors")
+	}
+
+	internal.SetupStdLogging()
+	internal.SetupHookLogging(cfg.Logging)
+	internal.SetupPprof()
+
+	logrus.Infof("Dendrite version %s", internal.VersionString())
+
+	if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled {
+		logrus.Warn("Open registration is enabled")
+	}
+
+	closer, err := cfg.SetupTracing()
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to start opentracing")
+	}
+	defer closer.Close() // nolint: errcheck
+
+	if cfg.Global.Sentry.Enabled {
+		logrus.Info("Setting up Sentry for debugging...")
+		err = sentry.Init(sentry.ClientOptions{
+			Dsn:              cfg.Global.Sentry.DSN,
+			Environment:      cfg.Global.Sentry.Environment,
+			Debug:            true,
+			ServerName:       string(cfg.Global.ServerName),
+			Release:          "dendrite@" + internal.VersionString(),
+			AttachStacktrace: true,
+		})
+		if err != nil {
+			logrus.WithError(err).Panic("failed to start Sentry")
+		}
+	}
+
+	processCtx := process.NewProcessContext()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+
+	basepkg.ConfigureAdminEndpoints(processCtx, routers)
+	defer func() {
+		processCtx.ShutdownDendrite()
+		processCtx.WaitForShutdown()
+	}() // nolint: errcheck
 
 	ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
 	if err != nil {
 		panic(err)
 	}
 
-	federation := ygg.CreateFederationClient(base)
+	federation := ygg.CreateFederationClient(cfg)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	rsAPI := roomserver.NewInternalAPI(
+	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
-		base,
+	natsInstance := jetstream.NATSInstance{}
-	)
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 
-	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
 
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, keyRing, true,
+		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{
-		Config:    base.Cfg,
+		Config:    cfg,
-		Client:    ygg.CreateClient(base),
+		Client:    ygg.CreateClient(),
 		FedClient: federation,
 		KeyRing:   keyRing,
 
@@ -184,21 +237,21 @@ func main() {
 			ygg, fsAPI, federation,
 		),
 	}
-	monolith.AddAllPublicRoutes(base)
+	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
-	if err := mscs.Enable(base, &monolith); err != nil {
+	if err := mscs.Enable(cfg, cm, routers, &monolith, caches); err != nil {
 		logrus.WithError(err).Fatalf("Failed to enable MSCs")
 	}
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
-	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin)
-	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin)
 	embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
 
 	yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
+	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(routers.Federation)
-	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
 
 	// Build both ends of a HTTP multiplex.
 	httpServer := &http.Server{
@@ -232,5 +285,5 @@ func main() {
 	}()
 
 	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
-	base.WaitForShutdown()
+	basepkg.WaitForShutdown(processCtx)
 }

cmd/dendrite-demo-yggdrasil/signing/fetcher.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 const KeyID = "ed25519:dendrite-demo-yggdrasil"
@@ -36,7 +37,7 @@ func (f *YggdrasilKeys) KeyRing() *gomatrixserverlib.KeyRing {
 
 func (f *YggdrasilKeys) FetchKeys(
 	ctx context.Context,
-	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
+	requests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp,
 ) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) {
 	res := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult)
 	for req := range requests {
@@ -54,7 +55,7 @@ func (f *YggdrasilKeys) FetchKeys(
 				Key: hexkey,
 			},
 			ExpiredTS:    gomatrixserverlib.PublicKeyNotExpired,
-			ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(24 * time.Hour * 365)),
+			ValidUntilTS: spec.AsTimestamp(time.Now().Add(24 * time.Hour * 365)),
 		}
 	}
 	return res, nil

cmd/dendrite-demo-yggdrasil/yggconn/client.go

@@ -4,8 +4,8 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 )
 
 type yggroundtripper struct {
@@ -17,9 +17,7 @@ func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) {
 	return y.inner.RoundTrip(req)
 }
 
-func (n *Node) CreateClient(
+func (n *Node) CreateClient() *fclient.Client {
-	base *base.BaseDendrite,
-) *gomatrixserverlib.Client {
 	tr := &http.Transport{}
 	tr.RegisterProtocol(
 		"matrix", &yggroundtripper{
@@ -33,14 +31,14 @@ func (n *Node) CreateClient(
 			},
 		},
 	)
-	return gomatrixserverlib.NewClient(
+	return fclient.NewClient(
-		gomatrixserverlib.WithTransport(tr),
+		fclient.WithTransport(tr),
 	)
 }
 
 func (n *Node) CreateFederationClient(
-	base *base.BaseDendrite,
+	cfg *config.Dendrite,
-) *gomatrixserverlib.FederationClient {
+) *fclient.FederationClient {
 	tr := &http.Transport{}
 	tr.RegisterProtocol(
 		"matrix", &yggroundtripper{
@@ -54,8 +52,8 @@ func (n *Node) CreateFederationClient(
 			},
 		},
 	)
-	return gomatrixserverlib.NewFederationClient(
+	return fclient.NewFederationClient(
-		base.Cfg.Global.SigningIdentities(),
+		cfg.Global.SigningIdentities(),
-		gomatrixserverlib.WithTransport(tr),
+		fclient.WithTransport(tr),
 	)
 }

cmd/dendrite-demo-yggdrasil/yggconn/node.go

@@ -23,7 +23,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/neilalexander/utp"
 	"github.com/sirupsen/logrus"
 
@@ -134,14 +134,14 @@ func (n *Node) PeerCount() int {
 	return len(n.core.GetPeers())
 }
 
-func (n *Node) KnownNodes() []gomatrixserverlib.ServerName {
+func (n *Node) KnownNodes() []spec.ServerName {
 	nodemap := map[string]struct{}{}
 	for _, peer := range n.core.GetPeers() {
 		nodemap[hex.EncodeToString(peer.Key)] = struct{}{}
 	}
-	var nodes []gomatrixserverlib.ServerName
+	var nodes []spec.ServerName
 	for node := range nodemap {
-		nodes = append(nodes, gomatrixserverlib.ServerName(node))
+		nodes = append(nodes, spec.ServerName(node))
 	}
 	return nodes
 }

cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go

@@ -21,18 +21,19 @@ import (
 
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
 	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 )
 
 type YggdrasilRoomProvider struct {
 	node      *yggconn.Node
 	fedSender api.FederationInternalAPI
-	fedClient *gomatrixserverlib.FederationClient
+	fedClient *fclient.FederationClient
 }
 
 func NewYggdrasilRoomProvider(
-	node *yggconn.Node, fedSender api.FederationInternalAPI, fedClient *gomatrixserverlib.FederationClient,
+	node *yggconn.Node, fedSender api.FederationInternalAPI, fedClient *fclient.FederationClient,
 ) *YggdrasilRoomProvider {
 	p := &YggdrasilRoomProvider{
 		node:      node,
@@ -42,10 +43,10 @@ func NewYggdrasilRoomProvider(
 	return p
 }
 
-func (p *YggdrasilRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
+func (p *YggdrasilRoomProvider) Rooms() []fclient.PublicRoom {
 	return bulkFetchPublicRoomsFromServers(
 		context.Background(), p.fedClient,
-		gomatrixserverlib.ServerName(p.node.DerivedServerName()),
+		spec.ServerName(p.node.DerivedServerName()),
 		p.node.KnownNodes(),
 	)
 }
@@ -53,14 +54,14 @@ func (p *YggdrasilRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
 // bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
 // Returns a list of public rooms.
 func bulkFetchPublicRoomsFromServers(
-	ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
+	ctx context.Context, fedClient *fclient.FederationClient,
-	origin gomatrixserverlib.ServerName,
+	origin spec.ServerName,
-	homeservers []gomatrixserverlib.ServerName,
+	homeservers []spec.ServerName,
-) (publicRooms []gomatrixserverlib.PublicRoom) {
+) (publicRooms []fclient.PublicRoom) {
 	limit := 200
 	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
 	// goroutines send rooms to this channel
-	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
+	roomCh := make(chan fclient.PublicRoom, int(limit))
 	// signalling channel to tell goroutines to stop sending rooms and quit
 	done := make(chan bool)
 	// signalling to say when we can close the room channel
@@ -68,7 +69,7 @@ func bulkFetchPublicRoomsFromServers(
 	wg.Add(len(homeservers))
 	// concurrently query for public rooms
 	for _, hs := range homeservers {
-		go func(homeserverDomain gomatrixserverlib.ServerName) {
+		go func(homeserverDomain spec.ServerName) {
 			defer wg.Done()
 			util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
 			fres, err := fedClient.GetPublicRooms(ctx, origin, homeserverDomain, int(limit), "", false, "")

cmd/dendrite-upgrade-tests/main.go

@@ -259,10 +259,20 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir
 
 func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Version, err error) {
 	u := "https://api.github.com/repos/matrix-org/dendrite/tags"
-	res, err := httpClient.Get(u)
+
-	if err != nil {
+	var res *http.Response
-		return nil, err
+	for i := 0; i < 3; i++ {
+		res, err = httpClient.Get(u)
+		if err != nil {
+			return nil, err
+		}
+		if res.StatusCode == 200 {
+			break
+		}
+		log.Printf("Github API returned HTTP %d, retrying\n", res.StatusCode)
+		time.Sleep(time.Second * 5)
 	}
+
 	if res.StatusCode != 200 {
 		return nil, fmt.Errorf("%s returned HTTP %d", u, res.StatusCode)
 	}

cmd/dendrite/main.go

@@ -16,8 +16,16 @@ package main
 
 import (
 	"flag"
-	"io/fs"
+	"time"
 
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/appservice"
@@ -34,8 +42,8 @@ var (
 	unixSocket = flag.String("unix-socket", "",
 		"EXPERIMENTAL(unstable): The HTTP listening unix socket for the server (disables http[s]-bind-address feature)",
 	)
-	unixSocketPermission = flag.Int("unix-socket-permission", 0755,
+	unixSocketPermission = flag.String("unix-socket-permission", "755",
-		"EXPERIMENTAL(unstable): The HTTP listening unix socket permission for the server",
+		"EXPERIMENTAL(unstable): The HTTP listening unix socket permission for the server (in chmod format like 755)",
 	)
 	httpBindAddr  = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
 	httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
@@ -59,27 +67,97 @@ func main() {
 		}
 		httpsAddr = https
 	} else {
-		httpAddr = config.UnixSocketAddress(*unixSocket, fs.FileMode(*unixSocketPermission))
+		socket, err := config.UnixSocketAddress(*unixSocket, *unixSocketPermission)
+		if err != nil {
+			logrus.WithError(err).Fatalf("Failed to parse unix socket")
+		}
+		httpAddr = socket
 	}
 
-	options := []basepkg.BaseDendriteOptions{}
+	configErrors := &config.ConfigErrors{}
+	cfg.Verify(configErrors)
+	if len(*configErrors) > 0 {
+		for _, err := range *configErrors {
+			logrus.Errorf("Configuration error: %s", err)
+		}
+		logrus.Fatalf("Failed to start due to configuration errors")
+	}
+	processCtx := process.NewProcessContext()
 
-	base := basepkg.NewBaseDendrite(cfg, options...)
+	internal.SetupStdLogging()
-	defer base.Close() // nolint: errcheck
+	internal.SetupHookLogging(cfg.Logging)
+	internal.SetupPprof()
 
-	federation := base.CreateFederationClient()
+	basepkg.PlatformSanityChecks()
 
-	rsAPI := roomserver.NewInternalAPI(base)
+	logrus.Infof("Dendrite version %s", internal.VersionString())
+	if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled {
+		logrus.Warn("Open registration is enabled")
+	}
 
+	// create DNS cache
+	var dnsCache *fclient.DNSCache
+	if cfg.Global.DNSCache.Enabled {
+		dnsCache = fclient.NewDNSCache(
+			cfg.Global.DNSCache.CacheSize,
+			cfg.Global.DNSCache.CacheLifetime,
+		)
+		logrus.Infof(
+			"DNS cache enabled (size %d, lifetime %s)",
+			cfg.Global.DNSCache.CacheSize,
+			cfg.Global.DNSCache.CacheLifetime,
+		)
+	}
+
+	// setup tracing
+	closer, err := cfg.SetupTracing()
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to start opentracing")
+	}
+	defer closer.Close() // nolint: errcheck
+
+	// setup sentry
+	if cfg.Global.Sentry.Enabled {
+		logrus.Info("Setting up Sentry for debugging...")
+		err = sentry.Init(sentry.ClientOptions{
+			Dsn:              cfg.Global.Sentry.DSN,
+			Environment:      cfg.Global.Sentry.Environment,
+			Debug:            true,
+			ServerName:       string(cfg.Global.ServerName),
+			Release:          "dendrite@" + internal.VersionString(),
+			AttachStacktrace: true,
+		})
+		if err != nil {
+			logrus.WithError(err).Panic("failed to start Sentry")
+		}
+		go func() {
+			processCtx.ComponentStarted()
+			<-processCtx.WaitForShutdown()
+			if !sentry.Flush(time.Second * 5) {
+				logrus.Warnf("failed to flush all Sentry events!")
+			}
+			processCtx.ComponentFinished()
+		}()
+	}
+
+	federationClient := basepkg.CreateFederationClient(cfg, dnsCache)
+	httpClient := basepkg.CreateClient(cfg, dnsCache)
+
+	// prepare required dependencies
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+
+	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
+	natsInstance := jetstream.NATSInstance{}
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, nil, false,
+		processCtx, cfg, cm, &natsInstance, federationClient, rsAPI, caches, nil, false,
 	)
 
 	keyRing := fsAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient)
-
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this
@@ -89,9 +167,9 @@ func main() {
 	rsAPI.SetUserAPI(userAPI)
 
 	monolith := setup.Monolith{
-		Config:    base.Cfg,
+		Config:    cfg,
-		Client:    base.CreateClient(),
+		Client:    httpClient,
-		FedClient: federation,
+		FedClient: federationClient,
 		KeyRing:   keyRing,
 
 		AppserviceAPI: asAPI,
@@ -101,25 +179,25 @@ func main() {
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
 	}
-	monolith.AddAllPublicRoutes(base)
+	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
 
-	if len(base.Cfg.MSCs.MSCs) > 0 {
+	if len(cfg.MSCs.MSCs) > 0 {
-		if err := mscs.Enable(base, &monolith); err != nil {
+		if err := mscs.Enable(cfg, cm, routers, &monolith, caches); err != nil {
 			logrus.WithError(err).Fatalf("Failed to enable MSCs")
 		}
 	}
 
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {
-		base.SetupAndServeHTTP(httpAddr, nil, nil)
+		basepkg.SetupAndServeHTTP(processCtx, cfg, routers, httpAddr, nil, nil)
 	}()
 	// Handle HTTPS if certificate and key are provided
 	if *unixSocket == "" && *certFile != "" && *keyFile != "" {
 		go func() {
-			base.SetupAndServeHTTP(httpsAddr, certFile, keyFile)
+			basepkg.SetupAndServeHTTP(processCtx, cfg, routers, httpsAddr, certFile, keyFile)
 		}()
 	}
 
 	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
-	base.WaitForShutdown()
+	basepkg.WaitForShutdown(processCtx)
 }

cmd/furl/main.go

@@ -13,6 +13,8 @@ import (
 	"os"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 var requestFrom = flag.String("from", "", "the server name that the request should originate from")
@@ -48,9 +50,9 @@ func main() {
 		panic("unexpected key block")
 	}
 
-	serverName := gomatrixserverlib.ServerName(*requestFrom)
+	serverName := spec.ServerName(*requestFrom)
-	client := gomatrixserverlib.NewFederationClient(
+	client := fclient.NewFederationClient(
-		[]*gomatrixserverlib.SigningIdentity{
+		[]*fclient.SigningIdentity{
 			{
 				ServerName: serverName,
 				KeyID:      gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]),
@@ -82,10 +84,10 @@ func main() {
 		}
 	}
 
-	req := gomatrixserverlib.NewFederationRequest(
+	req := fclient.NewFederationRequest(
 		method,
 		serverName,
-		gomatrixserverlib.ServerName(u.Host),
+		spec.ServerName(u.Host),
 		u.RequestURI(),
 	)
 
@@ -96,7 +98,7 @@ func main() {
 	}
 
 	if err = req.Sign(
-		gomatrixserverlib.ServerName(*requestFrom),
+		spec.ServerName(*requestFrom),
 		gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]),
 		privateKey,
 	); err != nil {

cmd/generate-config/main.go

@@ -5,11 +5,11 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 	"gopkg.in/yaml.v2"
 
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 func main() {
@@ -30,7 +30,7 @@ func main() {
 			SingleDatabase: true,
 		})
 		if *serverName != "" {
-			cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
+			cfg.Global.ServerName = spec.ServerName(*serverName)
 		}
 		uri := config.DataSource(*dbURI)
 		if uri.IsSQLite() || uri == "" {

cmd/resolve-state/main.go

@@ -10,12 +10,13 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -40,7 +41,7 @@ func main() {
 		Level: "error",
 	})
 	cfg.ClientAPI.RegistrationDisabled = true
-	base := base.NewBaseDendrite(cfg, base.DisableMetrics)
+
 	args := flag.Args()
 
 	fmt.Println("Room version", *roomVersion)
@@ -54,8 +55,10 @@ func main() {
 
 	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
 
+	processCtx := process.NewProcessContext()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 	roomserverDB, err := storage.Open(
-		base, &cfg.RoomServer.Database,
+		processCtx.Context(), cm, &cfg.RoomServer.Database,
 		caching.NewRistrettoCache(128*1024*1024, time.Hour, true),
 	)
 	if err != nil {

dendrite-sample.yaml

@@ -95,7 +95,7 @@ global:
   # We use this information to understand how Dendrite is being used in the wild.
   report_stats:
     enabled: false
-    endpoint: https://matrix.org/report-usage-stats/push
+    endpoint: https://panopticon.matrix.org/push
 
   # Server notices allows server admins to send messages to all users on the server.
   server_notices:

docs/FAQ.md

@@ -18,7 +18,7 @@ Mostly, although there are still bugs and missing features. If you are a confide
 
 ## Is Dendrite feature-complete?
 
-No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](../README.md) at the root of the repository for more information.
+No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](https://github.com/matrix-org/dendrite/blob/main/README.md) at the root of the repository for more information.
 
 ## Why doesn't Dendrite have "x" yet?
 

docs/Gemfile.lock

@@ -14,7 +14,7 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.23.7)
+    commonmarker (0.23.9)
     concurrent-ruby (1.2.0)
     dnsruby (1.61.9)
       simpleidn (~> 0.1)
@@ -231,9 +231,9 @@ GEM
       jekyll-seo-tag (~> 2.1)
     minitest (5.17.0)
     multipart-post (2.1.1)
-    nokogiri (1.13.10-arm64-darwin)
+    nokogiri (1.14.3-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.13.10-x86_64-linux)
+    nokogiri (1.14.3-x86_64-linux)
       racc (~> 1.4)
     octokit (4.22.0)
       faraday (>= 0.9)
@@ -241,7 +241,7 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.7)
-    racc (1.6.1)
+    racc (1.6.2)
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)

docs/administration/4_adminapi.md

@@ -32,7 +32,7 @@ UPDATE userapi_accounts SET account_type = 3 WHERE localpart = '$localpart';
 
 Where `$localpart` is the username only (e.g. `alice`).
 
-## GET `/_dendrite/admin/evacuateRoom/{roomID}`
+## POST `/_dendrite/admin/evacuateRoom/{roomID}`
 
 This endpoint will instruct Dendrite to part all local users from the given `roomID`
 in the URL. It may take some time to complete. A JSON body will be returned containing
@@ -41,7 +41,7 @@ the user IDs of all affected users.
 If the room has an alias set (e.g. is published), the room's ID will not be visible in the URL, but it can
 be found as the room's "internal ID" in Element Web (Settings -> Advanced)
 
-## GET `/_dendrite/admin/evacuateUser/{userID}`
+## POST `/_dendrite/admin/evacuateUser/{userID}`
 
 This endpoint will instruct Dendrite to part the given local `userID` in the URL from
 all rooms which they are currently joined. A JSON body will be returned containing

docs/development/sytest.md

@@ -61,7 +61,6 @@ When debugging, the following Docker `run` options may also be useful:
 * `-e "DENDRITE_TRACE_HTTP=1"`: Adds HTTP tracing to server logs.
 * `-e "DENDRITE_TRACE_INTERNAL=1"`: Adds roomserver internal API tracing to
   server logs.
-* `-e "DENDRITE_TRACE_SQL=1"`: Adds tracing to all SQL statements to server logs.
 
 The docker command also supports a single positional argument for the test file to
 run, so you can run a single `.pl` file rather than the whole test suite. For example:

docs/installation/7_configuration.md

@@ -10,7 +10,7 @@ permalink: /installation/configuration
 A YAML configuration file is used to configure Dendrite. A sample configuration file is
 present in the top level of the Dendrite repository:
 
-* [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml)
+* [`dendrite-sample.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.yaml)
 
 You will need to duplicate the sample, calling it `dendrite.yaml` for example, and then
 tailor it to your installation. At a minimum, you will need to populate the following
@@ -86,9 +86,8 @@ that you chose.
 
 ### Global connection pool
 
-If you are running a monolith deployment and want to use a single connection pool to a
+If you want to use a single connection pool to a single PostgreSQL database, then you must
-single PostgreSQL database, then you must uncomment and configure the `database` section
+uncomment and configure the `database` section within the `global` section:
-within the `global` section:
 
 ```yaml
 global:
@@ -102,15 +101,15 @@ global:
 
 **You must then remove or comment out** the `database` sections from other areas of the
 configuration file, e.g. under the `app_service_api`, `federation_api`, `key_server`,
-`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks, otherwise these will
+`media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks, otherwise
-override the `global` database configuration.
+these will override the `global` database configuration.
 
 ### Per-component connections (all other configurations)
 
 If you are are using SQLite databases or separate PostgreSQL
 databases per component, then you must instead configure the `database` sections under each
 of the component blocks ,e.g. under the `app_service_api`, `federation_api`, `key_server`,
-`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks.
+`media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks.
 
 For example, with PostgreSQL:
 

federationapi/api/api.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/federationapi/types"
 )
@@ -21,9 +23,9 @@ type FederationInternalAPI interface {
 	P2PFederationAPI
 
 	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
-	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
+	LookupServerKeys(ctx context.Context, s spec.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
-	MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
+	MSC2836EventRelationships(ctx context.Context, origin, dst spec.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res fclient.MSC2836EventRelationshipsResponse, err error)
-	MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
+	MSC2946Spaces(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error)
 
 	// Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos.
 	PerformBroadcastEDU(
@@ -66,9 +68,9 @@ type RoomserverFederationAPI interface {
 	// containing only the server names (without information for membership events).
 	// The response will include this server if they are joined to the room.
 	QueryJoinedHostServerNamesInRoom(ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse) error
-	GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
+	GetEventAuth(ctx context.Context, origin, s spec.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res fclient.RespEventAuth, err error)
-	GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
+	GetEvent(ctx context.Context, origin, s spec.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
-	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
+	LookupMissingEvents(ctx context.Context, origin, s spec.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error)
 }
 
 type P2PFederationAPI interface {
@@ -98,45 +100,45 @@ type P2PFederationAPI interface {
 // implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
 // this interface are of type FederationClientError
 type KeyserverFederationAPI interface {
-	GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
+	GetUserDevices(ctx context.Context, origin, s spec.ServerName, userID string) (res fclient.RespUserDevices, err error)
-	ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
+	ClaimKeys(ctx context.Context, origin, s spec.ServerName, oneTimeKeys map[string]map[string]string) (res fclient.RespClaimKeys, err error)
-	QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
+	QueryKeys(ctx context.Context, origin, s spec.ServerName, keys map[string][]string) (res fclient.RespQueryKeys, err error)
 }
 
 // an interface for gmsl.FederationClient - contains functions called by federationapi only.
 type FederationClient interface {
 	P2PFederationClient
 	gomatrixserverlib.KeyClient
-	SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error)
+	SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res fclient.RespSend, err error)
 
 	// Perform operations
-	LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error)
+	LookupRoomAlias(ctx context.Context, origin, s spec.ServerName, roomAlias string) (res fclient.RespDirectory, err error)
-	Peek(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error)
+	Peek(ctx context.Context, origin, s spec.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res fclient.RespPeek, err error)
-	MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error)
+	MakeJoin(ctx context.Context, origin, s spec.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res fclient.RespMakeJoin, err error)
-	SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error)
+	SendJoin(ctx context.Context, origin, s spec.ServerName, event *gomatrixserverlib.Event) (res fclient.RespSendJoin, err error)
-	MakeLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error)
+	MakeLeave(ctx context.Context, origin, s spec.ServerName, roomID, userID string) (res fclient.RespMakeLeave, err error)
-	SendLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error)
+	SendLeave(ctx context.Context, origin, s spec.ServerName, event *gomatrixserverlib.Event) (err error)
-	SendInviteV2(ctx context.Context, origin, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error)
+	SendInviteV2(ctx context.Context, origin, s spec.ServerName, request fclient.InviteV2Request) (res fclient.RespInviteV2, err error)
 
-	GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
+	GetEvent(ctx context.Context, origin, s spec.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
 
-	GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
+	GetEventAuth(ctx context.Context, origin, s spec.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res fclient.RespEventAuth, err error)
-	GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error)
+	GetUserDevices(ctx context.Context, origin, s spec.ServerName, userID string) (fclient.RespUserDevices, error)
-	ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error)
+	ClaimKeys(ctx context.Context, origin, s spec.ServerName, oneTimeKeys map[string]map[string]string) (fclient.RespClaimKeys, error)
-	QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error)
+	QueryKeys(ctx context.Context, origin, s spec.ServerName, keys map[string][]string) (fclient.RespQueryKeys, error)
-	Backfill(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
+	Backfill(ctx context.Context, origin, s spec.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
-	MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
+	MSC2836EventRelationships(ctx context.Context, origin, dst spec.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res fclient.MSC2836EventRelationshipsResponse, err error)
-	MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
+	MSC2946Spaces(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error)
 
-	ExchangeThirdPartyInvite(ctx context.Context, origin, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
+	ExchangeThirdPartyInvite(ctx context.Context, origin, s spec.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
-	LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error)
+	LookupState(ctx context.Context, origin, s spec.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespState, err error)
-	LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
+	LookupStateIDs(ctx context.Context, origin, s spec.ServerName, roomID string, eventID string) (res fclient.RespStateIDs, err error)
-	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
+	LookupMissingEvents(ctx context.Context, origin, s spec.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error)
 }
 
 type P2PFederationClient interface {
-	P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error)
+	P2PSendTransactionToRelay(ctx context.Context, u spec.UserID, t gomatrixserverlib.Transaction, forwardingServer spec.ServerName) (res fclient.EmptyResp, err error)
-	P2PGetTransactionFromRelay(ctx context.Context, u gomatrixserverlib.UserID, prev gomatrixserverlib.RelayEntry, relayServer gomatrixserverlib.ServerName) (res gomatrixserverlib.RespGetRelayTransaction, err error)
+	P2PGetTransactionFromRelay(ctx context.Context, u spec.UserID, prev fclient.RelayEntry, relayServer spec.ServerName) (res fclient.RespGetRelayTransaction, err error)
 }
 
 // FederationClientError is returned from FederationClient methods in the event of a problem.
@@ -152,7 +154,7 @@ func (e FederationClientError) Error() string {
 }
 
 type QueryServerKeysRequest struct {
-	ServerName      gomatrixserverlib.ServerName
+	ServerName      spec.ServerName
 	KeyIDToCriteria map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyNotaryQueryCriteria
 }
 
@@ -171,7 +173,7 @@ type QueryServerKeysResponse struct {
 }
 
 type QueryPublicKeysRequest struct {
-	Requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp `json:"requests"`
+	Requests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp `json:"requests"`
 }
 
 type QueryPublicKeysResponse struct {
@@ -179,13 +181,13 @@ type QueryPublicKeysResponse struct {
 }
 
 type PerformDirectoryLookupRequest struct {
-	RoomAlias  string                       `json:"room_alias"`
+	RoomAlias  string          `json:"room_alias"`
-	ServerName gomatrixserverlib.ServerName `json:"server_name"`
+	ServerName spec.ServerName `json:"server_name"`
 }
 
 type PerformDirectoryLookupResponse struct {
-	RoomID      string                         `json:"room_id"`
+	RoomID      string            `json:"room_id"`
-	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
+	ServerNames []spec.ServerName `json:"server_names"`
 }
 
 type PerformJoinRequest struct {
@@ -198,7 +200,7 @@ type PerformJoinRequest struct {
 }
 
 type PerformJoinResponse struct {
-	JoinedVia gomatrixserverlib.ServerName
+	JoinedVia spec.ServerName
 	LastError *gomatrix.HTTPError
 }
 
@@ -222,9 +224,9 @@ type PerformLeaveResponse struct {
 }
 
 type PerformInviteRequest struct {
-	RoomVersion     gomatrixserverlib.RoomVersion             `json:"room_version"`
+	RoomVersion     gomatrixserverlib.RoomVersion    `json:"room_version"`
-	Event           *gomatrixserverlib.HeaderedEvent          `json:"event"`
+	Event           *gomatrixserverlib.HeaderedEvent `json:"event"`
-	InviteRoomState []gomatrixserverlib.InviteV2StrippedState `json:"invite_room_state"`
+	InviteRoomState []fclient.InviteV2StrippedState  `json:"invite_room_state"`
 }
 
 type PerformInviteResponse struct {
@@ -240,7 +242,7 @@ type QueryJoinedHostServerNamesInRoomRequest struct {
 
 // QueryJoinedHostServerNamesInRoomResponse is a response to QueryJoinedHostServerNames
 type QueryJoinedHostServerNamesInRoomResponse struct {
-	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
+	ServerNames []spec.ServerName `json:"server_names"`
 }
 
 type PerformBroadcastEDURequest struct {
@@ -250,7 +252,7 @@ type PerformBroadcastEDUResponse struct {
 }
 
 type PerformWakeupServersRequest struct {
-	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
+	ServerNames []spec.ServerName `json:"server_names"`
 }
 
 type PerformWakeupServersResponse struct {
@@ -264,24 +266,24 @@ type InputPublicKeysResponse struct {
 }
 
 type P2PQueryRelayServersRequest struct {
-	Server gomatrixserverlib.ServerName
+	Server spec.ServerName
 }
 
 type P2PQueryRelayServersResponse struct {
-	RelayServers []gomatrixserverlib.ServerName
+	RelayServers []spec.ServerName
 }
 
 type P2PAddRelayServersRequest struct {
-	Server       gomatrixserverlib.ServerName
+	Server       spec.ServerName
-	RelayServers []gomatrixserverlib.ServerName
+	RelayServers []spec.ServerName
 }
 
 type P2PAddRelayServersResponse struct {
 }
 
 type P2PRemoveRelayServersRequest struct {
-	Server       gomatrixserverlib.ServerName
+	Server       spec.ServerName
-	RelayServers []gomatrixserverlib.ServerName
+	RelayServers []spec.ServerName
 }
 
 type P2PRemoveRelayServersResponse struct {

federationapi/api/servers.go

@@ -4,8 +4,9 @@ import (
 	"context"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 type ServersInRoomProvider interface {
-	GetServersForRoom(ctx context.Context, roomID string, event *gomatrixserverlib.Event) []gomatrixserverlib.ServerName
+	GetServersForRoom(ctx context.Context, roomID string, event *gomatrixserverlib.Event) []spec.ServerName
 }

federationapi/consumers/keychange.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 	"github.com/sirupsen/logrus"
 
@@ -40,7 +41,7 @@ type KeyChangeConsumer struct {
 	durable           string
 	db                storage.Database
 	queues            *queue.OutgoingQueues
-	isLocalServerName func(gomatrixserverlib.ServerName) bool
+	isLocalServerName func(spec.ServerName) bool
 	rsAPI             roomserverAPI.FederationRoomserverAPI
 	topic             string
 }
@@ -140,7 +141,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 	}
 	// Pack the EDU and marshal it
 	edu := &gomatrixserverlib.EDU{
-		Type:   gomatrixserverlib.MDeviceListUpdate,
+		Type:   spec.MDeviceListUpdate,
 		Origin: string(originServerName),
 	}
 	event := gomatrixserverlib.DeviceListUpdateEvent{

federationapi/consumers/presence.go

@@ -28,6 +28,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 )
@@ -39,7 +40,7 @@ type OutputPresenceConsumer struct {
 	durable                 string
 	db                      storage.Database
 	queues                  *queue.OutgoingQueues
-	isLocalServerName       func(gomatrixserverlib.ServerName) bool
+	isLocalServerName       func(spec.ServerName) bool
 	rsAPI                   roomserverAPI.FederationRoomserverAPI
 	topic                   string
 	outboundPresenceEnabled bool
@@ -127,7 +128,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
 		statusMsg = &status
 	}
 
-	p := types.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(ts)}
+	p := types.PresenceInternal{LastActiveTS: spec.Timestamp(ts)}
 
 	content := fedTypes.Presence{
 		Push: []fedTypes.PresenceContent{
@@ -142,7 +143,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
 	}
 
 	edu := &gomatrixserverlib.EDU{
-		Type:   gomatrixserverlib.MPresence,
+		Type:   spec.MPresence,
 		Origin: string(serverName),
 	}
 	if edu.Content, err = json.Marshal(content); err != nil {

federationapi/consumers/receipts.go

@@ -28,6 +28,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 )
@@ -39,7 +40,7 @@ type OutputReceiptConsumer struct {
 	durable           string
 	db                storage.Database
 	queues            *queue.OutgoingQueues
-	isLocalServerName func(gomatrixserverlib.ServerName) bool
+	isLocalServerName func(spec.ServerName) bool
 	topic             string
 }
 
@@ -107,7 +108,7 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg)
 		return true
 	}
 
-	receipt.Timestamp = gomatrixserverlib.Timestamp(timestamp)
+	receipt.Timestamp = spec.Timestamp(timestamp)
 
 	joined, err := t.db.GetJoinedHosts(ctx, receipt.RoomID)
 	if err != nil {
@@ -115,7 +116,7 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg)
 		return false
 	}
 
-	names := make([]gomatrixserverlib.ServerName, len(joined))
+	names := make([]spec.ServerName, len(joined))
 	for i := range joined {
 		names[i] = joined[i].ServerName
 	}
@@ -133,7 +134,7 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg)
 	}
 
 	edu := &gomatrixserverlib.EDU{
-		Type:   gomatrixserverlib.MReceipt,
+		Type:   spec.MReceipt,
 		Origin: string(receiptServerName),
 	}
 	if edu.Content, err = json.Marshal(content); err != nil {

federationapi/consumers/roomserver.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	syncAPITypes "github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
@@ -207,9 +208,9 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	}
 
 	// If we added new hosts, inform them about our known presence events for this room
-	if s.cfg.Matrix.Presence.EnableOutbound && len(addsJoinedHosts) > 0 && ore.Event.Type() == gomatrixserverlib.MRoomMember && ore.Event.StateKey() != nil {
+	if s.cfg.Matrix.Presence.EnableOutbound && len(addsJoinedHosts) > 0 && ore.Event.Type() == spec.MRoomMember && ore.Event.StateKey() != nil {
 		membership, _ := ore.Event.Membership()
-		if membership == gomatrixserverlib.Join {
+		if membership == spec.Join {
 			s.sendPresence(ore.Event.RoomID(), addsJoinedHosts)
 		}
 	}
@@ -239,12 +240,12 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 
 	// Send the event.
 	return s.queues.SendEvent(
-		ore.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent,
+		ore.Event, spec.ServerName(ore.SendAsServer), joinedHostsAtEvent,
 	)
 }
 
 func (s *OutputRoomEventConsumer) sendPresence(roomID string, addedJoined []types.JoinedHost) {
-	joined := make([]gomatrixserverlib.ServerName, 0, len(addedJoined))
+	joined := make([]spec.ServerName, 0, len(addedJoined))
 	for _, added := range addedJoined {
 		joined = append(joined, added.ServerName)
 	}
@@ -285,7 +286,7 @@ func (s *OutputRoomEventConsumer) sendPresence(roomID string, addedJoined []type
 			continue
 		}
 
-		p := syncAPITypes.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(lastActive)}
+		p := syncAPITypes.PresenceInternal{LastActiveTS: spec.Timestamp(lastActive)}
 
 		content.Push = append(content.Push, types.PresenceContent{
 			CurrentlyActive: p.CurrentlyActive(),
@@ -301,7 +302,7 @@ func (s *OutputRoomEventConsumer) sendPresence(roomID string, addedJoined []type
 	}
 
 	edu := &gomatrixserverlib.EDU{
-		Type:   gomatrixserverlib.MPresence,
+		Type:   spec.MPresence,
 		Origin: string(s.cfg.Matrix.ServerName),
 	}
 	if edu.Content, err = json.Marshal(content); err != nil {
@@ -326,7 +327,7 @@ func (s *OutputRoomEventConsumer) sendPresence(roomID string, addedJoined []type
 // Returns an error if there was a problem talking to the room server.
 func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 	ore api.OutputNewRoomEvent, oldJoinedHosts []types.JoinedHost,
-) ([]gomatrixserverlib.ServerName, error) {
+) ([]spec.ServerName, error) {
 	// Combine the delta into a single delta so that the adds and removes can
 	// cancel each other out. This should reduce the number of times we need
 	// to fetch a state event from the room server.
@@ -349,7 +350,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 		removed[eventID] = true
 	}
 
-	joined := map[gomatrixserverlib.ServerName]bool{}
+	joined := map[spec.ServerName]bool{}
 	for _, joinedHost := range oldJoinedHosts {
 		if removed[joinedHost.MemberEventID] {
 			// This m.room.member event is part of the current state of the
@@ -376,7 +377,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 		joined[inboundPeek.ServerName] = true
 	}
 
-	var result []gomatrixserverlib.ServerName
+	var result []spec.ServerName
 	for serverName, include := range joined {
 		if include {
 			result = append(result, serverName)
@@ -398,7 +399,7 @@ func JoinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost,
 		if err != nil {
 			return nil, err
 		}
-		if membership != gomatrixserverlib.Join {
+		if membership != spec.Join {
 			continue
 		}
 		_, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey())

federationapi/consumers/sendtodevice.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
@@ -39,7 +40,7 @@ type OutputSendToDeviceConsumer struct {
 	durable           string
 	db                storage.Database
 	queues            *queue.OutgoingQueues
-	isLocalServerName func(gomatrixserverlib.ServerName) bool
+	isLocalServerName func(spec.ServerName) bool
 	topic             string
 }
 
@@ -107,7 +108,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
 
 	// Pack the EDU and marshal it
 	edu := &gomatrixserverlib.EDU{
-		Type:   gomatrixserverlib.MDirectToDevice,
+		Type:   spec.MDirectToDevice,
 		Origin: string(originServerName),
 	}
 	tdm := gomatrixserverlib.ToDeviceMessage{
@@ -127,7 +128,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
 	}
 
 	log.Debugf("Sending send-to-device message into %q destination queue", destServerName)
-	if err := t.queues.SendEDU(edu, originServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil {
+	if err := t.queues.SendEDU(edu, originServerName, []spec.ServerName{destServerName}); err != nil {
 		log.WithError(err).Error("failed to send EDU")
 		return false
 	}

federationapi/consumers/typing.go

@@ -25,6 +25,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 )
@@ -36,7 +37,7 @@ type OutputTypingConsumer struct {
 	durable           string
 	db                storage.Database
 	queues            *queue.OutgoingQueues
-	isLocalServerName func(gomatrixserverlib.ServerName) bool
+	isLocalServerName func(spec.ServerName) bool
 	topic             string
 }
 
@@ -97,7 +98,7 @@ func (t *OutputTypingConsumer) onMessage(ctx context.Context, msgs []*nats.Msg)
 		return false
 	}
 
-	names := make([]gomatrixserverlib.ServerName, len(joined))
+	names := make([]spec.ServerName, len(joined))
 	for i := range joined {
 		names[i] = joined[i].ServerName
 	}

federationapi/federationapi.go

@@ -17,6 +17,11 @@ package federationapi
 import (
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
@@ -29,7 +34,6 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/internal/caching"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 
@@ -40,17 +44,21 @@ import (
 
 // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.
 func AddPublicRoutes(
-	base *base.BaseDendrite,
+	processContext *process.ProcessContext,
+	routers httputil.Routers,
+	dendriteConfig *config.Dendrite,
+	natsInstance *jetstream.NATSInstance,
 	userAPI userapi.FederationUserAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	fedAPI federationAPI.FederationInternalAPI,
 	servers federationAPI.ServersInRoomProvider,
+	enableMetrics bool,
 ) {
-	cfg := &base.Cfg.FederationAPI
+	cfg := &dendriteConfig.FederationAPI
-	mscCfg := &base.Cfg.MSCs
+	mscCfg := &dendriteConfig.MSCs
-	js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, _ := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream)
 	producer := &producers.SyncAPIProducer{
 		JetStream:              js,
 		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
@@ -75,26 +83,30 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		base,
+		routers,
+		dendriteConfig,
 		rsAPI, f, keyRing,
 		federation, userAPI, mscCfg,
-		servers, producer,
+		servers, producer, enableMetrics,
 	)
 }
 
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *base.BaseDendrite,
+	processContext *process.ProcessContext,
+	dendriteCfg *config.Dendrite,
+	cm sqlutil.Connections,
+	natsInstance *jetstream.NATSInstance,
 	federation api.FederationClient,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	caches *caching.Caches,
 	keyRing *gomatrixserverlib.KeyRing,
 	resetBlacklist bool,
 ) api.FederationInternalAPI {
-	cfg := &base.Cfg.FederationAPI
+	cfg := &dendriteCfg.FederationAPI
 
-	federationDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.IsLocalServerName)
+	federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName)
 	if err != nil {
 		logrus.WithError(err).Panic("failed to connect to federation sender db")
 	}
@@ -108,51 +120,51 @@ func NewInternalAPI(
 		cfg.FederationMaxRetries+1,
 		cfg.P2PFederationRetriesUntilAssumedOffline+1)
 
-	js, nats := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, nats := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream)
 
-	signingInfo := base.Cfg.Global.SigningIdentities()
+	signingInfo := dendriteCfg.Global.SigningIdentities()
 
 	queues := queue.NewOutgoingQueues(
-		federationDB, base.ProcessContext,
+		federationDB, processContext,
 		cfg.Matrix.DisableFederation,
 		cfg.Matrix.ServerName, federation, rsAPI, &stats,
 		signingInfo,
 	)
 
 	rsConsumer := consumers.NewOutputRoomEventConsumer(
-		base.ProcessContext, cfg, js, nats, queues,
+		processContext, cfg, js, nats, queues,
 		federationDB, rsAPI,
 	)
 	if err = rsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start room server consumer")
 	}
 	tsConsumer := consumers.NewOutputSendToDeviceConsumer(
-		base.ProcessContext, cfg, js, queues, federationDB,
+		processContext, cfg, js, queues, federationDB,
 	)
 	if err = tsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start send-to-device consumer")
 	}
 	receiptConsumer := consumers.NewOutputReceiptConsumer(
-		base.ProcessContext, cfg, js, queues, federationDB,
+		processContext, cfg, js, queues, federationDB,
 	)
 	if err = receiptConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start receipt consumer")
 	}
 	typingConsumer := consumers.NewOutputTypingConsumer(
-		base.ProcessContext, cfg, js, queues, federationDB,
+		processContext, cfg, js, queues, federationDB,
 	)
 	if err = typingConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start typing consumer")
 	}
 	keyConsumer := consumers.NewKeyChangeConsumer(
-		base.ProcessContext, &base.Cfg.KeyServer, js, queues, federationDB, rsAPI,
+		processContext, &dendriteCfg.KeyServer, js, queues, federationDB, rsAPI,
 	)
 	if err = keyConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start key server consumer")
 	}
 
 	presenceConsumer := consumers.NewOutputPresenceConsumer(
-		base.ProcessContext, cfg, js, queues, federationDB, rsAPI,
+		processContext, cfg, js, queues, federationDB, rsAPI,
 	)
 	if err = presenceConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start presence consumer")
@@ -161,7 +173,7 @@ func NewInternalAPI(
 	var cleanExpiredEDUs func()
 	cleanExpiredEDUs = func() {
 		logrus.Infof("Cleaning expired EDUs")
-		if err := federationDB.DeleteExpiredEDUs(base.Context()); err != nil {
+		if err := federationDB.DeleteExpiredEDUs(processContext.Context()); err != nil {
 			logrus.WithError(err).Error("Failed to clean expired EDUs")
 		}
 		time.AfterFunc(time.Hour, cleanExpiredEDUs)

federationapi/federationapi_keys_test.go

@@ -12,22 +12,26 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 type server struct {
-	name      gomatrixserverlib.ServerName        // server name
+	name      spec.ServerName           // server name
-	validity  time.Duration                       // key validity duration from now
+	validity  time.Duration             // key validity duration from now
-	config    *config.FederationAPI               // skeleton config, from TestMain
+	config    *config.FederationAPI     // skeleton config, from TestMain
-	fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper
+	fedclient *fclient.FederationClient // uses MockRoundTripper
-	cache     *caching.Caches                     // server-specific cache
+	cache     *caching.Caches           // server-specific cache
-	api       api.FederationInternalAPI           // server-specific server key API
+	api       api.FederationInternalAPI // server-specific server key API
 }
 
 func (s *server) renew() {
@@ -65,7 +69,7 @@ func TestMain(m *testing.M) {
 
 			// Create a new cache but don't enable prometheus!
 			s.cache = caching.NewRistrettoCache(8*1024*1024, time.Hour, false)
-
+			natsInstance := jetstream.NATSInstance{}
 			// Create a temporary directory for JetStream.
 			d, err := os.MkdirTemp("./", "jetstream*")
 			if err != nil {
@@ -80,7 +84,7 @@ func TestMain(m *testing.M) {
 				Generate:       true,
 				SingleDatabase: false,
 			})
-			cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
+			cfg.Global.ServerName = spec.ServerName(s.name)
 			cfg.Global.PrivateKey = testPriv
 			cfg.Global.JetStream.InMemory = true
 			cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
@@ -103,14 +107,15 @@ func TestMain(m *testing.M) {
 			transport.RegisterProtocol("matrix", &MockRoundTripper{})
 
 			// Create the federation client.
-			s.fedclient = gomatrixserverlib.NewFederationClient(
+			s.fedclient = fclient.NewFederationClient(
 				s.config.Matrix.SigningIdentities(),
-				gomatrixserverlib.WithTransport(transport),
+				fclient.WithTransport(transport),
 			)
 
 			// Finally, build the server key APIs.
-			sbase := base.NewBaseDendrite(cfg, base.DisableMetrics)
+			processCtx := process.NewProcessContext()
-			s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
+			cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+			s.api = NewInternalAPI(processCtx, cfg, cm, &natsInstance, s.fedclient, nil, s.cache, nil, true)
 		}
 
 		// Now that we have built our server key APIs, start the
@@ -137,7 +142,7 @@ func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err
 	}
 
 	// Get the keys and JSON-ify them.
-	keys := routing.LocalKeys(s.config, gomatrixserverlib.ServerName(req.Host))
+	keys := routing.LocalKeys(s.config, spec.ServerName(req.Host))
 	body, err := json.MarshalIndent(keys.JSON, "", "  ")
 	if err != nil {
 		return nil, err
@@ -162,8 +167,8 @@ func TestServersRequestOwnKeys(t *testing.T) {
 		}
 		res, err := s.api.FetchKeys(
 			context.Background(),
-			map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
+			map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp{
-				req: gomatrixserverlib.AsTimestamp(time.Now()),
+				req: spec.AsTimestamp(time.Now()),
 			},
 		)
 		if err != nil {
@@ -188,8 +193,8 @@ func TestRenewalBehaviour(t *testing.T) {
 
 	res, err := serverA.api.FetchKeys(
 		context.Background(),
-		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
+		map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp{
-			req: gomatrixserverlib.AsTimestamp(time.Now()),
+			req: spec.AsTimestamp(time.Now()),
 		},
 	)
 	if err != nil {
@@ -212,8 +217,8 @@ func TestRenewalBehaviour(t *testing.T) {
 
 	res, err = serverA.api.FetchKeys(
 		context.Background(),
-		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
+		map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp{
-			req: gomatrixserverlib.AsTimestamp(time.Now()),
+			req: spec.AsTimestamp(time.Now()),
 		},
 	)
 	if err != nil {

federationapi/federationapi_test.go

@@ -10,16 +10,19 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/internal"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
@@ -54,7 +57,7 @@ type fedClient struct {
 	fedClientMutex sync.Mutex
 	api.FederationClient
 	allowJoins []*test.Room
-	keys       map[gomatrixserverlib.ServerName]struct {
+	keys       map[spec.ServerName]struct {
 		key   ed25519.PrivateKey
 		keyID gomatrixserverlib.KeyID
 	}
@@ -62,7 +65,7 @@ type fedClient struct {
 	sentTxn bool
 }
 
-func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) {
+func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer spec.ServerName) (gomatrixserverlib.ServerKeys, error) {
 	f.fedClientMutex.Lock()
 	defer f.fedClientMutex.Unlock()
 	fmt.Println("GetServerKeys:", matrixServer)
@@ -81,11 +84,11 @@ func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserv
 	}
 
 	keys.ServerName = matrixServer
-	keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(time.Now().Add(10 * time.Hour))
+	keys.ValidUntilTS = spec.AsTimestamp(time.Now().Add(10 * time.Hour))
 	publicKey := pkey.Public().(ed25519.PublicKey)
 	keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
 		keyID: {
-			Key: gomatrixserverlib.Base64Bytes(publicKey),
+			Key: spec.Base64Bytes(publicKey),
 		},
 	}
 	toSign, err := json.Marshal(keys.ServerKeyFields)
@@ -103,7 +106,7 @@ func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserv
 	return keys, nil
 }
 
-func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) {
+func (f *fedClient) MakeJoin(ctx context.Context, origin, s spec.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res fclient.RespMakeJoin, err error) {
 	for _, r := range f.allowJoins {
 		if r.ID == roomID {
 			res.RoomVersion = r.Version
@@ -112,7 +115,7 @@ func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.Se
 				RoomID:     roomID,
 				Type:       "m.room.member",
 				StateKey:   &userID,
-				Content:    gomatrixserverlib.RawJSON([]byte(`{"membership":"join"}`)),
+				Content:    spec.RawJSON([]byte(`{"membership":"join"}`)),
 				PrevEvents: r.ForwardExtremities(),
 			}
 			var needed gomatrixserverlib.StateNeeded
@@ -127,7 +130,7 @@ func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.Se
 	}
 	return
 }
-func (f *fedClient) SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
+func (f *fedClient) SendJoin(ctx context.Context, origin, s spec.ServerName, event *gomatrixserverlib.Event) (res fclient.RespSendJoin, err error) {
 	f.fedClientMutex.Lock()
 	defer f.fedClientMutex.Unlock()
 	for _, r := range f.allowJoins {
@@ -141,11 +144,11 @@ func (f *fedClient) SendJoin(ctx context.Context, origin, s gomatrixserverlib.Se
 	return
 }
 
-func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
+func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res fclient.RespSend, err error) {
 	f.fedClientMutex.Lock()
 	defer f.fedClientMutex.Unlock()
 	for _, edu := range t.EDUs {
-		if edu.Type == gomatrixserverlib.MDeviceListUpdate {
+		if edu.Type == spec.MDeviceListUpdate {
 			f.sentTxn = true
 		}
 	}
@@ -162,21 +165,24 @@ func TestFederationAPIJoinThenKeyUpdate(t *testing.T) {
 }
 
 func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
-	base, close := testrig.CreateBaseDendrite(t, dbType)
+	cfg, processCtx, close := testrig.CreateConfig(t, dbType)
-	base.Cfg.FederationAPI.PreferDirectFetch = true
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
-	base.Cfg.FederationAPI.KeyPerspectives = nil
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	natsInstance := jetstream.NATSInstance{}
+	cfg.FederationAPI.PreferDirectFetch = true
+	cfg.FederationAPI.KeyPerspectives = nil
 	defer close()
-	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
-	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
 
-	serverA := gomatrixserverlib.ServerName("server.a")
+	serverA := spec.ServerName("server.a")
 	serverAKeyID := gomatrixserverlib.KeyID("ed25519:servera")
 	serverAPrivKey := test.PrivateKeyA
 	creator := test.NewUser(t, test.WithSigningServer(serverA, serverAKeyID, serverAPrivKey))
 
-	myServer := base.Cfg.Global.ServerName
+	myServer := cfg.Global.ServerName
-	myServerKeyID := base.Cfg.Global.KeyID
+	myServerKeyID := cfg.Global.KeyID
-	myServerPrivKey := base.Cfg.Global.PrivateKey
+	myServerPrivKey := cfg.Global.PrivateKey
 	joiningUser := test.NewUser(t, test.WithSigningServer(myServer, myServerKeyID, myServerPrivKey))
 	fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID)
 	room := test.NewRoom(t, creator)
@@ -198,7 +204,7 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 	fc := &fedClient{
 		allowJoins: []*test.Room{room},
 		t:          t,
-		keys: map[gomatrixserverlib.ServerName]struct {
+		keys: map[spec.ServerName]struct {
 			key   ed25519.PrivateKey
 			keyID gomatrixserverlib.KeyID
 		}{
@@ -212,13 +218,13 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 			},
 		},
 	}
-	fsapi := federationapi.NewInternalAPI(base, fc, rsapi, base.Caches, nil, false)
+	fsapi := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, fc, rsapi, caches, nil, false)
 
 	var resp api.PerformJoinResponse
 	fsapi.PerformJoin(context.Background(), &api.PerformJoinRequest{
 		RoomID:      room.ID,
 		UserID:      joiningUser.ID,
-		ServerNames: []gomatrixserverlib.ServerName{serverA},
+		ServerNames: []spec.ServerName{serverA},
 	}, &resp)
 	if resp.JoinedVia != serverA {
 		t.Errorf("PerformJoin: joined via %v want %v", resp.JoinedVia, serverA)
@@ -245,7 +251,7 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 	}
 
 	msg := &nats.Msg{
-		Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
+		Subject: cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
 		Header:  nats.Header{},
 		Data:    b,
 	}
@@ -263,30 +269,6 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 // Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404.
 // Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated.
 func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
-	_, privKey, _ := ed25519.GenerateKey(nil)
-	cfg := &config.Dendrite{}
-	cfg.Defaults(config.DefaultOpts{
-		Generate:       true,
-		SingleDatabase: false,
-	})
-	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
-	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
-	cfg.Global.PrivateKey = privKey
-	cfg.Global.JetStream.InMemory = true
-	b := base.NewBaseDendrite(cfg, base.DisableMetrics)
-	keyRing := &test.NopJSONVerifier{}
-	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
-	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(b, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil)
-	baseURL, cancel := test.ListenAndServe(t, b.PublicFederationAPIMux, true)
-	defer cancel()
-	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
-
-	fedCli := gomatrixserverlib.NewFederationClient(
-		cfg.Global.SigningIdentities(),
-		gomatrixserverlib.WithSkipVerify(true),
-	)
-
 	testCases := []struct {
 		roomVer   gomatrixserverlib.RoomVersion
 		eventJSON string
@@ -315,13 +297,36 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 		},
 	}
 
+	cfg, processCtx, close := testrig.CreateConfig(t, test.DBTypeSQLite)
+	defer close()
+	routers := httputil.NewRouters()
+
+	_, privKey, _ := ed25519.GenerateKey(nil)
+	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
+	cfg.Global.ServerName = spec.ServerName("localhost")
+	cfg.Global.PrivateKey = privKey
+	cfg.Global.JetStream.InMemory = true
+	keyRing := &test.NopJSONVerifier{}
+	natsInstance := jetstream.NATSInstance{}
+	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
+	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
+	federationapi.AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil, caching.DisableMetrics)
+	baseURL, cancel := test.ListenAndServe(t, routers.Federation, true)
+	defer cancel()
+	serverName := spec.ServerName(strings.TrimPrefix(baseURL, "https://"))
+
+	fedCli := fclient.NewFederationClient(
+		cfg.Global.SigningIdentities(),
+		fclient.WithSkipVerify(true),
+	)
+
 	for _, tc := range testCases {
 		ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(tc.eventJSON), false, tc.roomVer)
 		if err != nil {
 			t.Errorf("failed to parse event: %s", err)
 		}
 		he := ev.Headered(tc.roomVer)
-		invReq, err := gomatrixserverlib.NewInviteV2Request(he, nil)
+		invReq, err := fclient.NewInviteV2Request(he, nil)
 		if err != nil {
 			t.Errorf("failed to create invite v2 request: %s", err)
 			continue

federationapi/internal/api.go

@@ -17,6 +17,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 )
 
@@ -107,7 +108,7 @@ func NewFederationInternalAPI(
 	}
 }
 
-func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s gomatrixserverlib.ServerName) (*statistics.ServerStatistics, error) {
+func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) {
 	stats := a.statistics.ForServer(s)
 	if stats.Blacklisted() {
 		return stats, &api.FederationClientError{
@@ -144,7 +145,7 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti
 }
 
 func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted(
-	s gomatrixserverlib.ServerName, request func() (interface{}, error),
+	s spec.ServerName, request func() (interface{}, error),
 ) (interface{}, error) {
 	stats, err := a.isBlacklistedOrBackingOff(s)
 	if err != nil {
@@ -169,7 +170,7 @@ func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted(
 }
 
 func (a *FederationInternalAPI) doRequestIfNotBlacklisted(
-	s gomatrixserverlib.ServerName, request func() (interface{}, error),
+	s spec.ServerName, request func() (interface{}, error),
 ) (interface{}, error) {
 	stats := a.statistics.ForServer(s)
 	if blacklisted := stats.Blacklisted(); blacklisted {

federationapi/internal/federationclient.go

@@ -5,68 +5,70 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 // Functions here are "proxying" calls to the gomatrixserverlib federation
 // client.
 
 func (a *FederationInternalAPI) GetEventAuth(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName,
+	ctx context.Context, origin, s spec.ServerName,
 	roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string,
-) (res gomatrixserverlib.RespEventAuth, err error) {
+) (res fclient.RespEventAuth, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.GetEventAuth(ctx, origin, s, roomVersion, roomID, eventID)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespEventAuth{}, err
+		return fclient.RespEventAuth{}, err
 	}
-	return ires.(gomatrixserverlib.RespEventAuth), nil
+	return ires.(fclient.RespEventAuth), nil
 }
 
 func (a *FederationInternalAPI) GetUserDevices(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string,
+	ctx context.Context, origin, s spec.ServerName, userID string,
-) (gomatrixserverlib.RespUserDevices, error) {
+) (fclient.RespUserDevices, error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.GetUserDevices(ctx, origin, s, userID)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespUserDevices{}, err
+		return fclient.RespUserDevices{}, err
 	}
-	return ires.(gomatrixserverlib.RespUserDevices), nil
+	return ires.(fclient.RespUserDevices), nil
 }
 
 func (a *FederationInternalAPI) ClaimKeys(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string,
+	ctx context.Context, origin, s spec.ServerName, oneTimeKeys map[string]map[string]string,
-) (gomatrixserverlib.RespClaimKeys, error) {
+) (fclient.RespClaimKeys, error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.ClaimKeys(ctx, origin, s, oneTimeKeys)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespClaimKeys{}, err
+		return fclient.RespClaimKeys{}, err
 	}
-	return ires.(gomatrixserverlib.RespClaimKeys), nil
+	return ires.(fclient.RespClaimKeys), nil
 }
 
 func (a *FederationInternalAPI) QueryKeys(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string,
+	ctx context.Context, origin, s spec.ServerName, keys map[string][]string,
-) (gomatrixserverlib.RespQueryKeys, error) {
+) (fclient.RespQueryKeys, error) {
 	ires, err := a.doRequestIfNotBackingOffOrBlacklisted(s, func() (interface{}, error) {
 		return a.federation.QueryKeys(ctx, origin, s, keys)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespQueryKeys{}, err
+		return fclient.RespQueryKeys{}, err
 	}
-	return ires.(gomatrixserverlib.RespQueryKeys), nil
+	return ires.(fclient.RespQueryKeys), nil
 }
 
 func (a *FederationInternalAPI) Backfill(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string,
+	ctx context.Context, origin, s spec.ServerName, roomID string, limit int, eventIDs []string,
 ) (res gomatrixserverlib.Transaction, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
@@ -80,50 +82,51 @@ func (a *FederationInternalAPI) Backfill(
 }
 
 func (a *FederationInternalAPI) LookupState(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
+	ctx context.Context, origin, s spec.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
-) (res gomatrixserverlib.RespState, err error) {
+) (res gomatrixserverlib.StateResponse, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.LookupState(ctx, origin, s, roomID, eventID, roomVersion)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespState{}, err
+		return &fclient.RespState{}, err
 	}
-	return ires.(gomatrixserverlib.RespState), nil
+	r := ires.(fclient.RespState)
+	return &r, nil
 }
 
 func (a *FederationInternalAPI) LookupStateIDs(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string,
+	ctx context.Context, origin, s spec.ServerName, roomID, eventID string,
-) (res gomatrixserverlib.RespStateIDs, err error) {
+) (res gomatrixserverlib.StateIDResponse, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.LookupStateIDs(ctx, origin, s, roomID, eventID)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespStateIDs{}, err
+		return fclient.RespStateIDs{}, err
 	}
-	return ires.(gomatrixserverlib.RespStateIDs), nil
+	return ires.(fclient.RespStateIDs), nil
 }
 
 func (a *FederationInternalAPI) LookupMissingEvents(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string,
+	ctx context.Context, origin, s spec.ServerName, roomID string,
-	missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion,
+	missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion,
-) (res gomatrixserverlib.RespMissingEvents, err error) {
+) (res fclient.RespMissingEvents, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.LookupMissingEvents(ctx, origin, s, roomID, missing, roomVersion)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespMissingEvents{}, err
+		return fclient.RespMissingEvents{}, err
 	}
-	return ires.(gomatrixserverlib.RespMissingEvents), nil
+	return ires.(fclient.RespMissingEvents), nil
 }
 
 func (a *FederationInternalAPI) GetEvent(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string,
+	ctx context.Context, origin, s spec.ServerName, eventID string,
 ) (res gomatrixserverlib.Transaction, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
@@ -137,7 +140,7 @@ func (a *FederationInternalAPI) GetEvent(
 }
 
 func (a *FederationInternalAPI) LookupServerKeys(
-	ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
+	ctx context.Context, s spec.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp,
 ) ([]gomatrixserverlib.ServerKeys, error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)
 	defer cancel()
@@ -151,9 +154,9 @@ func (a *FederationInternalAPI) LookupServerKeys(
 }
 
 func (a *FederationInternalAPI) MSC2836EventRelationships(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest,
+	ctx context.Context, origin, s spec.ServerName, r fclient.MSC2836EventRelationshipsRequest,
 	roomVersion gomatrixserverlib.RoomVersion,
-) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) {
+) (res fclient.MSC2836EventRelationshipsResponse, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
@@ -162,12 +165,12 @@ func (a *FederationInternalAPI) MSC2836EventRelationships(
 	if err != nil {
 		return res, err
 	}
-	return ires.(gomatrixserverlib.MSC2836EventRelationshipsResponse), nil
+	return ires.(fclient.MSC2836EventRelationshipsResponse), nil
 }
 
 func (a *FederationInternalAPI) MSC2946Spaces(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, suggestedOnly bool,
+	ctx context.Context, origin, s spec.ServerName, roomID string, suggestedOnly bool,
-) (res gomatrixserverlib.MSC2946SpacesResponse, err error) {
+) (res fclient.MSC2946SpacesResponse, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
@@ -176,5 +179,5 @@ func (a *FederationInternalAPI) MSC2946Spaces(
 	if err != nil {
 		return res, err
 	}
-	return ires.(gomatrixserverlib.MSC2946SpacesResponse), nil
+	return ires.(fclient.MSC2946SpacesResponse), nil
 }

federationapi/internal/federationclient_test.go

@@ -24,7 +24,8 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -33,20 +34,20 @@ const (
 	FailuresUntilBlacklist      = 8
 )
 
-func (t *testFedClient) QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error) {
+func (t *testFedClient) QueryKeys(ctx context.Context, origin, s spec.ServerName, keys map[string][]string) (fclient.RespQueryKeys, error) {
 	t.queryKeysCalled = true
 	if t.shouldFail {
-		return gomatrixserverlib.RespQueryKeys{}, fmt.Errorf("Failure")
+		return fclient.RespQueryKeys{}, fmt.Errorf("Failure")
 	}
-	return gomatrixserverlib.RespQueryKeys{}, nil
+	return fclient.RespQueryKeys{}, nil
 }
 
-func (t *testFedClient) ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error) {
+func (t *testFedClient) ClaimKeys(ctx context.Context, origin, s spec.ServerName, oneTimeKeys map[string]map[string]string) (fclient.RespClaimKeys, error) {
 	t.claimKeysCalled = true
 	if t.shouldFail {
-		return gomatrixserverlib.RespClaimKeys{}, fmt.Errorf("Failure")
+		return fclient.RespClaimKeys{}, fmt.Errorf("Failure")
 	}
-	return gomatrixserverlib.RespClaimKeys{}, nil
+	return fclient.RespClaimKeys{}, nil
 }
 
 func TestFederationClientQueryKeys(t *testing.T) {
@@ -54,7 +55,7 @@ func TestFederationClientQueryKeys(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},
@@ -85,7 +86,7 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},
@@ -115,7 +116,7 @@ func TestFederationClientQueryKeysFailure(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},
@@ -145,7 +146,7 @@ func TestFederationClientClaimKeys(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},
@@ -176,7 +177,7 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},

federationapi/internal/keys.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 )
 
@@ -31,14 +32,14 @@ func (s *FederationInternalAPI) StoreKeys(
 
 func (s *FederationInternalAPI) FetchKeys(
 	_ context.Context,
-	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
+	requests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp,
 ) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) {
 	// Run in a background context - we don't want to stop this work just
 	// because the caller gives up waiting.
 	ctx := context.Background()
-	now := gomatrixserverlib.AsTimestamp(time.Now())
+	now := spec.AsTimestamp(time.Now())
 	results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
-	origRequests := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{}
+	origRequests := map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp{}
 	for k, v := range requests {
 		origRequests[k] = v
 	}
@@ -95,7 +96,7 @@ func (s *FederationInternalAPI) FetcherName() string {
 // a request for our own server keys, either current or old.
 func (s *FederationInternalAPI) handleLocalKeys(
 	_ context.Context,
-	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
+	requests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp,
 	results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult,
 ) {
 	for req := range requests {
@@ -111,10 +112,10 @@ func (s *FederationInternalAPI) handleLocalKeys(
 			// Insert our own key into the response.
 			results[req] = gomatrixserverlib.PublicKeyLookupResult{
 				VerifyKey: gomatrixserverlib.VerifyKey{
-					Key: gomatrixserverlib.Base64Bytes(s.cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey)),
+					Key: spec.Base64Bytes(s.cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey)),
 				},
 				ExpiredTS:    gomatrixserverlib.PublicKeyNotExpired,
-				ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(s.cfg.Matrix.KeyValidityPeriod)),
+				ValidUntilTS: spec.AsTimestamp(time.Now().Add(s.cfg.Matrix.KeyValidityPeriod)),
 			}
 		} else {
 			// The key request doesn't match our current key. Let's see
@@ -128,7 +129,7 @@ func (s *FederationInternalAPI) handleLocalKeys(
 					// Insert our own key into the response.
 					results[req] = gomatrixserverlib.PublicKeyLookupResult{
 						VerifyKey: gomatrixserverlib.VerifyKey{
-							Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PrivateKey.Public().(ed25519.PublicKey)),
+							Key: spec.Base64Bytes(oldVerifyKey.PrivateKey.Public().(ed25519.PublicKey)),
 						},
 						ExpiredTS:    oldVerifyKey.ExpiredAt,
 						ValidUntilTS: gomatrixserverlib.PublicKeyNotValid,
@@ -146,8 +147,8 @@ func (s *FederationInternalAPI) handleLocalKeys(
 // satisfied from our local database/cache.
 func (s *FederationInternalAPI) handleDatabaseKeys(
 	ctx context.Context,
-	now gomatrixserverlib.Timestamp,
+	now spec.Timestamp,
-	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
+	requests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp,
 	results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult,
 ) error {
 	// Ask the database/cache for the keys.
@@ -180,9 +181,9 @@ func (s *FederationInternalAPI) handleDatabaseKeys(
 // the remaining requests.
 func (s *FederationInternalAPI) handleFetcherKeys(
 	ctx context.Context,
-	_ gomatrixserverlib.Timestamp,
+	_ spec.Timestamp,
 	fetcher gomatrixserverlib.KeyFetcher,
-	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
+	requests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp,
 	results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult,
 ) error {
 	logrus.WithFields(logrus.Fields{

federationapi/internal/perform.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 
@@ -80,8 +82,8 @@ func (r *FederationInternalAPI) PerformJoin(
 	// Deduplicate the server names we were provided but keep the ordering
 	// as this encodes useful information about which servers are most likely
 	// to respond.
-	seenSet := make(map[gomatrixserverlib.ServerName]bool)
+	seenSet := make(map[spec.ServerName]bool)
-	var uniqueList []gomatrixserverlib.ServerName
+	var uniqueList []spec.ServerName
 	for _, srv := range request.ServerNames {
 		if seenSet[srv] || r.cfg.Matrix.IsLocalServerName(srv) {
 			continue
@@ -143,7 +145,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	ctx context.Context,
 	roomID, userID string,
 	content map[string]interface{},
-	serverName gomatrixserverlib.ServerName,
+	serverName spec.ServerName,
 	supportedVersions []gomatrixserverlib.RoomVersion,
 	unsigned map[string]interface{},
 ) error {
@@ -175,7 +177,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 
 	// Set all the fields to be what they should be, this should be a no-op
 	// but it's possible that the remote server returned us something "odd"
-	respMakeJoin.JoinEvent.Type = gomatrixserverlib.MRoomMember
+	respMakeJoin.JoinEvent.Type = spec.MRoomMember
 	respMakeJoin.JoinEvent.Sender = userID
 	respMakeJoin.JoinEvent.StateKey = &userID
 	respMakeJoin.JoinEvent.RoomID = roomID
@@ -184,7 +186,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		content = map[string]interface{}{}
 	}
 	_ = json.Unmarshal(respMakeJoin.JoinEvent.Content, &content)
-	content["membership"] = gomatrixserverlib.Join
+	content["membership"] = spec.Join
 	if err = respMakeJoin.JoinEvent.SetContent(content); err != nil {
 		return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err)
 	}
@@ -233,7 +235,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	// contain signatures that we don't know about.
 	if len(respSendJoin.Event) > 0 {
 		var remoteEvent *gomatrixserverlib.Event
-		remoteEvent, err = respSendJoin.Event.UntrustedEvent(respMakeJoin.RoomVersion)
+		remoteEvent, err = gomatrixserverlib.UntrustedEvent(respSendJoin.Event, respMakeJoin.RoomVersion)
 		if err == nil && isWellFormedMembershipEvent(
 			remoteEvent, roomID, userID,
 		) {
@@ -255,10 +257,10 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	// waste the effort.
 	// TODO: Can we expand Check here to return a list of missing auth
 	// events rather than failing one at a time?
-	var respState *gomatrixserverlib.RespState
+	var respState gomatrixserverlib.StateResponse
-	respState, err = respSendJoin.Check(
+	respState, err = gomatrixserverlib.CheckSendJoinResponse(
 		context.Background(),
-		respMakeJoin.RoomVersion,
+		respMakeJoin.RoomVersion, &respSendJoin,
 		r.keyRing,
 		event,
 		federatedAuthProvider(ctx, r.federation, r.keyRing, origin, serverName),
@@ -273,7 +275,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	// joining a room, waiting for 200 OK then changing device keys and have those keys not be sent
 	// to other servers (this was a cause of a flakey sytest "Local device key changes get to remote servers")
 	// The events are trusted now as we performed auth checks above.
-	joinedHosts, err := consumers.JoinedHostsFromEvents(respState.StateEvents.TrustedEvents(respMakeJoin.RoomVersion, false))
+	joinedHosts, err := consumers.JoinedHostsFromEvents(respState.GetStateEvents().TrustedEvents(respMakeJoin.RoomVersion, false))
 	if err != nil {
 		return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err)
 	}
@@ -315,7 +317,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string) bool {
 	if membership, err := event.Membership(); err != nil {
 		return false
-	} else if membership != gomatrixserverlib.Join {
+	} else if membership != spec.Join {
 		return false
 	}
 	if event.RoomID() != roomID {
@@ -342,8 +344,8 @@ func (r *FederationInternalAPI) PerformOutboundPeek(
 	// Deduplicate the server names we were provided but keep the ordering
 	// as this encodes useful information about which servers are most likely
 	// to respond.
-	seenSet := make(map[gomatrixserverlib.ServerName]bool)
+	seenSet := make(map[spec.ServerName]bool)
-	var uniqueList []gomatrixserverlib.ServerName
+	var uniqueList []spec.ServerName
 	for _, srv := range request.ServerNames {
 		if seenSet[srv] {
 			continue
@@ -409,7 +411,7 @@ func (r *FederationInternalAPI) PerformOutboundPeek(
 func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 	ctx context.Context,
 	roomID string,
-	serverName gomatrixserverlib.ServerName,
+	serverName spec.ServerName,
 	supportedVersions []gomatrixserverlib.RoomVersion,
 ) error {
 	if !r.shouldAttemptDirectFederation(serverName) {
@@ -468,11 +470,12 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 
 	// we have the peek state now so let's process regardless of whether upstream gives up
 	ctx = context.Background()
-	respState := respPeek.ToRespState()
 
 	// authenticate the state returned (check its auth events etc)
 	// the equivalent of CheckSendJoinResponse()
-	authEvents, _, err := respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, r.cfg.Matrix.ServerName, serverName))
+	authEvents, stateEvents, err := gomatrixserverlib.CheckStateResponse(
+		ctx, &respPeek, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, r.cfg.Matrix.ServerName, serverName),
+	)
 	if err != nil {
 		return fmt.Errorf("error checking state returned from peeking: %w", err)
 	}
@@ -496,7 +499,11 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 	if err = roomserverAPI.SendEventWithState(
 		ctx, r.rsAPI, r.cfg.Matrix.ServerName,
 		roomserverAPI.KindNew,
-		&respState,
+		// use the authorized state from CheckStateResponse
+		&fclient.RespState{
+			StateEvents: gomatrixserverlib.NewEventJSONsFromEvents(stateEvents),
+			AuthEvents:  gomatrixserverlib.NewEventJSONsFromEvents(authEvents),
+		},
 		respPeek.LatestEvent.Headered(respPeek.RoomVersion),
 		serverName,
 		nil,
@@ -547,7 +554,7 @@ func (r *FederationInternalAPI) PerformLeave(
 
 		// Set all the fields to be what they should be, this should be a no-op
 		// but it's possible that the remote server returned us something "odd"
-		respMakeLeave.LeaveEvent.Type = gomatrixserverlib.MRoomMember
+		respMakeLeave.LeaveEvent.Type = spec.MRoomMember
 		respMakeLeave.LeaveEvent.Sender = request.UserID
 		respMakeLeave.LeaveEvent.StateKey = &request.UserID
 		respMakeLeave.LeaveEvent.RoomID = request.RoomID
@@ -643,7 +650,7 @@ func (r *FederationInternalAPI) PerformInvite(
 		"destination":  destination,
 	}).Info("Sending invite")
 
-	inviteReq, err := gomatrixserverlib.NewInviteV2Request(request.Event, request.InviteRoomState)
+	inviteReq, err := fclient.NewInviteV2Request(request.Event, request.InviteRoomState)
 	if err != nil {
 		return fmt.Errorf("gomatrixserverlib.NewInviteV2Request: %w", err)
 	}
@@ -653,7 +660,7 @@ func (r *FederationInternalAPI) PerformInvite(
 		return fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err)
 	}
 
-	inviteEvent, err := inviteRes.Event.UntrustedEvent(request.RoomVersion)
+	inviteEvent, err := gomatrixserverlib.UntrustedEvent(inviteRes.Event, request.RoomVersion)
 	if err != nil {
 		return fmt.Errorf("r.federation.SendInviteV2 failed to decode event response: %w", err)
 	}
@@ -699,7 +706,7 @@ func (r *FederationInternalAPI) PerformWakeupServers(
 	return nil
 }
 
-func (r *FederationInternalAPI) MarkServersAlive(destinations []gomatrixserverlib.ServerName) {
+func (r *FederationInternalAPI) MarkServersAlive(destinations []spec.ServerName) {
 	for _, srv := range destinations {
 		wasBlacklisted := r.statistics.ForServer(srv).MarkServerAlive()
 		r.queues.RetryServer(srv, wasBlacklisted)
@@ -709,7 +716,7 @@ func (r *FederationInternalAPI) MarkServersAlive(destinations []gomatrixserverli
 func sanityCheckAuthChain(authChain []*gomatrixserverlib.Event) error {
 	// sanity check we have a create event and it has a known room version
 	for _, ev := range authChain {
-		if ev.Type() == gomatrixserverlib.MRoomCreate && ev.StateKeyEquals("") {
+		if ev.Type() == spec.MRoomCreate && ev.StateKeyEquals("") {
 			// make sure the room version is known
 			content := ev.Content()
 			verBody := struct {
@@ -761,7 +768,7 @@ func setDefaultRoomVersionFromJoinEvent(
 // FederatedAuthProvider is an auth chain provider which fetches events from the server provided
 func federatedAuthProvider(
 	ctx context.Context, federation api.FederationClient,
-	keyRing gomatrixserverlib.JSONVerifier, origin, server gomatrixserverlib.ServerName,
+	keyRing gomatrixserverlib.JSONVerifier, origin, server spec.ServerName,
 ) gomatrixserverlib.AuthChainProvider {
 	// A list of events that we have retried, if they were not included in
 	// the auth events supplied in the send_join.
@@ -867,7 +874,7 @@ func (r *FederationInternalAPI) P2PRemoveRelayServers(
 }
 
 func (r *FederationInternalAPI) shouldAttemptDirectFederation(
-	destination gomatrixserverlib.ServerName,
+	destination spec.ServerName,
 ) bool {
 	var shouldRelay bool
 	stats := r.statistics.ForServer(destination)