Merge remote-tracking branch 'ldap/syncloud' into daniel/ldap-integration

.github/workflows/helm.yml

@@ -6,6 +6,7 @@ on:
       - main
     paths:
       - 'helm/**' # only execute if we have helm chart changes
+  workflow_dispatch:
 
 jobs:
   release:

.github/workflows/k8s.yml

@@ -84,6 +84,7 @@ jobs:
           kubectl get pods -A
           kubectl get services
           kubectl get ingress
+          kubectl logs -l app.kubernetes.io/name=dendrite
       - name: Run create account
         run: |
           podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name)

.github/workflows/schedules.yaml

@@ -65,10 +65,11 @@ jobs:
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
+          name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }})
           path: |
             /logs/results.tap
             /logs/**/*.log*
+            /logs/**/covdatafiles/**
 
   sytest-coverage:
     timeout-minutes: 5
@@ -85,16 +86,15 @@ jobs:
           cache: true
       - name: Download all artifacts
         uses: actions/download-artifact@v3
-      - name: Install gocovmerge
-        run: go install github.com/wadey/gocovmerge@latest
-      - name: Run gocovmerge
+      - name: Collect coverage
         run: |
-          find -name 'integrationcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > sytest.cov
-          go tool cover -func=sytest.cov
+          go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov
+          grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov
+          go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          files: ./sytest.cov
+          files: ./final.cov
           flags: sytest
           fail_ci_if_error: true
 
@@ -167,7 +167,7 @@ jobs:
           cat <<EOF > /tmp/posttest.sh
           #!/bin/bash
           mkdir -p /tmp/Complement/logs/\$2/\$1/
-          docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
+          docker cp \$1:/tmp/covdatafiles/. /tmp/Complement/logs/\$2/\$1/
           EOF
           
           chmod +x /tmp/posttest.sh
@@ -188,9 +188,9 @@ jobs:
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
+          name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }})
           path: |
-            /tmp/Complement/**/complementcover.log
+            /tmp/Complement/logs/**
 
   complement-coverage:
     timeout-minutes: 5
@@ -207,20 +207,19 @@ jobs:
           cache: true
       - name: Download all artifacts
         uses: actions/download-artifact@v3
-      - name: Install gocovmerge
-        run: go install github.com/wadey/gocovmerge@latest
-      - name: Run gocovmerge
+      - name: Collect coverage
         run: |
-          find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
-          go tool cover -func=complement.cov
+          go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov
+          grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
+          go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          files: ./complement.cov
+          files: ./final.cov
           flags: complement
           fail_ci_if_error: true
 
-  element_web:
+  element-web:
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -258,3 +257,42 @@ jobs:
         env:
           PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
           TMPDIR: ${{ runner.temp }}
+
+  element-web-pinecone:
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tecolicom/actions-use-apt-tools@v1
+        with:
+          # Our test suite includes some screenshot tests with unusual diacritics, which are
+          # supposed to be covered by STIXGeneral.
+          tools: fonts-stix
+      - uses: actions/checkout@v2
+        with:
+          repository: matrix-org/matrix-react-sdk
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+      - name: Fetch layered build
+        run: scripts/ci/layered.sh
+      - name: Copy config
+        run: cp element.io/develop/config.json config.json
+        working-directory: ./element-web
+      - name: Build
+        env:
+          CI_PACKAGE: true
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        run: yarn build
+        working-directory: ./element-web
+      - name: Edit Test Config
+        run: |
+          sed -i '/HOMESERVER/c\        HOMESERVER: "dendritePinecone",' cypress.config.ts
+      - name: "Run cypress tests"
+        uses: cypress-io/github-action@v4.1.1
+        with:
+          browser: chrome
+          start: npx serve -p 8080 ./element-web/webapp
+          wait-on: 'http://localhost:8080'
+        env:
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          TMPDIR: ${{ runner.temp }}

.gitignore

@@ -80,3 +80,4 @@ media_store/
 __debug_bin
 
 cmd/dendrite-monolith-server/dendrite-monolith-server
+build

appservice/api/query.go

@@ -22,8 +22,6 @@ import (
 	"encoding/json"
 	"errors"
 
-	"github.com/matrix-org/gomatrixserverlib"
-
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
@@ -150,6 +148,10 @@ type ASLocationResponse struct {
 	Fields   json.RawMessage `json:"fields"`
 }
 
+// ErrProfileNotExists is returned when trying to lookup a user's profile that
+// doesn't exist locally.
+var ErrProfileNotExists = errors.New("no known profile for given user ID")
+
 // RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
 // TODO: Remove this, it's called from federationapi and clientapi but is a pure function
@@ -157,25 +159,11 @@ func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
 	asAPI AppServiceInternalAPI,
-	profileAPI userapi.ClientUserAPI,
+	profileAPI userapi.ProfileAPI,
 ) (*authtypes.Profile, error) {
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
-	if err != nil {
-		return nil, err
-	}
-
 	// Try to query the user from the local database
-	res := &userapi.QueryProfileResponse{}
-	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
-	if err != nil {
-		return nil, err
-	}
-	profile := &authtypes.Profile{
-		Localpart:   localpart,
-		DisplayName: res.DisplayName,
-		AvatarURL:   res.AvatarURL,
-	}
-	if res.UserExists {
+	profile, err := profileAPI.QueryProfile(ctx, userID)
+	if err == nil {
 		return profile, nil
 	}
 
@@ -188,19 +176,15 @@ func RetrieveUserProfile(
 
 	// If no user exists, return
 	if !userResp.UserIDExists {
-		return nil, errors.New("no known profile for given user ID")
+		return nil, ErrProfileNotExists
 	}
 
 	// Try to query the user from the local database again
-	err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res)
+	profile, err = profileAPI.QueryProfile(ctx, userID)
 	if err != nil {
 		return nil, err
 	}
 
 	// profile should not be nil at this point
-	return &authtypes.Profile{
-		Localpart:   localpart,
-		DisplayName: res.DisplayName,
-		AvatarURL:   res.AvatarURL,
-	}, nil
+	return profile, nil
 }

appservice/appservice.go

@@ -16,11 +16,10 @@ package appservice
 
 import (
 	"context"
-	"crypto/tls"
-	"net/http"
 	"sync"
-	"time"
 
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -29,7 +28,6 @@ import (
 	"github.com/matrix-org/dendrite/appservice/consumers"
 	"github.com/matrix-org/dendrite/appservice/query"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
@@ -37,39 +35,31 @@ import (
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *base.BaseDendrite,
+	processContext *process.ProcessContext,
+	cfg *config.Dendrite,
+	natsInstance *jetstream.NATSInstance,
 	userAPI userapi.AppserviceUserAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceInternalAPI {
-	client := &http.Client{
-		Timeout: time.Second * 30,
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation,
-			},
-			Proxy: http.ProxyFromEnvironment,
-		},
-	}
+
 	// Create appserivce query API with an HTTP client that will be used for all
 	// outbound and inbound requests (inbound only for the internal API)
 	appserviceQueryAPI := &query.AppServiceQueryAPI{
-		HTTPClient:    client,
-		Cfg:           &base.Cfg.AppServiceAPI,
+		Cfg:           &cfg.AppServiceAPI,
 		ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{},
 		CacheMu:       sync.Mutex{},
 	}
 
-	if len(base.Cfg.Derived.ApplicationServices) == 0 {
+	if len(cfg.Derived.ApplicationServices) == 0 {
 		return appserviceQueryAPI
 	}
 
 	// Wrap application services in a type that relates the application service and
 	// a sync.Cond object that can be used to notify workers when there are new
 	// events to be sent out.
-	for _, appservice := range base.Cfg.Derived.ApplicationServices {
+	for _, appservice := range cfg.Derived.ApplicationServices {
 		// Create bot account for this AS if it doesn't already exist
-		if err := generateAppServiceAccount(userAPI, appservice, base.Cfg.Global.ServerName); err != nil {
+		if err := generateAppServiceAccount(userAPI, appservice, cfg.Global.ServerName); err != nil {
 			logrus.WithFields(logrus.Fields{
 				"appservice": appservice.ID,
 			}).WithError(err).Panicf("failed to generate bot account for appservice")
@@ -78,10 +68,10 @@ func NewInternalAPI(
 
 	// Only consume if we actually have ASes to track, else we'll just chew cycles needlessly.
 	// We can't add ASes at runtime so this is safe to do.
-	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	js, _ := natsInstance.Prepare(processContext, &cfg.Global.JetStream)
 	consumer := consumers.NewOutputRoomEventConsumer(
-		base.ProcessContext, &base.Cfg.AppServiceAPI,
-		client, js, rsAPI,
+		processContext, &cfg.AppServiceAPI,
+		js, rsAPI,
 	)
 	if err := consumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")

appservice/appservice_test.go

@@ -3,19 +3,31 @@ package appservice_test
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"net"
 	"net/http"
 	"net/http/httptest"
+	"path"
 	"reflect"
 	"regexp"
 	"strings"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
 
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/appservice/consumers"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/test/testrig"
 )
@@ -104,12 +116,11 @@ func TestAppserviceInternalAPI(t *testing.T) {
 	}
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, closeBase := testrig.CreateBaseDendrite(t, dbType)
-		defer closeBase()
+		cfg, ctx, close := testrig.CreateConfig(t, dbType)
+		defer close()
 
 		// Create a dummy application service
-		base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
-			{
+		as := &config.ApplicationService{
 			ID:              "someID",
 			URL:             srv.URL,
 			ASToken:         "",
@@ -120,18 +131,123 @@ func TestAppserviceInternalAPI(t *testing.T) {
 				"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
 			},
 			Protocols: []string{existingProtocol},
-			},
 		}
+		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
 
+		t.Cleanup(func() {
+			ctx.ShutdownDendrite()
+			ctx.WaitForShutdown()
+		})
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		// Create required internal APIs
-		rsAPI := roomserver.NewInternalAPI(base)
-		usrAPI := userapi.NewInternalAPI(base, rsAPI, nil)
-		asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
+		natsInstance := jetstream.NATSInstance{}
+		cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+		asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 		runCases(t, asAPI)
 	})
 }
 
+func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) {
+
+	// Set expected results
+	existingProtocol := "irc"
+	wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}}
+
+	// create a dummy AS url, handling some cases
+	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.Contains(r.URL.Path, "location"):
+			// Check if we've got an existing protocol, if so, return a proper response.
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "user"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "protocol"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode(nil); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		default:
+			t.Logf("hit location: %s", r.URL.Path)
+		}
+	}))
+
+	tmpDir := t.TempDir()
+	socket := path.Join(tmpDir, "socket")
+	l, err := net.Listen("unix", socket)
+	assert.NoError(t, err)
+	_ = srv.Listener.Close()
+	srv.Listener = l
+	srv.Start()
+	defer srv.Close()
+
+	cfg, ctx, tearDown := testrig.CreateConfig(t, test.DBTypeSQLite)
+	defer tearDown()
+
+	// Create a dummy application service
+	as := &config.ApplicationService{
+		ID:              "someID",
+		URL:             fmt.Sprintf("unix://%s", socket),
+		ASToken:         "",
+		HSToken:         "",
+		SenderLocalpart: "senderLocalPart",
+		NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+			"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
+			"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
+		},
+		Protocols: []string{existingProtocol},
+	}
+	as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
+	cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+	t.Cleanup(func() {
+		ctx.ShutdownDendrite()
+		ctx.WaitForShutdown()
+	})
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	// Create required internal APIs
+	natsInstance := jetstream.NATSInstance{}
+	cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
+	rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+	asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
+
+	t.Run("UserIDExists", func(t *testing.T) {
+		testUserIDExists(t, asAPI, "@as-testing:test", true)
+		testUserIDExists(t, asAPI, "@as1-testing:test", false)
+	})
+
+}
+
 func testUserIDExists(t *testing.T, asAPI api.AppServiceInternalAPI, userID string, wantExists bool) {
 	ctx := context.Background()
 	userResp := &api.UserIDExistsResponse{}
@@ -201,3 +317,87 @@ func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, w
 		t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
 	}
 }
+
+// Tests that the roomserver consumer only receives one invite
+func TestRoomserverConsumerOneInvite(t *testing.T) {
+
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	// Invite Bob
+	room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "invite",
+	}, test.WithStateKey(bob.ID))
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		natsInstance := &jetstream.NATSInstance{}
+
+		evChan := make(chan struct{})
+		// create a dummy AS url, handling the events
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var txn consumers.ApplicationServiceTransaction
+			err := json.NewDecoder(r.Body).Decode(&txn)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, ev := range txn.Events {
+				if ev.Type != gomatrixserverlib.MRoomMember {
+					continue
+				}
+				// Usually we would check the event content for the membership, but since
+				// we only invited bob, this should be fine for this test.
+				if ev.StateKey != nil && *ev.StateKey == bob.ID {
+					evChan <- struct{}{}
+				}
+			}
+		}))
+		defer srv.Close()
+
+		as := &config.ApplicationService{
+			ID:              "someID",
+			URL:             srv.URL,
+			ASToken:         "",
+			HSToken:         "",
+			SenderLocalpart: "senderLocalPart",
+			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+				"users":   {{RegexpObject: regexp.MustCompile(bob.ID)}},
+				"aliases": {{RegexpObject: regexp.MustCompile(room.ID)}},
+			},
+		}
+		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
+
+		// Create a dummy application service
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		// Create required internal APIs
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		// start the consumer
+		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
+
+		// Create the room
+		if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+		var seenInvitesForBob int
+	waitLoop:
+		for {
+			select {
+			case <-time.After(time.Millisecond * 50): // wait for the AS to process the events
+				break waitLoop
+			case <-evChan:
+				seenInvitesForBob++
+				if seenInvitesForBob != 1 {
+					t.Fatalf("received unexpected invites: %d", seenInvitesForBob)
+				}
+			}
+		}
+		close(evChan)
+	})
+}

appservice/consumers/roomserver.go

@@ -32,15 +32,21 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/syncapi/synctypes"
 
 	log "github.com/sirupsen/logrus"
 )
 
+// ApplicationServiceTransaction is the transaction that is sent off to an
+// application service.
+type ApplicationServiceTransaction struct {
+	Events []synctypes.ClientEvent `json:"events"`
+}
+
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
 	ctx       context.Context
 	cfg       *config.AppServiceAPI
-	client    *http.Client
 	jetstream nats.JetStreamContext
 	topic     string
 	rsAPI     api.AppserviceRoomserverAPI
@@ -56,14 +62,12 @@ type appserviceState struct {
 func NewOutputRoomEventConsumer(
 	process *process.ProcessContext,
 	cfg *config.AppServiceAPI,
-	client *http.Client,
 	js nats.JetStreamContext,
 	rsAPI api.AppserviceRoomserverAPI,
 ) *OutputRoomEventConsumer {
 	return &OutputRoomEventConsumer{
 		ctx:       process.Context(),
 		cfg:       cfg,
-		client:    client,
 		jetstream: js,
 		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
 		rsAPI:     rsAPI,
@@ -140,12 +144,6 @@ func (s *OutputRoomEventConsumer) onMessage(
 				}
 			}
 
-		case api.OutputTypeNewInviteEvent:
-			if output.NewInviteEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewInviteEvent.Event, state.ApplicationService) {
-				continue
-			}
-			events = append(events, output.NewInviteEvent.Event)
-
 		default:
 			continue
 		}
@@ -180,8 +178,8 @@ func (s *OutputRoomEventConsumer) sendEvents(
 ) error {
 	// Create the transaction body.
 	transaction, err := json.Marshal(
-		gomatrixserverlib.ApplicationServiceTransaction{
-			Events: gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll),
+		ApplicationServiceTransaction{
+			Events: synctypes.HeaderedToClientEvents(events, synctypes.FormatAll),
 		},
 	)
 	if err != nil {
@@ -195,13 +193,13 @@ func (s *OutputRoomEventConsumer) sendEvents(
 
 	// Send the transaction to the appservice.
 	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
-	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.URL, txnID, url.QueryEscape(state.HSToken))
+	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.RequestUrl(), txnID, url.QueryEscape(state.HSToken))
 	req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
 	if err != nil {
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	resp, err := s.client.Do(req)
+	resp, err := state.HTTPClient.Do(req)
 	if err != nil {
 		return state.backoffAndPause(err)
 	}
@@ -212,7 +210,7 @@ func (s *OutputRoomEventConsumer) sendEvents(
 	case http.StatusOK:
 		state.backoff = 0
 	default:
-		return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice", resp.StatusCode))
+		return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice url %s", resp.StatusCode, address))
 	}
 	return nil
 }

appservice/query/query.go

@@ -37,7 +37,6 @@ const userIDExistsPath = "/users/"
 
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
-	HTTPClient    *http.Client
 	Cfg           *config.AppServiceAPI
 	ProtocolCache map[string]api.ASProtocolResponse
 	CacheMu       sync.Mutex
@@ -57,7 +56,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.URL + roomAliasExistsPath)
+			URL, err := url.Parse(appservice.RequestUrl() + roomAliasExistsPath)
 			if err != nil {
 				return err
 			}
@@ -73,7 +72,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			}
 			req = req.WithContext(ctx)
 
-			resp, err := a.HTTPClient.Do(req)
+			resp, err := appservice.HTTPClient.Do(req)
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
@@ -124,7 +123,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.URL + userIDExistsPath)
+			URL, err := url.Parse(appservice.RequestUrl() + userIDExistsPath)
 			if err != nil {
 				return err
 			}
@@ -137,7 +136,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
-			resp, err := a.HTTPClient.Do(req.WithContext(ctx))
+			resp, err := appservice.HTTPClient.Do(req.WithContext(ctx))
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
@@ -212,12 +211,12 @@ func (a *AppServiceQueryAPI) Locations(
 		var asLocations []api.ASLocationResponse
 		params.Set("access_token", as.HSToken)
 
-		url := as.URL + api.ASLocationPath
+		url := as.RequestUrl() + api.ASLocationPath
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
+		if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
 			log.WithError(err).Error("unable to get 'locations' from application service")
 			continue
 		}
@@ -247,12 +246,12 @@ func (a *AppServiceQueryAPI) User(
 		var asUsers []api.ASUserResponse
 		params.Set("access_token", as.HSToken)
 
-		url := as.URL + api.ASUserPath
+		url := as.RequestUrl() + api.ASUserPath
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
+		if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
 			log.WithError(err).Error("unable to get 'user' from application service")
 			continue
 		}
@@ -290,7 +289,7 @@ func (a *AppServiceQueryAPI) Protocols(
 		response := api.ASProtocolResponse{}
 		for _, as := range a.Cfg.Derived.ApplicationServices {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil {
 				log.WithError(err).Error("unable to get 'protocol' from application service")
 				continue
 			}
@@ -320,7 +319,7 @@ func (a *AppServiceQueryAPI) Protocols(
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		for _, p := range as.Protocols {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil {
 				log.WithError(err).Error("unable to get 'protocol' from application service")
 				continue
 			}

build/gobind-yggdrasil/monolith.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
@@ -19,11 +20,15 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
@@ -148,25 +153,71 @@ func (m *DendriteMonolith) Start() {
 		panic(err)
 	}
 
-	base := base.NewBaseDendrite(cfg)
-	base.ConfigureAdminEndpoints()
-	m.processContext = base.ProcessContext
-	defer base.Close() // nolint: errcheck
+	configErrors := &config.ConfigErrors{}
+	cfg.Verify(configErrors)
+	if len(*configErrors) > 0 {
+		for _, err := range *configErrors {
+			logrus.Errorf("Configuration error: %s", err)
+		}
+		logrus.Fatalf("Failed to start due to configuration errors")
+	}
 
-	federation := ygg.CreateFederationClient(base)
+	internal.SetupStdLogging()
+	internal.SetupHookLogging(cfg.Logging)
+	internal.SetupPprof()
+
+	logrus.Infof("Dendrite version %s", internal.VersionString())
+
+	if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled {
+		logrus.Warn("Open registration is enabled")
+	}
+
+	closer, err := cfg.SetupTracing()
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to start opentracing")
+	}
+	defer closer.Close()
+
+	if cfg.Global.Sentry.Enabled {
+		logrus.Info("Setting up Sentry for debugging...")
+		err = sentry.Init(sentry.ClientOptions{
+			Dsn:              cfg.Global.Sentry.DSN,
+			Environment:      cfg.Global.Sentry.Environment,
+			Debug:            true,
+			ServerName:       string(cfg.Global.ServerName),
+			Release:          "dendrite@" + internal.VersionString(),
+			AttachStacktrace: true,
+		})
+		if err != nil {
+			logrus.WithError(err).Panic("failed to start Sentry")
+		}
+	}
+	processCtx := process.NewProcessContext()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+	basepkg.ConfigureAdminEndpoints(processCtx, routers)
+	m.processContext = processCtx
+	defer func() {
+		processCtx.ShutdownDendrite()
+		processCtx.WaitForShutdown()
+	}() // nolint: errcheck
+
+	federation := ygg.CreateFederationClient(cfg)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	rsAPI := roomserver.NewInternalAPI(base)
+	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
+	natsInstance := jetstream.NATSInstance{}
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, keyRing, true,
+		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
-	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
 
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
@@ -174,8 +225,8 @@ func (m *DendriteMonolith) Start() {
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    ygg.CreateClient(base),
+		Config:    cfg,
+		Client:    ygg.CreateClient(),
 		FedClient: federation,
 		KeyRing:   keyRing,
 
@@ -187,17 +238,17 @@ func (m *DendriteMonolith) Start() {
 			ygg, fsAPI, federation,
 		),
 	}
-	monolith.AddAllPublicRoutes(base)
+	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
 
 	httpRouter := mux.NewRouter()
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
-	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin)
 
 	yggRouter := mux.NewRouter()
-	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
-	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(routers.Federation)
+	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
 
 	// Build both ends of a HTTP multiplex.
 	m.httpServer = &http.Server{

build/scripts/Complement.Dockerfile

@@ -1,6 +1,6 @@
 #syntax=docker/dockerfile:1.2
 
-FROM golang:1.18-stretch as build
+FROM golang:1.20-bullseye as build
 RUN apt-get update && apt-get install -y sqlite3
 WORKDIR /build
 
@@ -17,7 +17,7 @@ RUN --mount=target=. \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
     CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
-    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
     cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite

build/scripts/ComplementPostgres.Dockerfile

@@ -1,19 +1,19 @@
 #syntax=docker/dockerfile:1.2
 
-FROM golang:1.18-stretch as build
+FROM golang:1.20-bullseye as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 
 # No password when connecting over localhost
-RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/9.6/main/pg_hba.conf && \
+RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/13/main/pg_hba.conf && \
     # Bump up max conns for moar concurrency
-    sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf
+    sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/13/main/postgresql.conf
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
     #!/bin/bash -eu \n\
     pg_lsclusters \n\
-    pg_ctlcluster 9.6 main start \n\
+    pg_ctlcluster 13 main start \n\
     \n\
     until pg_isready \n\
     do \n\
@@ -35,7 +35,7 @@ RUN --mount=target=. \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
     CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
-    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
     cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite

build/scripts/complement-cmd.sh

@@ -2,14 +2,15 @@
 
 # This script is intended to be used inside a docker container for Complement
 
+export GOCOVERDIR=/tmp/covdatafiles
+mkdir -p "${GOCOVERDIR}"
 if [[ "${COVER}" -eq 1 ]]; then
   echo "Running with coverage"
   exec /dendrite/dendrite-cover \
     --really-enable-open-registration \
     --tls-cert server.crt \
     --tls-key server.key \
-    --config dendrite.yaml \
-    --test.coverprofile=complementcover.log
+    --config dendrite.yaml
 else
   echo "Not running with coverage"
   exec /dendrite/dendrite \

clientapi/admin_test.go

@@ -4,15 +4,22 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"testing"
+	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 	"github.com/tidwall/gjson"
 
@@ -29,54 +36,30 @@ func TestAdminResetPassword(t *testing.T) {
 
 	ctx := context.Background()
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
-		defer baseClose()
-
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
 		// add a vhost
-		base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
+		cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{
+			SigningIdentity: fclient.SigningIdentity{ServerName: "vh1"},
 		})
 
-		rsAPI := roomserver.NewInternalAPI(base)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		// Needed for changing the password/login
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
-		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil)
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
 		// Create the users in the userapi and login
-		accessTokens := map[*test.User]string{
-			aliceAdmin: "",
-			bob:        "",
-			vhUser:     "",
-		}
-		for u := range accessTokens {
-			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
-			userRes := &uapi.PerformAccountCreationResponse{}
-			password := util.RandomString(8)
-			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
-				AccountType: u.AccountType,
-				Localpart:   localpart,
-				ServerName:  serverName,
-				Password:    password,
-			}, userRes); err != nil {
-				t.Errorf("failed to create account: %s", err)
-			}
-
-			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
-				"type": authtypes.LoginTypePassword,
-				"identifier": map[string]interface{}{
-					"type": "m.id.user",
-					"user": u.ID,
-				},
-				"password": password,
-			}))
-			rec := httptest.NewRecorder()
-			base.PublicClientAPIMux.ServeHTTP(rec, req)
-			if rec.Code != http.StatusOK {
-				t.Fatalf("failed to login: %s", rec.Body.String())
-			}
-			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+			bob:        {},
+			vhUser:     {},
 		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
 
 		testCases := []struct {
 			name           string
@@ -120,11 +103,11 @@ func TestAdminResetPassword(t *testing.T) {
 				}
 
 				if tc.withHeader {
-					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser])
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken)
 				}
 
 				rec := httptest.NewRecorder()
-				base.DendriteAdminMux.ServeHTTP(rec, req)
+				routers.DendriteAdmin.ServeHTTP(rec, req)
 				t.Logf("%s", rec.Body.String())
 				if tc.wantOK && rec.Code != http.StatusOK {
 					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
@@ -147,16 +130,19 @@ func TestPurgeRoom(t *testing.T) {
 	ctx := context.Background()
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
-		defer baseClose()
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
 
-		fedClient := base.CreateFederationClient()
-		rsAPI := roomserver.NewInternalAPI(base)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		// this starts the JetStream consumers
-		syncapi.AddPublicRoutes(base, userAPI, rsAPI)
-		federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true)
+		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
+		federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(nil, nil)
 
 		// Create the room
@@ -165,40 +151,13 @@ func TestPurgeRoom(t *testing.T) {
 		}
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
-		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil)
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
 		// Create the users in the userapi and login
-		accessTokens := map[*test.User]string{
-			aliceAdmin: "",
-		}
-		for u := range accessTokens {
-			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
-			userRes := &uapi.PerformAccountCreationResponse{}
-			password := util.RandomString(8)
-			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
-				AccountType: u.AccountType,
-				Localpart:   localpart,
-				ServerName:  serverName,
-				Password:    password,
-			}, userRes); err != nil {
-				t.Errorf("failed to create account: %s", err)
-			}
-
-			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
-				"type": authtypes.LoginTypePassword,
-				"identifier": map[string]interface{}{
-					"type": "m.id.user",
-					"user": u.ID,
-				},
-				"password": password,
-			}))
-			rec := httptest.NewRecorder()
-			base.PublicClientAPIMux.ServeHTTP(rec, req)
-			if rec.Code != http.StatusOK {
-				t.Fatalf("failed to login: %s", rec.Body.String())
-			}
-			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
 		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
 
 		testCases := []struct {
 			name   string
@@ -215,10 +174,10 @@ func TestPurgeRoom(t *testing.T) {
 			t.Run(tc.name, func(t *testing.T) {
 				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/purgeRoom/"+tc.roomID)
 
-				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin])
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
 
 				rec := httptest.NewRecorder()
-				base.DendriteAdminMux.ServeHTTP(rec, req)
+				routers.DendriteAdmin.ServeHTTP(rec, req)
 				t.Logf("%s", rec.Body.String())
 				if tc.wantOK && rec.Code != http.StatusOK {
 					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
@@ -228,3 +187,256 @@ func TestPurgeRoom(t *testing.T) {
 
 	})
 }
+
+func TestAdminEvacuateRoom(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, aliceAdmin)
+
+	// Join Bob
+	room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// this starts the JetStream consumers
+		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
+		rsAPI.SetFederationAPI(fsAPI, nil)
+
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+
+		testCases := []struct {
+			name         string
+			roomID       string
+			wantOK       bool
+			wantAffected []string
+		}{
+			{name: "Can evacuate existing room", wantOK: true, roomID: room.ID, wantAffected: []string{aliceAdmin.ID, bob.ID}},
+			{name: "Can not evacuate non-existent room", wantOK: false, roomID: "!doesnotexist:localhost", wantAffected: []string{}},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateRoom/"+tc.roomID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
+
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+
+				affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array()
+				affected := make([]string, 0, len(affectedArr))
+				for _, x := range affectedArr {
+					affected = append(affected, x.Str)
+				}
+				if !reflect.DeepEqual(affected, tc.wantAffected) {
+					t.Fatalf("expected affected %#v, but got %#v", tc.wantAffected, affected)
+				}
+			})
+		}
+
+		// Wait for the FS API to have consumed every message
+		js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		timeout := time.After(time.Second)
+		for {
+			select {
+			case <-timeout:
+				t.Fatalf("FS API didn't process all events in time")
+			default:
+			}
+			info, err := js.ConsumerInfo(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), cfg.Global.JetStream.Durable("FederationAPIRoomServerConsumer")+"Pull")
+			if err != nil {
+				time.Sleep(time.Millisecond * 10)
+				continue
+			}
+			if info.NumPending == 0 && info.NumAckPending == 0 {
+				break
+			}
+		}
+	})
+}
+
+func TestAdminEvacuateUser(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, aliceAdmin)
+	room2 := test.NewRoom(t, aliceAdmin)
+
+	// Join Bob
+	room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+	room2.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// this starts the JetStream consumers
+		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true)
+		rsAPI.SetFederationAPI(fsAPI, nil)
+
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room2.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+
+		testCases := []struct {
+			name              string
+			userID            string
+			wantOK            bool
+			wantAffectedRooms []string
+		}{
+			{name: "Can evacuate existing user", wantOK: true, userID: bob.ID, wantAffectedRooms: []string{room.ID, room2.ID}},
+			{name: "invalid userID is rejected", wantOK: false, userID: "!notauserid:test", wantAffectedRooms: []string{}},
+			{name: "Can not evacuate user from different server", wantOK: false, userID: "@doesnotexist:localhost", wantAffectedRooms: []string{}},
+			{name: "Can not evacuate non-existent user", wantOK: false, userID: "@doesnotexist:test", wantAffectedRooms: []string{}},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateUser/"+tc.userID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
+
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+
+				affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array()
+				affected := make([]string, 0, len(affectedArr))
+				for _, x := range affectedArr {
+					affected = append(affected, x.Str)
+				}
+				if !reflect.DeepEqual(affected, tc.wantAffectedRooms) {
+					t.Fatalf("expected affected %#v, but got %#v", tc.wantAffectedRooms, affected)
+				}
+
+			})
+		}
+		// Wait for the FS API to have consumed every message
+		js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		timeout := time.After(time.Second)
+		for {
+			select {
+			case <-timeout:
+				t.Fatalf("FS API didn't process all events in time")
+			default:
+			}
+			info, err := js.ConsumerInfo(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), cfg.Global.JetStream.Durable("FederationAPIRoomServerConsumer")+"Pull")
+			if err != nil {
+				time.Sleep(time.Millisecond * 10)
+				continue
+			}
+			if info.NumPending == 0 && info.NumAckPending == 0 {
+				break
+			}
+		}
+	})
+}
+
+func TestAdminMarkAsStale(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		defer close()
+
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]userDevice{
+			aliceAdmin: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, ctx, routers)
+
+		testCases := []struct {
+			name   string
+			userID string
+			wantOK bool
+		}{
+			{name: "local user is not allowed", userID: aliceAdmin.ID},
+			{name: "invalid userID", userID: "!notvalid:test"},
+			{name: "remote user is allowed", userID: "@alice:localhost", wantOK: true},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/refreshDevices/"+tc.userID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
+
+				rec := httptest.NewRecorder()
+				routers.DendriteAdmin.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}

clientapi/api/api.go

@@ -14,10 +14,10 @@
 
 package api
 
-import "github.com/matrix-org/gomatrixserverlib"
+import "github.com/matrix-org/gomatrixserverlib/fclient"
 
 // ExtraPublicRoomsProvider provides a way to inject extra published rooms into /publicRooms requests.
 type ExtraPublicRoomsProvider interface {
 	// Rooms returns the extra rooms. This is called on-demand by clients, so cache appropriately.
-	Rooms() []gomatrixserverlib.PublicRoom
+	Rooms() []fclient.PublicRoom
 }

clientapi/auth/authtypes/threepid.go

@@ -18,4 +18,6 @@ package authtypes
 type ThreePID struct {
 	Address     string `json:"address"`
 	Medium      string `json:"medium"`
+	AddedAt     int64  `json:"added_at"`
+	ValidatedAt int64  `json:"validated_at"`
 }

clientapi/auth/login.go

@@ -63,6 +63,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.C
 			Config:        cfg,
 			Rt:            rt,
 			InhibitDevice: header.InhibitDevice,
+			UserAPI:       useraccountAPI,
 		}
 	case authtypes.LoginTypeToken:
 		typ = &LoginTypeToken{

clientapi/auth/login_test.go

@@ -26,7 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -69,7 +69,7 @@ func TestLoginFromJSONReader(t *testing.T) {
 			var userAPI fakeUserInternalAPI
 			cfg := &config.ClientAPI{
 				Matrix: &config.Global{
-					SigningIdentity: gomatrixserverlib.SigningIdentity{
+					SigningIdentity: fclient.SigningIdentity{
 						ServerName: serverName,
 					},
 				},
@@ -152,7 +152,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 			var userAPI fakeUserInternalAPI
 			cfg := &config.ClientAPI{
 				Matrix: &config.Global{
-					SigningIdentity: gomatrixserverlib.SigningIdentity{
+					SigningIdentity: fclient.SigningIdentity{
 						ServerName: serverName,
 					},
 				},

clientapi/auth/password.go

@@ -16,9 +16,14 @@ package auth
 
 import (
 	"context"
+	"database/sql"
 	"net/http"
 	"strings"
 
+	"github.com/go-ldap/ldap/v3"
+	"github.com/google/uuid"
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
@@ -29,8 +34,6 @@ import (
 	"github.com/matrix-org/util"
 )
 
-type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error
-
 type PasswordRequest struct {
 	Login
 	Password string `json:"password"`
@@ -46,6 +49,7 @@ type LoginTypePassword struct {
 	Config        *config.ClientAPI
 	Rt            *ratelimit.RtFailedLogin
 	InhibitDevice bool
+	UserAPI       api.UserLoginAPI
 }
 
 func (t *LoginTypePassword) Name() string {
@@ -123,6 +127,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			JSON: jsonerror.InvalidUsername("The server name is not known."),
 		}
 	}
+
 	// Squash username to all lowercase letters
 	res := &api.QueryAccountByPasswordResponse{}
 	if t.Rt != nil {
@@ -146,13 +151,53 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		}
 	}
 
+	var account *api.Account
+	if t.Config.Ldap.Enabled {
+		isAdmin, err := t.authenticateLdap(localpart, r.Password)
+		if err != nil {
+			return nil, err
+		}
+		acc, err := t.getOrCreateAccount(ctx, localpart, domain, isAdmin)
+		if err != nil {
+			return nil, err
+		}
+		account = acc
+	} else {
+		acc, err := t.authenticateDb(ctx, localpart, domain, r.Password)
+		if err != nil {
+			return nil, err
+		}
+		account = acc
+	}
+
+	// Set the user, so login.Username() can do the right thing
+	r.Identifier.User = account.UserID
+	r.User = account.UserID
+	r.Login.User = username
+	return &r.Login, nil
+}
+
+func (t *LoginTypePassword) authenticateDb(ctx context.Context, localpart string, domain gomatrixserverlib.ServerName, password string) (*api.Account, *util.JSONResponse) {
+	res := &api.QueryAccountByPasswordResponse{}
+	err := t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+		Localpart:         strings.ToLower(localpart),
+		ServerName:        domain,
+		PlaintextPassword: password,
+	}, res)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.Unknown("Unable to fetch account by password."),
+		}
+	}
+
 	// If we couldn't find the user by the lower cased localpart, try the provided
 	// localpart as is.
 	if !res.Exists {
-		err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+		err = t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
 			Localpart:         localpart,
 			ServerName:        domain,
-			PlaintextPassword: r.Password,
+			PlaintextPassword: password,
 		}, res)
 		if err != nil {
 			return nil, &util.JSONResponse{
@@ -163,18 +208,149 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
 		// but that would leak the existence of the user.
 		if !res.Exists {
-			if t.Rt != nil {
-				t.Rt.Act(localpart)
-			}
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("Invalid username or password"),
+				JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."),
 			}
 		}
 	}
-	// Set the user, so login.Username() can do the right thing
-	r.Identifier.User = res.Account.UserID
-	r.User = res.Account.UserID
-	r.Login.User = username
-	return &r.Login, nil
+	return res.Account, nil
+}
+
+func (t *LoginTypePassword) authenticateLdap(username, password string) (bool, *util.JSONResponse) {
+	var conn *ldap.Conn
+	conn, err := ldap.DialURL(t.Config.Ldap.Uri)
+	if err != nil {
+		return false, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.Unknown("unable to connect to ldap: " + err.Error()),
+		}
+	}
+	defer conn.Close()
+
+	if t.Config.Ldap.AdminBindEnabled {
+		err = conn.Bind(t.Config.Ldap.AdminBindDn, t.Config.Ldap.AdminBindPassword)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: jsonerror.Unknown("unable to bind to ldap: " + err.Error()),
+			}
+		}
+		filter := strings.ReplaceAll(t.Config.Ldap.SearchFilter, "{username}", username)
+		searchRequest := ldap.NewSearchRequest(
+			t.Config.Ldap.BaseDn, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases,
+			0, 0, false, filter, []string{t.Config.Ldap.SearchAttribute}, nil,
+		)
+		var result *ldap.SearchResult
+		result, err = conn.Search(searchRequest)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: jsonerror.Unknown("unable to bind to search ldap: " + err.Error()),
+			}
+		}
+		if len(result.Entries) > 1 {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: jsonerror.BadJSON("'user' must be duplicated."),
+			}
+		}
+		if len(result.Entries) < 1 {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: jsonerror.BadJSON("'user' not found."),
+			}
+		}
+
+		userDN := result.Entries[0].DN
+		err = conn.Bind(userDN, password)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: jsonerror.InvalidUsername(err.Error()),
+			}
+		}
+	} else {
+		bindDn := strings.ReplaceAll(t.Config.Ldap.UserBindDn, "{username}", username)
+		err = conn.Bind(bindDn, password)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: jsonerror.InvalidUsername(err.Error()),
+			}
+		}
+	}
+
+	isAdmin, err := t.isLdapAdmin(conn, username)
+	if err != nil {
+		return false, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.InvalidUsername(err.Error()),
+		}
+	}
+	return isAdmin, nil
+}
+
+func (t *LoginTypePassword) isLdapAdmin(conn *ldap.Conn, username string) (bool, error) {
+	searchRequest := ldap.NewSearchRequest(
+		t.Config.Ldap.AdminGroupDn,
+		ldap.ScopeWholeSubtree, ldap.DerefAlways, 0, 0, false,
+		strings.ReplaceAll(t.Config.Ldap.AdminGroupFilter, "{username}", username),
+		[]string{t.Config.Ldap.AdminGroupAttribute},
+		nil)
+
+	sr, err := conn.Search(searchRequest)
+	if err != nil {
+		return false, err
+	}
+
+	if len(sr.Entries) < 1 {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, username string, domain gomatrixserverlib.ServerName, admin bool) (*api.Account, *util.JSONResponse) {
+	var existing api.QueryAccountByLocalpartResponse
+	err := t.UserAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{
+		Localpart:  username,
+		ServerName: domain,
+	}, &existing)
+
+	if err == nil {
+		return existing.Account, nil
+	}
+	if err != sql.ErrNoRows {
+		return nil, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.InvalidUsername(err.Error()),
+		}
+	}
+
+	accountType := api.AccountTypeUser
+	if admin {
+		accountType = api.AccountTypeAdmin
+	}
+	var created api.PerformAccountCreationResponse
+	err = t.UserAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{
+		AppServiceID: "ldap",
+		Localpart:    username,
+		Password:     uuid.New().String(),
+		AccountType:  accountType,
+		OnConflict:   api.ConflictAbort,
+	}, &created)
+
+	if err != nil {
+		if _, ok := err.(*api.ErrorConflict); ok {
+			return nil, &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.UserInUse("Desired user ID is already taken."),
+			}
+		}
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.Unknown("failed to create account: " + err.Error()),
+		}
+	}
+	return created.Account, nil
 }

clientapi/auth/user_interactive.go

@@ -115,6 +115,7 @@ type UserInteractive struct {
 func NewUserInteractive(userAccountAPI api.ClientUserAPI, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
 		UserApi: userAccountAPI,
+		UserAPI: userAccountAPI,
 		Config:  cfg,
 	}
 	return &UserInteractive{
@@ -141,7 +142,7 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
 	return false
 }
 
-func (u *UserInteractive) AddCompletedStage(sessionID, authType string) {
+func (u *UserInteractive) AddCompletedStage(sessionID, _ string) {
 	u.Lock()
 	// TODO: Handle multi-stage flows
 	delete(u.Sessions, sessionID)
@@ -215,7 +216,7 @@ func (u *UserInteractive) ResponseWithChallenge(sessionID string, response inter
 // Verify returns an error/challenge response to send to the client, or nil if the user is authenticated.
 // `bodyBytes` is the HTTP request body which must contain an `auth` key.
 // Returns the login that was verified for additional checks if required.
-func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *api.Device) (*Login, *util.JSONResponse) {
+func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, _ *api.Device) (*Login, *util.JSONResponse) {
 	// TODO: rate limit
 
 	// "A client should first make a request with no auth parameter. The homeserver returns an HTTP 401 response, with a JSON body"

clientapi/auth/user_interactive_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -49,7 +50,7 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a
 func setup() *UserInteractive {
 	cfg := &config.ClientAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: serverName,
 			},
 		},

clientapi/clientapi.go

@@ -15,8 +15,11 @@
 package clientapi
 
 import (
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/api"
@@ -25,42 +28,42 @@ import (
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 )
 
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
 func AddPublicRoutes(
-	base *base.BaseDendrite,
-	federation *gomatrixserverlib.FederationClient,
+	processContext *process.ProcessContext,
+	routers httputil.Routers,
+	cfg *config.Dendrite,
+	natsInstance *jetstream.NATSInstance,
+	federation *fclient.FederationClient,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 	transactionsCache *transactions.Cache,
 	fsAPI federationAPI.ClientFederationAPI,
 	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.QuerySearchProfilesAPI,
-	extRoomsProvider api.ExtraPublicRoomsProvider,
+	extRoomsProvider api.ExtraPublicRoomsProvider, enableMetrics bool,
 ) {
-	cfg := &base.Cfg.ClientAPI
-	mscCfg := &base.Cfg.MSCs
-	js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, natsClient := natsInstance.Prepare(processContext, &cfg.Global.JetStream)
 
 	syncProducer := &producers.SyncAPIProducer{
 		JetStream:              js,
-		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
-		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
-		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
-		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
-		TopicMultiRoomCast:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputMultiRoomCast),
+		TopicReceiptEvent:      cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+		TopicSendToDeviceEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		TopicTypingEvent:       cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicPresenceEvent:     cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		TopicMultiRoomCast:     cfg.Global.JetStream.Prefixed(jetstream.OutputMultiRoomCast),
 		UserAPI:                userAPI,
-		ServerName:             cfg.Matrix.ServerName,
+		ServerName:             cfg.Global.ServerName,
 	}
 
 	routing.Setup(
-		base,
+		routers,
 		cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
 		syncProducer, transactionsCache, fsAPI,
-		extRoomsProvider, mscCfg, natsClient,
+		extRoomsProvider, natsClient, enableMetrics,
 	)
 }

clientapi/routing/admin.go

@@ -22,23 +22,16 @@ import (
 	"github.com/matrix-org/dendrite/userapi/api"
 )
 
-func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	roomID, ok := vars["roomID"]
-	if !ok {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting room ID."),
-		}
-	}
 	res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
 	if err := rsAPI.PerformAdminEvacuateRoom(
 		req.Context(),
 		&roomserverAPI.PerformAdminEvacuateRoomRequest{
-			RoomID: roomID,
+			RoomID: vars["roomID"],
 		},
 		res,
 	); err != nil {
@@ -55,18 +48,13 @@ func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Dev
 	}
 }
 
-func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	userID, ok := vars["userID"]
-	if !ok {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting user ID."),
-		}
-	}
+	userID := vars["userID"]
+
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		return util.MessageResponse(http.StatusBadRequest, err.Error())
@@ -103,13 +91,8 @@ func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	roomID, ok := vars["roomID"]
-	if !ok {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting room ID."),
-		}
-	}
+	roomID := vars["roomID"]
+
 	res := &roomserverAPI.PerformAdminPurgeRoomResponse{}
 	if err := rsAPI.PerformAdminPurgeRoom(
 		context.Background(),

clientapi/routing/auth_fallback_test.go

@@ -10,30 +10,28 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/test/testrig"
 )
 
 func Test_AuthFallback(t *testing.T) {
-	base, _, _ := testrig.Base(nil)
-	defer base.Close()
-
+	cfg := config.Dendrite{}
+	cfg.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
 	for _, useHCaptcha := range []bool{false, true} {
 		for _, recaptchaEnabled := range []bool{false, true} {
 			for _, wantErr := range []bool{false, true} {
 				t.Run(fmt.Sprintf("useHCaptcha(%v) - recaptchaEnabled(%v) - wantErr(%v)", useHCaptcha, recaptchaEnabled, wantErr), func(t *testing.T) {
 					// Set the defaults for each test
-					base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
-					base.Cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled
-					base.Cfg.ClientAPI.RecaptchaPublicKey = "pub"
-					base.Cfg.ClientAPI.RecaptchaPrivateKey = "priv"
+					cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
+					cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled
+					cfg.ClientAPI.RecaptchaPublicKey = "pub"
+					cfg.ClientAPI.RecaptchaPrivateKey = "priv"
 					if useHCaptcha {
-						base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify"
-						base.Cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js"
-						base.Cfg.ClientAPI.RecaptchaFormField = "h-captcha-response"
-						base.Cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha"
+						cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify"
+						cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js"
+						cfg.ClientAPI.RecaptchaFormField = "h-captcha-response"
+						cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha"
 					}
 					cfgErrs := &config.ConfigErrors{}
-					base.Cfg.ClientAPI.Verify(cfgErrs)
+					cfg.ClientAPI.Verify(cfgErrs)
 					if len(*cfgErrs) > 0 {
 						t.Fatalf("(hCaptcha=%v) unexpected config errors: %s", useHCaptcha, cfgErrs.Error())
 					}
@@ -41,7 +39,7 @@ func Test_AuthFallback(t *testing.T) {
 					req := httptest.NewRequest(http.MethodGet, "/?session=1337", nil)
 					rec := httptest.NewRecorder()
 
-					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 					if !recaptchaEnabled {
 						if rec.Code != http.StatusBadRequest {
 							t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
@@ -50,8 +48,8 @@ func Test_AuthFallback(t *testing.T) {
 							t.Fatalf("unexpected response body: %s", rec.Body.String())
 						}
 					} else {
-						if !strings.Contains(rec.Body.String(), base.Cfg.ClientAPI.RecaptchaSitekeyClass) {
-							t.Fatalf("body does not contain %s: %s", base.Cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String())
+						if !strings.Contains(rec.Body.String(), cfg.ClientAPI.RecaptchaSitekeyClass) {
+							t.Fatalf("body does not contain %s: %s", cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String())
 						}
 					}
 
@@ -64,14 +62,14 @@ func Test_AuthFallback(t *testing.T) {
 					}))
 					defer srv.Close() // nolint: errcheck
 
-					base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
+					cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
 
 					// check the result after sending the captcha
 					req = httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
 					req.Form = url.Values{}
-					req.Form.Add(base.Cfg.ClientAPI.RecaptchaFormField, "someRandomValue")
+					req.Form.Add(cfg.ClientAPI.RecaptchaFormField, "someRandomValue")
 					rec = httptest.NewRecorder()
-					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 					if recaptchaEnabled {
 						if !wantErr {
 							if rec.Code != http.StatusOK {
@@ -105,7 +103,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("unknown fallbacks are handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, "DoesNotExist", &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, "DoesNotExist", &cfg.ClientAPI)
 		if rec.Code != http.StatusNotImplemented {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusNotImplemented)
 		}
@@ -114,7 +112,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("unknown methods are handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodDelete, "/?session=1337", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 		if rec.Code != http.StatusMethodNotAllowed {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusMethodNotAllowed)
 		}
@@ -123,7 +121,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodGet, "/", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 		if rec.Code != http.StatusBadRequest {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
 		}
@@ -132,7 +130,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodGet, "/", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 		if rec.Code != http.StatusBadRequest {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
 		}
@@ -141,7 +139,7 @@ func Test_AuthFallback(t *testing.T) {
 	t.Run("missing 'response' is handled correctly", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
 		rec := httptest.NewRecorder()
-		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI)
 		if rec.Code != http.StatusBadRequest {
 			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
 		}

clientapi/routing/capabilities.go

@@ -17,26 +17,21 @@ package routing
 import (
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-
+	"github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 // GetCapabilities returns information about the server's supported feature set
 // and other relevant capabilities to an authenticated user.
-func GetCapabilities(
-	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
-) util.JSONResponse {
-	roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{}
-	roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{}
-	if err := rsAPI.QueryRoomVersionCapabilities(
-		req.Context(),
-		&roomVersionsQueryReq,
-		&roomVersionsQueryRes,
-	); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryRoomVersionCapabilities failed")
-		return jsonerror.InternalServerError()
+func GetCapabilities() util.JSONResponse {
+	versionsMap := map[gomatrixserverlib.RoomVersion]string{}
+	for v, desc := range version.SupportedRoomVersions() {
+		if desc.Stable {
+			versionsMap[v] = "stable"
+		} else {
+			versionsMap[v] = "unstable"
+		}
 	}
 
 	response := map[string]interface{}{
@@ -44,7 +39,10 @@ func GetCapabilities(
 			"m.change_password": map[string]bool{
 				"enabled": true,
 			},
-			"m.room_versions": roomVersionsQueryRes,
+			"m.room_versions": map[string]interface{}{
+				"default":   version.DefaultRoomVersion(),
+				"available": versionsMap,
+			},
 		},
 	}
 

clientapi/routing/deactivate.go

@@ -37,7 +37,8 @@ func Deactivate(
 	} else {
 		userId = deviceAPI.UserID
 	}
-	localpart, _, err := gomatrixserverlib.SplitID('@', userId)
+
+	localpart, serverName, err := gomatrixserverlib.SplitID('@', userId)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
 		return jsonerror.InternalServerError()
@@ -46,6 +47,7 @@ func Deactivate(
 	var res api.PerformAccountDeactivationResponse
 	err = accountAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{
 		Localpart:  localpart,
+		ServerName: serverName,
 	}, &res)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.PerformAccountDeactivation failed")

clientapi/routing/device.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"encoding/json"
 	"io"
 	"net"
 	"net/http"
@@ -146,12 +147,6 @@ func UpdateDeviceByID(
 			JSON: jsonerror.Forbidden("device does not exist"),
 		}
 	}
-	if performRes.Forbidden {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("device not owned by current user"),
-		}
-	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -189,7 +184,7 @@ func DeleteDeviceById(
 		if dev != deviceID {
 			return util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("session & device mismatch"),
+				JSON: jsonerror.Forbidden("session and device mismatch"),
 			}
 		}
 	}
@@ -242,16 +237,37 @@ func DeleteDeviceById(
 
 // DeleteDevices handles POST requests to /delete_devices
 func DeleteDevices(
-	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
+	req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.ClientUserAPI, device *api.Device,
 ) util.JSONResponse {
 	ctx := req.Context()
-	payload := devicesDeleteJSON{}
 
-	if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil {
-		return *resErr
+	bodyBytes, err := io.ReadAll(req.Body)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
+		}
+	}
+	defer req.Body.Close() // nolint:errcheck
+
+	// initiate UIA
+	login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, device)
+	if errRes != nil {
+		return *errRes
 	}
 
-	defer req.Body.Close() // nolint: errcheck
+	if login.Username() != device.UserID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("unable to delete devices for other user"),
+		}
+	}
+
+	payload := devicesDeleteJSON{}
+	if err = json.Unmarshal(bodyBytes, &payload); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("unable to unmarshal device deletion request")
+		return jsonerror.InternalServerError()
+	}
 
 	var res api.PerformDeviceDeletionResponse
 	if err := userAPI.PerformDeviceDeletion(ctx, &api.PerformDeviceDeletionRequest{

clientapi/routing/directory.go

@@ -19,6 +19,7 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -45,7 +46,7 @@ func (r *roomDirectoryResponse) fillServers(servers []gomatrixserverlib.ServerNa
 func DirectoryRoom(
 	req *http.Request,
 	roomAlias string,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	fedSenderAPI federationAPI.ClientFederationAPI,

clientapi/routing/directory_public.go

@@ -24,6 +24,7 @@ import (
 	"sync"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/api"
@@ -35,7 +36,7 @@ import (
 
 var (
 	cacheMu          sync.Mutex
-	publicRoomsCache []gomatrixserverlib.PublicRoom
+	publicRoomsCache []fclient.PublicRoom
 )
 
 type PublicRoomReq struct {
@@ -56,7 +57,7 @@ type filter struct {
 func GetPostPublicRooms(
 	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var request PublicRoomReq
@@ -102,10 +103,10 @@ func GetPostPublicRooms(
 
 func publicRooms(
 	ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
-) (*gomatrixserverlib.RespPublicRooms, error) {
+) (*fclient.RespPublicRooms, error) {
 
-	response := gomatrixserverlib.RespPublicRooms{
-		Chunk: []gomatrixserverlib.PublicRoom{},
+	response := fclient.RespPublicRooms{
+		Chunk: []fclient.PublicRoom{},
 	}
 	var limit int64
 	var offset int64
@@ -122,7 +123,7 @@ func publicRooms(
 	}
 	err = nil
 
-	var rooms []gomatrixserverlib.PublicRoom
+	var rooms []fclient.PublicRoom
 	if request.Since == "" {
 		rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider, request)
 	} else {
@@ -146,14 +147,14 @@ func publicRooms(
 	return &response, err
 }
 
-func filterRooms(rooms []gomatrixserverlib.PublicRoom, searchTerm string) []gomatrixserverlib.PublicRoom {
+func filterRooms(rooms []fclient.PublicRoom, searchTerm string) []fclient.PublicRoom {
 	if searchTerm == "" {
 		return rooms
 	}
 
 	normalizedTerm := strings.ToLower(searchTerm)
 
-	result := make([]gomatrixserverlib.PublicRoom, 0)
+	result := make([]fclient.PublicRoom, 0)
 	for _, room := range rooms {
 		if strings.Contains(strings.ToLower(room.Name), normalizedTerm) ||
 			strings.Contains(strings.ToLower(room.Topic), normalizedTerm) ||
@@ -214,7 +215,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 //	 limit=3&since=6  => G     (prev='3', next='')
 //
 //	A value of '-1' for prev/next indicates no position.
-func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
+func sliceInto(slice []fclient.PublicRoom, since int64, limit int64) (subset []fclient.PublicRoom, prev, next int) {
 	prev = -1
 	next = -1
 
@@ -241,10 +242,10 @@ func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) (
 func refreshPublicRoomCache(
 	ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
 	request PublicRoomReq,
-) []gomatrixserverlib.PublicRoom {
+) []fclient.PublicRoom {
 	cacheMu.Lock()
 	defer cacheMu.Unlock()
-	var extraRooms []gomatrixserverlib.PublicRoom
+	var extraRooms []fclient.PublicRoom
 	if extRoomsProvider != nil {
 		extraRooms = extRoomsProvider.Rooms()
 	}
@@ -269,7 +270,7 @@ func refreshPublicRoomCache(
 		util.GetLogger(ctx).WithError(err).Error("PopulatePublicRooms failed")
 		return publicRoomsCache
 	}
-	publicRoomsCache = []gomatrixserverlib.PublicRoom{}
+	publicRoomsCache = []fclient.PublicRoom{}
 	publicRoomsCache = append(publicRoomsCache, pubRooms...)
 	publicRoomsCache = append(publicRoomsCache, extraRooms...)
 	publicRoomsCache = dedupeAndShuffle(publicRoomsCache)
@@ -281,16 +282,16 @@ func refreshPublicRoomCache(
 	return publicRoomsCache
 }
 
-func getPublicRoomsFromCache() []gomatrixserverlib.PublicRoom {
+func getPublicRoomsFromCache() []fclient.PublicRoom {
 	cacheMu.Lock()
 	defer cacheMu.Unlock()
 	return publicRoomsCache
 }
 
-func dedupeAndShuffle(in []gomatrixserverlib.PublicRoom) []gomatrixserverlib.PublicRoom {
+func dedupeAndShuffle(in []fclient.PublicRoom) []fclient.PublicRoom {
 	// de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers
 	// are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit)
-	var publicRooms []gomatrixserverlib.PublicRoom
+	var publicRooms []fclient.PublicRoom
 	haveRoomIDs := make(map[string]bool)
 	rand.Shuffle(len(in), func(i, j int) {
 		in[i], in[j] = in[j], in[i]

clientapi/routing/directory_public_test.go

@@ -4,17 +4,17 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 )
 
-func pubRoom(name string) gomatrixserverlib.PublicRoom {
-	return gomatrixserverlib.PublicRoom{
+func pubRoom(name string) fclient.PublicRoom {
+	return fclient.PublicRoom{
 		Name: name,
 	}
 }
 
 func TestSliceInto(t *testing.T) {
-	slice := []gomatrixserverlib.PublicRoom{
+	slice := []fclient.PublicRoom{
 		pubRoom("a"), pubRoom("b"), pubRoom("c"), pubRoom("d"), pubRoom("e"), pubRoom("f"), pubRoom("g"),
 	}
 	limit := int64(3)
@@ -22,7 +22,7 @@ func TestSliceInto(t *testing.T) {
 		since      int64
 		wantPrev   int
 		wantNext   int
-		wantSubset []gomatrixserverlib.PublicRoom
+		wantSubset []fclient.PublicRoom
 	}{
 		{
 			since:      0,

clientapi/routing/joinroom.go

@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"time"
 
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
@@ -61,21 +62,19 @@ func JoinRoomByIDOrAlias(
 	// Work out our localpart for the client profile request.
 
 	// Request our profile content to populate the request content with.
-	res := &api.QueryProfileResponse{}
-	err := profileAPI.QueryProfile(req.Context(), &api.QueryProfileRequest{UserID: device.UserID}, res)
-	if err != nil || !res.UserExists {
-		if !res.UserExists {
+	profile, err := profileAPI.QueryProfile(req.Context(), device.UserID)
+
+	switch err {
+	case nil:
+		joinReq.Content["displayname"] = profile.DisplayName
+		joinReq.Content["avatar_url"] = profile.AvatarURL
+	case appserviceAPI.ErrProfileNotExists:
 		util.GetLogger(req.Context()).Error("Unable to query user profile, no profile found.")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.Unknown("Unable to query user profile, no profile found."),
 		}
-		}
-
-		util.GetLogger(req.Context()).WithError(err).Error("UserProfileAPI.QueryProfile failed")
-	} else {
-		joinReq.Content["displayname"] = res.DisplayName
-		joinReq.Content["avatar_url"] = res.AvatarURL
+	default:
 	}
 
 	// Ask the roomserver to perform the join.

clientapi/routing/joinroom_test.go

@@ -7,6 +7,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/appservice"
@@ -24,12 +27,15 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 
 	ctx := context.Background()
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
-		defer baseClose()
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
 
-		rsAPI := roomserver.NewInternalAPI(base)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
-		asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
 
 		// Create the users in the userapi
@@ -61,7 +67,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 			RoomAliasName: "alias",
 			Invite:        []string{bob.ID},
 			GuestCanJoin:  false,
-		}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
+		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
 		crResp, ok := resp.JSON.(createRoomResponse)
 		if !ok {
 			t.Fatalf("response is not a createRoomResponse: %+v", resp)
@@ -76,7 +82,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 			Preset:       presetPublicChat,
 			Invite:       []string{charlie.ID},
 			GuestCanJoin: true,
-		}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
+		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
 		crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse)
 		if !ok {
 			t.Fatalf("response is not a createRoomResponse: %+v", resp)

clientapi/routing/key_crosssigning.go

@@ -32,7 +32,7 @@ type crossSigningRequest struct {
 }
 
 func UploadCrossSigningDeviceKeys(
-	req *http.Request, userInteractiveAuth *auth.UserInteractive,
+	req *http.Request,
 	keyserverAPI api.ClientKeyAPI, device *api.Device,
 	accountAPI api.ClientUserAPI, cfg *config.ClientAPI,
 ) util.JSONResponse {
@@ -63,6 +63,7 @@ func UploadCrossSigningDeviceKeys(
 	}
 	typePassword := auth.LoginTypePassword{
 		UserApi: accountAPI,
+		UserAPI: accountAPI,
 		Config:  cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {

clientapi/routing/login_test.go

@@ -7,11 +7,17 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/test"
@@ -28,20 +34,24 @@ func TestLogin(t *testing.T) {
 
 	ctx := context.Background()
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
-		defer baseClose()
-		base.Cfg.ClientAPI.RateLimiting.Enabled = false
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
+		cfg.ClientAPI.RateLimiting.Enabled = false
+		natsInstance := jetstream.NATSInstance{}
 		// add a vhost
-		base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
+		cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{
+			SigningIdentity: fclient.SigningIdentity{ServerName: "vh1"},
 		})
 
-		rsAPI := roomserver.NewInternalAPI(base)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		routers := httputil.NewRouters()
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		// Needed for /login
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
-		Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, &base.Cfg.MSCs, nil)
+		Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics)
 
 		// Create password
 		password := util.RandomString(8)
@@ -114,7 +124,7 @@ func TestLogin(t *testing.T) {
 					"password": password,
 				}))
 				rec := httptest.NewRecorder()
-				base.PublicClientAPIMux.ServeHTTP(rec, req)
+				routers.Client.ServeHTTP(rec, req)
 				if tc.wantOK && rec.Code != http.StatusOK {
 					t.Fatalf("failed to login: %s", rec.Body.String())
 				}

clientapi/routing/membership.go

@@ -16,11 +16,12 @@ package routing
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -31,76 +32,56 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/util"
 )
 
-var errMissingUserID = errors.New("'user_id' must be supplied")
-
 func SendBan(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
-	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
 		return *reqErr
 	}
 
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing user_id"),
+		}
+	}
+
 	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
 	if errRes != nil {
 		return *errRes
 	}
 
-	plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
-		EventType: gomatrixserverlib.MRoomPowerLevels,
-		StateKey:  "",
-	})
-	if plEvent == nil {
-		return util.JSONResponse{
-			Code: 403,
-			JSON: jsonerror.Forbidden("You don't have permission to ban this user, no power_levels event in this room."),
-		}
-	}
-	pl, err := plEvent.PowerLevels()
-	if err != nil {
-		return util.JSONResponse{
-			Code: 403,
-			JSON: jsonerror.Forbidden("You don't have permission to ban this user, the power_levels event for this room is malformed so auth checks cannot be performed."),
-		}
+	pl, errRes := getPowerlevels(req, rsAPI, roomID)
+	if errRes != nil {
+		return *errRes
 	}
 	allowedToBan := pl.UserLevel(device.UserID) >= pl.Ban
 	if !allowedToBan {
 		return util.JSONResponse{
-			Code: 403,
+			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("You don't have permission to ban this user, power level too low."),
 		}
 	}
 
-	return sendMembership(req.Context(), profileAPI, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+	return sendMembership(req.Context(), profileAPI, device, roomID, gomatrixserverlib.Ban, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI)
 }
 
 func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time,
-	roomVer gomatrixserverlib.RoomVersion,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI) util.JSONResponse {
 
 	event, err := buildMembershipEvent(
 		ctx, targetUserID, reason, profileAPI, device, membership,
 		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
-	if err == errMissingUserID {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
-		}
-	} else if err == eventutil.ErrRoomNoExists {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(err.Error()),
-		}
-	} else if err != nil {
+	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
 		return jsonerror.InternalServerError()
 	}
@@ -109,7 +90,7 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic
 	if err = roomserverAPI.SendEvents(
 		ctx, rsAPI,
 		roomserverAPI.KindNew,
-		[]*gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)},
+		[]*gomatrixserverlib.HeaderedEvent{event},
 		device.UserDomain(),
 		serverName,
 		serverName,
@@ -131,13 +112,65 @@ func SendKick(
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
-	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
+	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
 		return *reqErr
 	}
 	if body.UserID == "" {
 		return util.JSONResponse{
-			Code: 400,
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing user_id"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+
+	pl, errRes := getPowerlevels(req, rsAPI, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+	allowedToKick := pl.UserLevel(device.UserID) >= pl.Kick
+	if !allowedToKick {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You don't have permission to kick this user, power level too low."),
+		}
+	}
+
+	var queryRes roomserverAPI.QueryMembershipForUserResponse
+	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+		RoomID: roomID,
+		UserID: body.UserID,
+	}, &queryRes)
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	// kick is only valid if the user is not currently banned or left (that is, they are joined or invited)
+	if queryRes.Membership != gomatrixserverlib.Join && queryRes.Membership != gomatrixserverlib.Invite {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Unknown("cannot /kick banned or left users"),
+		}
+	}
+	// TODO: should we be using SendLeave instead?
+	return sendMembership(req.Context(), profileAPI, device, roomID, gomatrixserverlib.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI)
+}
+
+func SendUnban(
+	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
+	roomID string, cfg *config.ClientAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
+) util.JSONResponse {
+	body, evTime, reqErr := extractRequestData(req)
+	if reqErr != nil {
+		return *reqErr
+	}
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
 			JSON: jsonerror.BadJSON("missing user_id"),
 		}
 	}
@@ -155,56 +188,16 @@ func SendKick(
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	// kick is only valid if the user is not currently banned or left (that is, they are joined or invited)
-	if queryRes.Membership != "join" && queryRes.Membership != "invite" {
-		return util.JSONResponse{
-			Code: 403,
-			JSON: jsonerror.Unknown("cannot /kick banned or left users"),
-		}
-	}
-	// TODO: should we be using SendLeave instead?
-	return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
-}
 
-func SendUnban(
-	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
-	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
-) util.JSONResponse {
-	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
-	if reqErr != nil {
-		return *reqErr
-	}
-	if body.UserID == "" {
-		return util.JSONResponse{
-			Code: 400,
-			JSON: jsonerror.BadJSON("missing user_id"),
-		}
-	}
-
-	var queryRes roomserverAPI.QueryMembershipForUserResponse
-	err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
-		RoomID: roomID,
-		UserID: body.UserID,
-	}, &queryRes)
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-	if !queryRes.RoomExists {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("room does not exist"),
-		}
-	}
 	// unban is only valid if the user is currently banned
-	if queryRes.Membership != "ban" {
+	if queryRes.Membership != gomatrixserverlib.Ban {
 		return util.JSONResponse{
-			Code: 400,
+			Code: http.StatusBadRequest,
 			JSON: jsonerror.Unknown("can only /unban users that are banned"),
 		}
 	}
 	// TODO: should we be using SendLeave instead?
-	return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
+	return sendMembership(req.Context(), profileAPI, device, roomID, gomatrixserverlib.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI)
 }
 
 func SendInvite(
@@ -212,7 +205,7 @@ func SendInvite(
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
-	body, evTime, _, reqErr := extractRequestData(req, roomID, rsAPI)
+	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
 		return *reqErr
 	}
@@ -234,6 +227,18 @@ func SendInvite(
 		}
 	}
 
+	if body.UserID == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing user_id"),
+		}
+	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+
 	// We already received the return value, so no need to check for an error here.
 	response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime)
 	return response
@@ -250,20 +255,10 @@ func sendInvite(
 	asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
 ) (util.JSONResponse, error) {
 	event, err := buildMembershipEvent(
-		ctx, userID, reason, profileAPI, device, "invite",
+		ctx, userID, reason, profileAPI, device, gomatrixserverlib.Invite,
 		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
-	if err == errMissingUserID {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
-		}, err
-	} else if err == eventutil.ErrRoomNoExists {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(err.Error()),
-		}, err
-	} else if err != nil {
+	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
 		return jsonerror.InternalServerError(), err
 	}
@@ -357,19 +352,7 @@ func loadProfile(
 	return profile, err
 }
 
-func extractRequestData(req *http.Request, roomID string, rsAPI roomserverAPI.ClientRoomserverAPI) (
-	body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse,
-) {
-	verReq := roomserverAPI.QueryRoomVersionForRoomRequest{RoomID: roomID}
-	verRes := roomserverAPI.QueryRoomVersionForRoomResponse{}
-	if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
-		resErr = &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
-		}
-		return
-	}
-	roomVer = verRes.RoomVersion
+func extractRequestData(req *http.Request) (body *threepid.MembershipRequest, evTime time.Time, resErr *util.JSONResponse) {
 
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		resErr = reqErr
@@ -432,34 +415,17 @@ func checkAndProcessThreepid(
 }
 
 func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID, roomID string) *util.JSONResponse {
-	tuple := gomatrixserverlib.StateKeyTuple{
-		EventType: gomatrixserverlib.MRoomMember,
-		StateKey:  userID,
-	}
-	var membershipRes roomserverAPI.QueryCurrentStateResponse
-	err := rsAPI.QueryCurrentState(ctx, &roomserverAPI.QueryCurrentStateRequest{
+	var membershipRes roomserverAPI.QueryMembershipForUserResponse
+	err := rsAPI.QueryMembershipForUser(ctx, &roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
-		StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
+		UserID: userID,
 	}, &membershipRes)
 	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("QueryCurrentState: could not query membership for user")
+		util.GetLogger(ctx).WithError(err).Error("QueryMembershipForUser: could not query membership for user")
 		e := jsonerror.InternalServerError()
 		return &e
 	}
-	ev := membershipRes.StateEvents[tuple]
-	if ev == nil {
-		return &util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("user does not belong to room"),
-		}
-	}
-	membership, err := ev.Membership()
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("Member event isn't valid")
-		e := jsonerror.InternalServerError()
-		return &e
-	}
-	if membership != gomatrixserverlib.Join {
+	if !membershipRes.IsInRoom {
 		return &util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("user does not belong to room"),
@@ -511,3 +477,24 @@ func SendForget(
 		JSON: struct{}{},
 	}
 }
+
+func getPowerlevels(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, roomID string) (*gomatrixserverlib.PowerLevelContent, *util.JSONResponse) {
+	plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomPowerLevels,
+		StateKey:  "",
+	})
+	if plEvent == nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You don't have permission to perform this action, no power_levels event in this room."),
+		}
+	}
+	pl, err := plEvent.PowerLevels()
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You don't have permission to perform this action, the power_levels event for this room is malformed so auth checks cannot be performed."),
+		}
+	}
+	return pl, nil
+}

clientapi/routing/password.go

@@ -88,7 +88,7 @@ func Password(
 			bound bool
 			err   error
 		)
-		bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg)
+		bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg, nil)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
 			return jsonerror.InternalServerError()

clientapi/routing/profile.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@@ -36,14 +37,14 @@ import (
 
 // GetProfile implements GET /profile/{userID}
 func GetProfile(
-	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
 	if err != nil {
-		if err == eventutil.ErrProfileNoExists {
+		if err == appserviceAPI.ErrProfileNotExists {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
 				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
@@ -56,7 +57,7 @@ func GetProfile(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: eventutil.ProfileResponse{
+		JSON: eventutil.UserProfile{
 			AvatarURL:   profile.AvatarURL,
 			DisplayName: profile.DisplayName,
 		},
@@ -65,34 +66,28 @@ func GetProfile(
 
 // GetAvatarURL implements GET /profile/{userID}/avatar_url
 func GetAvatarURL(
-	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 ) util.JSONResponse {
-	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
-	if err != nil {
-		if err == eventutil.ErrProfileNoExists {
-			return util.JSONResponse{
-				Code: http.StatusNotFound,
-				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
-			}
-		}
-
-		util.GetLogger(req.Context()).WithError(err).Error("getProfile failed")
-		return jsonerror.InternalServerError()
+	profile := GetProfile(req, profileAPI, cfg, userID, asAPI, federation)
+	p, ok := profile.JSON.(eventutil.UserProfile)
+	// not a profile response, so most likely an error, return that
+	if !ok {
+		return profile
 	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: eventutil.AvatarURL{
-			AvatarURL: profile.AvatarURL,
+		JSON: eventutil.UserProfile{
+			AvatarURL: p.AvatarURL,
 		},
 	}
 }
 
 // SetAvatarURL implements PUT /profile/{userID}/avatar_url
 func SetAvatarURL(
-	req *http.Request, profileAPI userapi.ClientUserAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI,
 	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -102,7 +97,7 @@ func SetAvatarURL(
 		}
 	}
 
-	var r eventutil.AvatarURL
+	var r eventutil.UserProfile
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
@@ -128,24 +123,20 @@ func SetAvatarURL(
 		}
 	}
 
-	setRes := &userapi.PerformSetAvatarURLResponse{}
-	if err = profileAPI.SetAvatarURL(req.Context(), &userapi.PerformSetAvatarURLRequest{
-		Localpart:  localpart,
-		ServerName: domain,
-		AvatarURL:  r.AvatarURL,
-	}, setRes); err != nil {
+	profile, changed, err := profileAPI.SetAvatarURL(req.Context(), localpart, domain, r.AvatarURL)
+	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetAvatarURL failed")
 		return jsonerror.InternalServerError()
 	}
 	// No need to build new membership events, since nothing changed
-	if !setRes.Changed {
+	if !changed {
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: struct{}{},
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, setRes.Profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
 	if err != nil {
 		return response
 	}
@@ -158,34 +149,28 @@ func SetAvatarURL(
 
 // GetDisplayName implements GET /profile/{userID}/displayname
 func GetDisplayName(
-	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 ) util.JSONResponse {
-	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
-	if err != nil {
-		if err == eventutil.ErrProfileNoExists {
-			return util.JSONResponse{
-				Code: http.StatusNotFound,
-				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
-			}
-		}
-
-		util.GetLogger(req.Context()).WithError(err).Error("getProfile failed")
-		return jsonerror.InternalServerError()
+	profile := GetProfile(req, profileAPI, cfg, userID, asAPI, federation)
+	p, ok := profile.JSON.(eventutil.UserProfile)
+	// not a profile response, so most likely an error, return that
+	if !ok {
+		return profile
 	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: eventutil.DisplayName{
-			DisplayName: profile.DisplayName,
+		JSON: eventutil.UserProfile{
+			DisplayName: p.DisplayName,
 		},
 	}
 }
 
 // SetDisplayName implements PUT /profile/{userID}/displayname
 func SetDisplayName(
-	req *http.Request, profileAPI userapi.ClientUserAPI,
+	req *http.Request, profileAPI userapi.ProfileAPI,
 	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -195,7 +180,7 @@ func SetDisplayName(
 		}
 	}
 
-	var r eventutil.DisplayName
+	var r eventutil.UserProfile
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
@@ -227,25 +212,20 @@ func SetDisplayName(
 		}
 	}
 
-	profileRes := &userapi.PerformUpdateDisplayNameResponse{}
-	err = profileAPI.SetDisplayName(req.Context(), &userapi.PerformUpdateDisplayNameRequest{
-		Localpart:   localpart,
-		ServerName:  domain,
-		DisplayName: r.DisplayName,
-	}, profileRes)
+	profile, changed, err := profileAPI.SetDisplayName(req.Context(), localpart, domain, r.DisplayName)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetDisplayName failed")
 		return jsonerror.InternalServerError()
 	}
 	// No need to build new membership events, since nothing changed
-	if !profileRes.Changed {
+	if !changed {
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: struct{}{},
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, profileRes.Profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
 	if err != nil {
 		return response
 	}
@@ -302,12 +282,12 @@ func updateProfile(
 // getProfile gets the full profile of a user by querying the database or a
 // remote homeserver.
 // Returns an error when something goes wrong or specifically
-// eventutil.ErrProfileNoExists when the profile doesn't exist.
+// eventutil.ErrProfileNotExists when the profile doesn't exist.
 func getProfile(
-	ctx context.Context, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	ctx context.Context, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 ) (*authtypes.Profile, error) {
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
@@ -319,7 +299,7 @@ func getProfile(
 		if fedErr != nil {
 			if x, ok := fedErr.(gomatrix.HTTPError); ok {
 				if x.Code == http.StatusNotFound {
-					return nil, eventutil.ErrProfileNoExists
+					return nil, appserviceAPI.ErrProfileNotExists
 				}
 			}
 

clientapi/routing/register.go

@@ -734,7 +734,7 @@ func handleRegistrationFlow(
 			bound bool
 			err   error
 		)
-		bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg)
+		bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg, nil)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
 			return jsonerror.InternalServerError()
@@ -930,13 +930,7 @@ func completeRegistration(
 	}
 
 	if displayName != "" {
-		nameReq := userapi.PerformUpdateDisplayNameRequest{
-			Localpart:   username,
-			ServerName:  serverName,
-			DisplayName: displayName,
-		}
-		var nameRes userapi.PerformUpdateDisplayNameResponse
-		err = userAPI.SetDisplayName(ctx, &nameReq, &nameRes)
+		_, _, err = userAPI.SetDisplayName(ctx, username, serverName, displayName)
 		if err != nil {
 			return util.JSONResponse{
 				Code: http.StatusInternalServerError,

clientapi/routing/register_test.go

@@ -30,8 +30,11 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi"
@@ -404,11 +407,15 @@ func Test_register(t *testing.T) {
 	}
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
-		defer baseClose()
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
 
-		rsAPI := roomserver.NewInternalAPI(base)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
@@ -430,16 +437,16 @@ func Test_register(t *testing.T) {
 						}
 					}))
 					defer srv.Close()
-					base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
+					cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
 				}
 
-				if err := base.Cfg.Derive(); err != nil {
+				if err := cfg.Derive(); err != nil {
 					t.Fatalf("failed to derive config: %s", err)
 				}
 
-				base.Cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha
-				base.Cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled
-				base.Cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled
+				cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha
+				cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled
+				cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled
 
 				if tc.kind == "" {
 					tc.kind = "user"
@@ -467,15 +474,15 @@ func Test_register(t *testing.T) {
 
 				req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/?kind=%s", tc.kind), body)
 
-				resp := Register(req, userAPI, &base.Cfg.ClientAPI)
+				resp := Register(req, userAPI, &cfg.ClientAPI)
 				t.Logf("Resp: %+v", resp)
 
 				// The first request should return a userInteractiveResponse
 				switch r := resp.JSON.(type) {
 				case userInteractiveResponse:
 					// Check that the flows are the ones we configured
-					if !reflect.DeepEqual(r.Flows, base.Cfg.Derived.Registration.Flows) {
-						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, base.Cfg.Derived.Registration.Flows)
+					if !reflect.DeepEqual(r.Flows, cfg.Derived.Registration.Flows) {
+						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows)
 					}
 				case *jsonerror.MatrixError:
 					if !reflect.DeepEqual(tc.wantResponse, resp) {
@@ -531,7 +538,7 @@ func Test_register(t *testing.T) {
 
 				req = httptest.NewRequest(http.MethodPost, "/", body)
 
-				resp = Register(req, userAPI, &base.Cfg.ClientAPI)
+				resp = Register(req, userAPI, &cfg.ClientAPI)
 
 				switch resp.JSON.(type) {
 				case *jsonerror.MatrixError:
@@ -574,16 +581,19 @@ func Test_register(t *testing.T) {
 
 func TestRegisterUserWithDisplayName(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
-		defer baseClose()
-		base.Cfg.Global.ServerName = "server"
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
+		cfg.Global.ServerName = "server"
 
-		rsAPI := roomserver.NewInternalAPI(base)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 		deviceName, deviceID := "deviceName", "deviceID"
 		expectedDisplayName := "DisplayName"
 		response := completeRegistration(
-			base.Context(),
+			processCtx.Context(),
 			userAPI,
 			"user",
 			"server",
@@ -602,24 +612,25 @@ func TestRegisterUserWithDisplayName(t *testing.T) {
 
 		assert.Equal(t, http.StatusOK, response.Code)
 
-		req := api.QueryProfileRequest{UserID: "@user:server"}
-		var res api.QueryProfileResponse
-		err := userAPI.QueryProfile(base.Context(), &req, &res)
+		profile, err := userAPI.QueryProfile(processCtx.Context(), "@user:server")
 		assert.NoError(t, err)
-		assert.Equal(t, expectedDisplayName, res.DisplayName)
+		assert.Equal(t, expectedDisplayName, profile.DisplayName)
 	})
 }
 
 func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
-		defer baseClose()
-		base.Cfg.Global.ServerName = "server"
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		cfg.Global.ServerName = "server"
 		sharedSecret := "dendritetest"
-		base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret
+		cfg.ClientAPI.RegistrationSharedSecret = sharedSecret
 
-		rsAPI := roomserver.NewInternalAPI(base)
-		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		expectedDisplayName := "rabbit"
 		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
@@ -643,17 +654,15 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 		ssrr := httptest.NewRequest(http.MethodPost, "/", body)
 
 		response := handleSharedSecretRegistration(
-			&base.Cfg.ClientAPI,
+			&cfg.ClientAPI,
 			userAPI,
 			r,
 			ssrr,
 		)
 		assert.Equal(t, http.StatusOK, response.Code)
 
-		profilReq := api.QueryProfileRequest{UserID: "@alice:server"}
-		var profileRes api.QueryProfileResponse
-		err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes)
+		profile, err := userAPI.QueryProfile(processCtx.Context(), "@alice:server")
 		assert.NoError(t, err)
-		assert.Equal(t, expectedDisplayName, profileRes.DisplayName)
+		assert.Equal(t, expectedDisplayName, profile.DisplayName)
 	})
 }

clientapi/routing/routing.go

@@ -18,12 +18,12 @@ import (
 	"context"
 	"net/http"
 	"strings"
-	"sync"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/setup/base"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	"github.com/prometheus/client_golang/prometheus"
@@ -51,25 +51,27 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	base *base.BaseDendrite,
-	cfg *config.ClientAPI,
+	routers httputil.Routers,
+	dendriteCfg *config.Dendrite,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.QuerySearchProfilesAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
 	federationSender federationAPI.ClientFederationAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
-	mscCfg *config.MSCs, natsClient *nats.Conn,
+	natsClient *nats.Conn, enableMetrics bool,
 ) {
-	publicAPIMux := base.PublicClientAPIMux
-	wkMux := base.PublicWellKnownAPIMux
-	synapseAdminRouter := base.SynapseAdminMux
-	dendriteAdminRouter := base.DendriteAdminMux
+	cfg := &dendriteCfg.ClientAPI
+	mscCfg := &dendriteCfg.MSCs
+	publicAPIMux := routers.Client
+	wkMux := routers.WellKnown
+	synapseAdminRouter := routers.SynapseAdmin
+	dendriteAdminRouter := routers.DendriteAdmin
 
-	if base.EnableMetrics {
+	if enableMetrics {
 		prometheus.MustRegister(amtRegUsers, sendEventDuration)
 	}
 
@@ -156,15 +158,15 @@ func Setup(
 
 	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
 		httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateRoom(req, cfg, device, rsAPI)
+			return AdminEvacuateRoom(req, rsAPI)
 		}),
-	).Methods(http.MethodGet, http.MethodOptions)
+	).Methods(http.MethodPost, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}",
 		httputil.MakeAdminAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateUser(req, cfg, device, rsAPI)
+			return AdminEvacuateUser(req, cfg, rsAPI)
 		}),
-	).Methods(http.MethodGet, http.MethodOptions)
+	).Methods(http.MethodPost, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/purgeRoom/{roomID}",
 		httputil.MakeAdminAPI("admin_purge_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -199,18 +201,13 @@ func Setup(
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
-		var serverNotificationSender *userapi.Device
-		var err error
-		notificationSenderOnce := &sync.Once{}
-
-		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
-			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-				notificationSenderOnce.Do(func() {
-					serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
+		serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
 		if err != nil {
 			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
 		}
-				})
+
+		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
+			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 				// not specced, but ensure we're rate limiting requests to this endpoint
 				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
@@ -232,12 +229,6 @@ func Setup(
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice",
 			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-				notificationSenderOnce.Do(func() {
-					serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
-					if err != nil {
-						logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
-					}
-				})
 				// not specced, but ensure we're rate limiting requests to this endpoint
 				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
@@ -669,7 +660,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	v3mux.Handle("/auth/{authType}/fallback/web",
-		httputil.MakeHTMLAPI("auth_fallback", base.EnableMetrics, func(w http.ResponseWriter, req *http.Request) {
+		httputil.MakeHTMLAPI("auth_fallback", enableMetrics, func(w http.ResponseWriter, req *http.Request) {
 			vars := mux.Vars(req)
 			AuthFallback(w, req, vars["authType"], cfg)
 		}),
@@ -873,6 +864,8 @@ func Setup(
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
 	// PUT requests, so we need to allow this method
 
+	threePIDClient := base.CreateClient(dendriteCfg, nil) // TODO: Move this somewhere else, e.g. pass in as parameter
+
 	v3mux.Handle("/account/3pid",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetAssociated3PIDs(req, userAPI, device)
@@ -881,11 +874,11 @@ func Setup(
 
 	v3mux.Handle("/account/3pid",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return CheckAndSave3PIDAssociation(req, userAPI, device, cfg)
+			return CheckAndSave3PIDAssociation(req, userAPI, device, cfg, threePIDClient)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	unstableMux.Handle("/account/3pid/delete",
+	v3mux.Handle("/account/3pid/delete",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return Forget3PID(req, userAPI)
 		}),
@@ -893,7 +886,7 @@ func Setup(
 
 	v3mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken",
 		httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse {
-			return RequestEmailToken(req, userAPI, cfg)
+			return RequestEmailToken(req, userAPI, cfg, threePIDClient)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -1127,7 +1120,7 @@ func Setup(
 
 	v3mux.Handle("/delete_devices",
 		httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return DeleteDevices(req, userAPI, device)
+			return DeleteDevices(req, userInteractiveAuth, userAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -1206,7 +1199,7 @@ func Setup(
 			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
-			return GetCapabilities(req, rsAPI)
+			return GetCapabilities()
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -1383,7 +1376,7 @@ func Setup(
 	// Cross-signing device keys
 
 	postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg)
+		return UploadCrossSigningDeviceKeys(req, userAPI, device, userAPI, cfg)
 	})
 
 	postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {

clientapi/routing/sendtyping.go

@@ -15,12 +15,13 @@ package routing
 import (
 	"net/http"
 
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/util"
 )
 
 type typingContentJSON struct {

clientapi/routing/server_notices.go

@@ -295,30 +295,28 @@ func getSenderDevice(
 	}
 
 	// Set the avatarurl for the user
-	avatarRes := &userapi.PerformSetAvatarURLResponse{}
-	if err = userAPI.SetAvatarURL(ctx, &userapi.PerformSetAvatarURLRequest{
-		Localpart:  cfg.Matrix.ServerNotices.LocalPart,
-		ServerName: cfg.Matrix.ServerName,
-		AvatarURL:  cfg.Matrix.ServerNotices.AvatarURL,
-	}, avatarRes); err != nil {
+	profile, avatarChanged, err := userAPI.SetAvatarURL(ctx,
+		cfg.Matrix.ServerNotices.LocalPart,
+		cfg.Matrix.ServerName,
+		cfg.Matrix.ServerNotices.AvatarURL,
+	)
+	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.SetAvatarURL failed")
 		return nil, err
 	}
 
-	profile := avatarRes.Profile
-
 	// Set the displayname for the user
-	displayNameRes := &userapi.PerformUpdateDisplayNameResponse{}
-	if err = userAPI.SetDisplayName(ctx, &userapi.PerformUpdateDisplayNameRequest{
-		Localpart:   cfg.Matrix.ServerNotices.LocalPart,
-		ServerName:  cfg.Matrix.ServerName,
-		DisplayName: cfg.Matrix.ServerNotices.DisplayName,
-	}, displayNameRes); err != nil {
+	_, displayNameChanged, err := userAPI.SetDisplayName(ctx,
+		cfg.Matrix.ServerNotices.LocalPart,
+		cfg.Matrix.ServerName,
+		cfg.Matrix.ServerNotices.DisplayName,
+	)
+	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.SetDisplayName failed")
 		return nil, err
 	}
 
-	if displayNameRes.Changed {
+	if displayNameChanged {
 		profile.DisplayName = cfg.Matrix.ServerNotices.DisplayName
 	}
 
@@ -334,7 +332,7 @@ func getSenderDevice(
 	// We've got an existing account, return the first device of it
 	if len(deviceRes.Devices) > 0 {
 		// If there were changes to the profile, create a new membership event
-		if displayNameRes.Changed || avatarRes.Changed {
+		if displayNameChanged || avatarChanged {
 			_, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, cfg, time.Now())
 			if err != nil {
 				return nil, err

clientapi/routing/state.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/synctypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -29,7 +30,7 @@ import (
 )
 
 type stateEventInStateResp struct {
-	gomatrixserverlib.ClientEvent
+	synctypes.ClientEvent
 	PrevContent   json.RawMessage `json:"prev_content,omitempty"`
 	ReplacesState string          `json:"replaces_state,omitempty"`
 }
@@ -122,7 +123,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		"state_at_event": !wantLatestState,
 	}).Info("Fetching all state")
 
-	stateEvents := []gomatrixserverlib.ClientEvent{}
+	stateEvents := []synctypes.ClientEvent{}
 	if wantLatestState {
 		// If we are happy to use the latest state, either because the user is
 		// still in the room, or because the room is world-readable, then just
@@ -131,7 +132,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		for _, ev := range stateRes.StateEvents {
 			stateEvents = append(
 				stateEvents,
-				gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll),
+				synctypes.HeaderedToClientEvent(ev, synctypes.FormatAll),
 			)
 		}
 	} else {
@@ -150,7 +151,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		for _, ev := range stateAfterRes.StateEvents {
 			stateEvents = append(
 				stateEvents,
-				gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll),
+				synctypes.HeaderedToClientEvent(ev, synctypes.FormatAll),
 			)
 		}
 	}
@@ -309,7 +310,7 @@ func OnIncomingStateTypeRequest(
 	}
 
 	stateEvent := stateEventInStateResp{
-		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(event, gomatrixserverlib.FormatAll),
+		ClientEvent: synctypes.HeaderedToClientEvent(event, synctypes.FormatAll),
 	}
 
 	var res interface{}

clientapi/routing/threepid.go

@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	userdb "github.com/matrix-org/dendrite/userapi/storage"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -33,7 +34,7 @@ type reqTokenResponse struct {
 	SID string `json:"sid"`
 }
 
-type threePIDsResponse struct {
+type ThreePIDsResponse struct {
 	ThreePIDs []authtypes.ThreePID `json:"threepids"`
 }
 
@@ -41,7 +42,7 @@ type threePIDsResponse struct {
 //
 //	POST /account/3pid/email/requestToken
 //	POST /register/email/requestToken
-func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI) util.JSONResponse {
+func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI, client *fclient.Client) util.JSONResponse {
 	var body threepid.EmailAssociationRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr
@@ -72,7 +73,7 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *co
 		}
 	}
 
-	resp.SID, err = threepid.CreateSession(req.Context(), body, cfg)
+	resp.SID, err = threepid.CreateSession(req.Context(), body, cfg, client)
 	if err == threepid.ErrNotTrusted {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -92,7 +93,7 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *co
 // CheckAndSave3PIDAssociation implements POST /account/3pid
 func CheckAndSave3PIDAssociation(
 	req *http.Request, threePIDAPI api.ClientUserAPI, device *api.Device,
-	cfg *config.ClientAPI,
+	cfg *config.ClientAPI, client *fclient.Client,
 ) util.JSONResponse {
 	var body threepid.EmailAssociationCheckRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
@@ -100,7 +101,7 @@ func CheckAndSave3PIDAssociation(
 	}
 
 	// Check if the association has been validated
-	verified, address, medium, err := threepid.CheckAssociation(req.Context(), body.Creds, cfg)
+	verified, address, medium, err := threepid.CheckAssociation(req.Context(), body.Creds, cfg, client)
 	if err == threepid.ErrNotTrusted {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -123,13 +124,8 @@ func CheckAndSave3PIDAssociation(
 
 	if body.Bind {
 		// Publish the association on the identity server if requested
-		err = threepid.PublishAssociation(body.Creds, device.UserID, cfg)
-		if err == threepid.ErrNotTrusted {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: jsonerror.NotTrusted(body.Creds.IDServer),
-			}
-		} else if err != nil {
+		err = threepid.PublishAssociation(req.Context(), body.Creds, device.UserID, cfg, client)
+		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("threepid.PublishAssociation failed")
 			return jsonerror.InternalServerError()
 		}
@@ -180,7 +176,7 @@ func GetAssociated3PIDs(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: threePIDsResponse{res.ThreePIDs},
+		JSON: ThreePIDsResponse{res.ThreePIDs},
 	}
 }
 
@@ -191,7 +187,10 @@ func Forget3PID(req *http.Request, threepidAPI api.ClientUserAPI) util.JSONRespo
 		return *reqErr
 	}
 
-	if err := threepidAPI.PerformForgetThreePID(req.Context(), &api.PerformForgetThreePIDRequest{}, &struct{}{}); err != nil {
+	if err := threepidAPI.PerformForgetThreePID(req.Context(), &api.PerformForgetThreePIDRequest{
+		ThreePID: body.Address,
+		Medium:   body.Medium,
+	}, &struct{}{}); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.PerformForgetThreePID failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/userdirectory.go

@@ -26,6 +26,7 @@ import (
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -41,7 +42,7 @@ func SearchUserDirectory(
 	provider userapi.QuerySearchProfilesAPI,
 	searchString string,
 	limit int,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 	localServerName gomatrixserverlib.ServerName,
 ) util.JSONResponse {
 	if limit < 10 {

clientapi/threepid/invites.go

@@ -209,24 +209,17 @@ func queryIDServerStoreInvite(
 	body *MembershipRequest, roomID string,
 ) (*idServerStoreInviteResponse, error) {
 	// Retrieve the sender's profile to get their display name
-	localpart, serverName, err := gomatrixserverlib.SplitID('@', device.UserID)
+	_, serverName, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		return nil, err
 	}
 
 	var profile *authtypes.Profile
 	if cfg.Matrix.IsLocalServerName(serverName) {
-		res := &userapi.QueryProfileResponse{}
-		err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res)
+		profile, err = userAPI.QueryProfile(ctx, device.UserID)
 		if err != nil {
 			return nil, err
 		}
-		profile = &authtypes.Profile{
-			Localpart:   localpart,
-			DisplayName: res.DisplayName,
-			AvatarURL:   res.AvatarURL,
-		}
-
 	} else {
 		profile = &authtypes.Profile{}
 	}

clientapi/threepid/threepid.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 )
 
 // EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken
@@ -37,7 +38,7 @@ type EmailAssociationRequest struct {
 
 // EmailAssociationCheckRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-account-3pid
 type EmailAssociationCheckRequest struct {
-	Creds Credentials `json:"threePidCreds"`
+	Creds Credentials `json:"three_pid_creds"`
 	Bind  bool        `json:"bind"`
 }
 
@@ -48,12 +49,16 @@ type Credentials struct {
 	Secret   string `json:"client_secret"`
 }
 
+type SID struct {
+	SID string `json:"sid"`
+}
+
 // CreateSession creates a session on an identity server.
 // Returns the session's ID.
 // Returns an error if there was a problem sending the request or decoding the
 // response, or if the identity server responded with a non-OK status.
 func CreateSession(
-	ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI,
+	ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI, client *fclient.Client,
 ) (string, error) {
 	if err := isTrusted(req.IDServer, cfg); err != nil {
 		return "", err
@@ -73,8 +78,7 @@ func CreateSession(
 	}
 	request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 
-	client := http.Client{}
-	resp, err := client.Do(request.WithContext(ctx))
+	resp, err := client.DoHTTPRequest(ctx, request)
 	if err != nil {
 		return "", err
 	}
@@ -85,14 +89,20 @@ func CreateSession(
 	}
 
 	// Extract the SID from the response and return it
-	var sid struct {
-		SID string `json:"sid"`
-	}
+	var sid SID
 	err = json.NewDecoder(resp.Body).Decode(&sid)
 
 	return sid.SID, err
 }
 
+type GetValidatedResponse struct {
+	Medium      string `json:"medium"`
+	ValidatedAt int64  `json:"validated_at"`
+	Address     string `json:"address"`
+	ErrCode     string `json:"errcode"`
+	Error       string `json:"error"`
+}
+
 // CheckAssociation checks the status of an ongoing association validation on an
 // identity server.
 // Returns a boolean set to true if the association has been validated, false if not.
@@ -102,6 +112,7 @@ func CreateSession(
 // response, or if the identity server responded with a non-OK status.
 func CheckAssociation(
 	ctx context.Context, creds Credentials, cfg *config.ClientAPI,
+	client *fclient.Client,
 ) (bool, string, string, error) {
 
 	requestURL := fmt.Sprintf("%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", cfg.ThreePidDelegate, creds.SID, creds.Secret)
@@ -109,19 +120,20 @@ func CheckAssociation(
 	if err != nil {
 		return false, "", "", err
 	}
-	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	var resp *http.Response
+	if client != nil {
+		resp, err = client.DoHTTPRequest(ctx, req)
+		if err != nil {
+			return false, "", "", err
+		}
+	} else {
+		resp, err = http.DefaultClient.Do(req.WithContext(ctx))
 		if err != nil {
 			return false, "", "", err
 		}
-
-	var respBody struct {
-		Medium      string `json:"medium"`
-		ValidatedAt int64  `json:"validated_at"`
-		Address     string `json:"address"`
-		ErrCode     string `json:"errcode"`
-		Error       string `json:"error"`
 	}
 
+	var respBody GetValidatedResponse
 	if err = json.NewDecoder(resp.Body).Decode(&respBody); err != nil {
 		return false, "", "", err
 	}
@@ -139,7 +151,7 @@ func CheckAssociation(
 // identifier and a Matrix ID.
 // Returns an error if there was a problem sending the request or decoding the
 // response, or if the identity server responded with a non-OK status.
-func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI) error {
+func PublishAssociation(ctx context.Context, creds Credentials, userID string, cfg *config.ClientAPI, client *fclient.Client) error {
 	if err := isTrusted(creds.IDServer, cfg); err != nil {
 		return err
 	}
@@ -157,8 +169,7 @@ func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI)
 	}
 	request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 
-	client := http.Client{}
-	resp, err := client.Do(request)
+	resp, err := client.DoHTTPRequest(ctx, request)
 	if err != nil {
 		return err
 	}

clientapi/userutil/userutil_test.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 )
 
 var (
@@ -30,7 +31,7 @@ var (
 // TestGoodUserID checks that correct localpart is returned for a valid user ID.
 func TestGoodUserID(t *testing.T) {
 	cfg := &config.Global{
-		SigningIdentity: gomatrixserverlib.SigningIdentity{
+		SigningIdentity: fclient.SigningIdentity{
 			ServerName: serverName,
 		},
 	}
@@ -49,7 +50,7 @@ func TestGoodUserID(t *testing.T) {
 // TestWithLocalpartOnly checks that localpart is returned when usernameParam contains only localpart.
 func TestWithLocalpartOnly(t *testing.T) {
 	cfg := &config.Global{
-		SigningIdentity: gomatrixserverlib.SigningIdentity{
+		SigningIdentity: fclient.SigningIdentity{
 			ServerName: serverName,
 		},
 	}
@@ -68,7 +69,7 @@ func TestWithLocalpartOnly(t *testing.T) {
 // TestIncorrectDomain checks for error when there's server name mismatch.
 func TestIncorrectDomain(t *testing.T) {
 	cfg := &config.Global{
-		SigningIdentity: gomatrixserverlib.SigningIdentity{
+		SigningIdentity: fclient.SigningIdentity{
 			ServerName: invalidServerName,
 		},
 	}
@@ -83,7 +84,7 @@ func TestIncorrectDomain(t *testing.T) {
 // TestBadUserID checks that ParseUsernameParam fails for invalid user ID
 func TestBadUserID(t *testing.T) {
 	cfg := &config.Global{
-		SigningIdentity: gomatrixserverlib.SigningIdentity{
+		SigningIdentity: fclient.SigningIdentity{
 			ServerName: serverName,
 		},
 	}

cmd/dendrite-demo-yggdrasil/main.go

@@ -27,6 +27,11 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/gorilla/mux"
@@ -41,7 +46,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/test"
@@ -57,6 +62,7 @@ var (
 	instanceDir    = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
 )
 
+// nolint: gocyclo
 func main() {
 	flag.Parse()
 	internal.SetupPprof()
@@ -142,37 +148,83 @@ func main() {
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
-	base := base.NewBaseDendrite(cfg)
-	base.ConfigureAdminEndpoints()
-	defer base.Close() // nolint: errcheck
+	configErrors := &config.ConfigErrors{}
+	cfg.Verify(configErrors)
+	if len(*configErrors) > 0 {
+		for _, err := range *configErrors {
+			logrus.Errorf("Configuration error: %s", err)
+		}
+		logrus.Fatalf("Failed to start due to configuration errors")
+	}
+
+	internal.SetupStdLogging()
+	internal.SetupHookLogging(cfg.Logging)
+	internal.SetupPprof()
+
+	logrus.Infof("Dendrite version %s", internal.VersionString())
+
+	if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled {
+		logrus.Warn("Open registration is enabled")
+	}
+
+	closer, err := cfg.SetupTracing()
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to start opentracing")
+	}
+	defer closer.Close() // nolint: errcheck
+
+	if cfg.Global.Sentry.Enabled {
+		logrus.Info("Setting up Sentry for debugging...")
+		err = sentry.Init(sentry.ClientOptions{
+			Dsn:              cfg.Global.Sentry.DSN,
+			Environment:      cfg.Global.Sentry.Environment,
+			Debug:            true,
+			ServerName:       string(cfg.Global.ServerName),
+			Release:          "dendrite@" + internal.VersionString(),
+			AttachStacktrace: true,
+		})
+		if err != nil {
+			logrus.WithError(err).Panic("failed to start Sentry")
+		}
+	}
+
+	processCtx := process.NewProcessContext()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+
+	basepkg.ConfigureAdminEndpoints(processCtx, routers)
+	defer func() {
+		processCtx.ShutdownDendrite()
+		processCtx.WaitForShutdown()
+	}() // nolint: errcheck
 
 	ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
 	if err != nil {
 		panic(err)
 	}
 
-	federation := ygg.CreateFederationClient(base)
+	federation := ygg.CreateFederationClient(cfg)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	rsAPI := roomserver.NewInternalAPI(
-		base,
-	)
+	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
+	natsInstance := jetstream.NATSInstance{}
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 
-	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
 
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, keyRing, true,
+		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    ygg.CreateClient(base),
+		Config:    cfg,
+		Client:    ygg.CreateClient(),
 		FedClient: federation,
 		KeyRing:   keyRing,
 
@@ -184,21 +236,21 @@ func main() {
 			ygg, fsAPI, federation,
 		),
 	}
-	monolith.AddAllPublicRoutes(base)
-	if err := mscs.Enable(base, &monolith); err != nil {
+	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
+	if err := mscs.Enable(cfg, cm, routers, &monolith, caches); err != nil {
 		logrus.WithError(err).Fatalf("Failed to enable MSCs")
 	}
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
-	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin)
 	embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
 
 	yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
-	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(routers.Federation)
+	yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media)
 
 	// Build both ends of a HTTP multiplex.
 	httpServer := &http.Server{
@@ -232,5 +284,5 @@ func main() {
 	}()
 
 	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
-	base.WaitForShutdown()
+	basepkg.WaitForShutdown(processCtx)
 }

cmd/dendrite-demo-yggdrasil/yggconn/client.go

@@ -4,8 +4,8 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 )
 
 type yggroundtripper struct {
@@ -17,9 +17,7 @@ func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) {
 	return y.inner.RoundTrip(req)
 }
 
-func (n *Node) CreateClient(
-	base *base.BaseDendrite,
-) *gomatrixserverlib.Client {
+func (n *Node) CreateClient() *fclient.Client {
 	tr := &http.Transport{}
 	tr.RegisterProtocol(
 		"matrix", &yggroundtripper{
@@ -33,14 +31,14 @@ func (n *Node) CreateClient(
 			},
 		},
 	)
-	return gomatrixserverlib.NewClient(
-		gomatrixserverlib.WithTransport(tr),
+	return fclient.NewClient(
+		fclient.WithTransport(tr),
 	)
 }
 
 func (n *Node) CreateFederationClient(
-	base *base.BaseDendrite,
-) *gomatrixserverlib.FederationClient {
+	cfg *config.Dendrite,
+) *fclient.FederationClient {
 	tr := &http.Transport{}
 	tr.RegisterProtocol(
 		"matrix", &yggroundtripper{
@@ -54,8 +52,8 @@ func (n *Node) CreateFederationClient(
 			},
 		},
 	)
-	return gomatrixserverlib.NewFederationClient(
-		base.Cfg.Global.SigningIdentities(),
-		gomatrixserverlib.WithTransport(tr),
+	return fclient.NewFederationClient(
+		cfg.Global.SigningIdentities(),
+		fclient.WithTransport(tr),
 	)
 }

cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go

@@ -22,17 +22,18 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
 type YggdrasilRoomProvider struct {
 	node      *yggconn.Node
 	fedSender api.FederationInternalAPI
-	fedClient *gomatrixserverlib.FederationClient
+	fedClient *fclient.FederationClient
 }
 
 func NewYggdrasilRoomProvider(
-	node *yggconn.Node, fedSender api.FederationInternalAPI, fedClient *gomatrixserverlib.FederationClient,
+	node *yggconn.Node, fedSender api.FederationInternalAPI, fedClient *fclient.FederationClient,
 ) *YggdrasilRoomProvider {
 	p := &YggdrasilRoomProvider{
 		node:      node,
@@ -42,7 +43,7 @@ func NewYggdrasilRoomProvider(
 	return p
 }
 
-func (p *YggdrasilRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
+func (p *YggdrasilRoomProvider) Rooms() []fclient.PublicRoom {
 	return bulkFetchPublicRoomsFromServers(
 		context.Background(), p.fedClient,
 		gomatrixserverlib.ServerName(p.node.DerivedServerName()),
@@ -53,14 +54,14 @@ func (p *YggdrasilRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
 // bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
 // Returns a list of public rooms.
 func bulkFetchPublicRoomsFromServers(
-	ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
+	ctx context.Context, fedClient *fclient.FederationClient,
 	origin gomatrixserverlib.ServerName,
 	homeservers []gomatrixserverlib.ServerName,
-) (publicRooms []gomatrixserverlib.PublicRoom) {
+) (publicRooms []fclient.PublicRoom) {
 	limit := 200
 	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
 	// goroutines send rooms to this channel
-	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
+	roomCh := make(chan fclient.PublicRoom, int(limit))
 	// signalling channel to tell goroutines to stop sending rooms and quit
 	done := make(chan bool)
 	// signalling to say when we can close the room channel

cmd/dendrite-upgrade-tests/main.go

@@ -259,10 +259,20 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir
 
 func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Version, err error) {
 	u := "https://api.github.com/repos/matrix-org/dendrite/tags"
-	res, err := httpClient.Get(u)
+
+	var res *http.Response
+	for i := 0; i < 3; i++ {
+		res, err = httpClient.Get(u)
 		if err != nil {
 			return nil, err
 		}
+		if res.StatusCode == 200 {
+			break
+		}
+		log.Printf("Github API returned HTTP %d, retrying\n", res.StatusCode)
+		time.Sleep(time.Second * 5)
+	}
+
 	if res.StatusCode != 200 {
 		return nil, fmt.Errorf("%s returned HTTP %d", u, res.StatusCode)
 	}

cmd/dendrite/main.go

@@ -16,8 +16,16 @@ package main
 
 import (
 	"flag"
-	"io/fs"
+	"time"
 
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/appservice"
@@ -34,8 +42,8 @@ var (
 	unixSocket = flag.String("unix-socket", "",
 		"EXPERIMENTAL(unstable): The HTTP listening unix socket for the server (disables http[s]-bind-address feature)",
 	)
-	unixSocketPermission = flag.Int("unix-socket-permission", 0755,
-		"EXPERIMENTAL(unstable): The HTTP listening unix socket permission for the server",
+	unixSocketPermission = flag.String("unix-socket-permission", "755",
+		"EXPERIMENTAL(unstable): The HTTP listening unix socket permission for the server (in chmod format like 755)",
 	)
 	httpBindAddr  = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
 	httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
@@ -59,27 +67,97 @@ func main() {
 		}
 		httpsAddr = https
 	} else {
-		httpAddr = config.UnixSocketAddress(*unixSocket, fs.FileMode(*unixSocketPermission))
+		socket, err := config.UnixSocketAddress(*unixSocket, *unixSocketPermission)
+		if err != nil {
+			logrus.WithError(err).Fatalf("Failed to parse unix socket")
+		}
+		httpAddr = socket
 	}
 
-	options := []basepkg.BaseDendriteOptions{}
+	configErrors := &config.ConfigErrors{}
+	cfg.Verify(configErrors)
+	if len(*configErrors) > 0 {
+		for _, err := range *configErrors {
+			logrus.Errorf("Configuration error: %s", err)
+		}
+		logrus.Fatalf("Failed to start due to configuration errors")
+	}
+	processCtx := process.NewProcessContext()
 
-	base := basepkg.NewBaseDendrite(cfg, options...)
-	defer base.Close() // nolint: errcheck
+	internal.SetupStdLogging()
+	internal.SetupHookLogging(cfg.Logging)
+	internal.SetupPprof()
 
-	federation := base.CreateFederationClient()
+	basepkg.PlatformSanityChecks()
 
-	rsAPI := roomserver.NewInternalAPI(base)
+	logrus.Infof("Dendrite version %s", internal.VersionString())
+	if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled {
+		logrus.Warn("Open registration is enabled")
+	}
 
+	// create DNS cache
+	var dnsCache *fclient.DNSCache
+	if cfg.Global.DNSCache.Enabled {
+		dnsCache = fclient.NewDNSCache(
+			cfg.Global.DNSCache.CacheSize,
+			cfg.Global.DNSCache.CacheLifetime,
+		)
+		logrus.Infof(
+			"DNS cache enabled (size %d, lifetime %s)",
+			cfg.Global.DNSCache.CacheSize,
+			cfg.Global.DNSCache.CacheLifetime,
+		)
+	}
+
+	// setup tracing
+	closer, err := cfg.SetupTracing()
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to start opentracing")
+	}
+	defer closer.Close() // nolint: errcheck
+
+	// setup sentry
+	if cfg.Global.Sentry.Enabled {
+		logrus.Info("Setting up Sentry for debugging...")
+		err = sentry.Init(sentry.ClientOptions{
+			Dsn:              cfg.Global.Sentry.DSN,
+			Environment:      cfg.Global.Sentry.Environment,
+			Debug:            true,
+			ServerName:       string(cfg.Global.ServerName),
+			Release:          "dendrite@" + internal.VersionString(),
+			AttachStacktrace: true,
+		})
+		if err != nil {
+			logrus.WithError(err).Panic("failed to start Sentry")
+		}
+		go func() {
+			processCtx.ComponentStarted()
+			<-processCtx.WaitForShutdown()
+			if !sentry.Flush(time.Second * 5) {
+				logrus.Warnf("failed to flush all Sentry events!")
+			}
+			processCtx.ComponentFinished()
+		}()
+	}
+
+	federationClient := basepkg.CreateFederationClient(cfg, dnsCache)
+	httpClient := basepkg.CreateClient(cfg, dnsCache)
+
+	// prepare required dependencies
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	routers := httputil.NewRouters()
+
+	caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics)
+	natsInstance := jetstream.NATSInstance{}
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, nil, false,
+		processCtx, cfg, cm, &natsInstance, federationClient, rsAPI, caches, nil, false,
 	)
 
 	keyRing := fsAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
-
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient)
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this
@@ -89,9 +167,9 @@ func main() {
 	rsAPI.SetUserAPI(userAPI)
 
 	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    base.CreateClient(),
-		FedClient: federation,
+		Config:    cfg,
+		Client:    httpClient,
+		FedClient: federationClient,
 		KeyRing:   keyRing,
 
 		AppserviceAPI: asAPI,
@@ -101,25 +179,25 @@ func main() {
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
 	}
-	monolith.AddAllPublicRoutes(base)
+	monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics)
 
-	if len(base.Cfg.MSCs.MSCs) > 0 {
-		if err := mscs.Enable(base, &monolith); err != nil {
+	if len(cfg.MSCs.MSCs) > 0 {
+		if err := mscs.Enable(cfg, cm, routers, &monolith, caches); err != nil {
 			logrus.WithError(err).Fatalf("Failed to enable MSCs")
 		}
 	}
 
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {
-		base.SetupAndServeHTTP(httpAddr, nil, nil)
+		basepkg.SetupAndServeHTTP(processCtx, cfg, routers, httpAddr, nil, nil)
 	}()
 	// Handle HTTPS if certificate and key are provided
 	if *unixSocket == "" && *certFile != "" && *keyFile != "" {
 		go func() {
-			base.SetupAndServeHTTP(httpsAddr, certFile, keyFile)
+			basepkg.SetupAndServeHTTP(processCtx, cfg, routers, httpsAddr, certFile, keyFile)
 		}()
 	}
 
 	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
-	base.WaitForShutdown()
+	basepkg.WaitForShutdown(processCtx)
 }

cmd/furl/main.go

@@ -13,6 +13,7 @@ import (
 	"os"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 )
 
 var requestFrom = flag.String("from", "", "the server name that the request should originate from")
@@ -49,8 +50,8 @@ func main() {
 	}
 
 	serverName := gomatrixserverlib.ServerName(*requestFrom)
-	client := gomatrixserverlib.NewFederationClient(
-		[]*gomatrixserverlib.SigningIdentity{
+	client := fclient.NewFederationClient(
+		[]*fclient.SigningIdentity{
 			{
 				ServerName: serverName,
 				KeyID:      gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]),

cmd/resolve-state/main.go

@@ -10,12 +10,13 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -40,7 +41,7 @@ func main() {
 		Level: "error",
 	})
 	cfg.ClientAPI.RegistrationDisabled = true
-	base := base.NewBaseDendrite(cfg, base.DisableMetrics)
+
 	args := flag.Args()
 
 	fmt.Println("Room version", *roomVersion)
@@ -54,8 +55,10 @@ func main() {
 
 	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
 
+	processCtx := process.NewProcessContext()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 	roomserverDB, err := storage.Open(
-		base, &cfg.RoomServer.Database,
+		processCtx.Context(), cm, &cfg.RoomServer.Database,
 		caching.NewRistrettoCache(128*1024*1024, time.Hour, true),
 	)
 	if err != nil {

docs/FAQ.md

@@ -18,7 +18,7 @@ Mostly, although there are still bugs and missing features. If you are a confide
 
 ## Is Dendrite feature-complete?
 
-No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](../README.md) at the root of the repository for more information.
+No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](https://github.com/matrix-org/dendrite/blob/main/README.md) at the root of the repository for more information.
 
 ## Why doesn't Dendrite have "x" yet?
 

docs/administration/4_adminapi.md

@@ -32,7 +32,7 @@ UPDATE userapi_accounts SET account_type = 3 WHERE localpart = '$localpart';
 
 Where `$localpart` is the username only (e.g. `alice`).
 
-## GET `/_dendrite/admin/evacuateRoom/{roomID}`
+## POST `/_dendrite/admin/evacuateRoom/{roomID}`
 
 This endpoint will instruct Dendrite to part all local users from the given `roomID`
 in the URL. It may take some time to complete. A JSON body will be returned containing
@@ -41,7 +41,7 @@ the user IDs of all affected users.
 If the room has an alias set (e.g. is published), the room's ID will not be visible in the URL, but it can
 be found as the room's "internal ID" in Element Web (Settings -> Advanced)
 
-## GET `/_dendrite/admin/evacuateUser/{userID}`
+## POST `/_dendrite/admin/evacuateUser/{userID}`
 
 This endpoint will instruct Dendrite to part the given local `userID` in the URL from
 all rooms which they are currently joined. A JSON body will be returned containing

docs/development/sytest.md

@@ -61,7 +61,6 @@ When debugging, the following Docker `run` options may also be useful:
 * `-e "DENDRITE_TRACE_HTTP=1"`: Adds HTTP tracing to server logs.
 * `-e "DENDRITE_TRACE_INTERNAL=1"`: Adds roomserver internal API tracing to
   server logs.
-* `-e "DENDRITE_TRACE_SQL=1"`: Adds tracing to all SQL statements to server logs.
 
 The docker command also supports a single positional argument for the test file to
 run, so you can run a single `.pl` file rather than the whole test suite. For example:

docs/installation/7_configuration.md

@@ -10,7 +10,7 @@ permalink: /installation/configuration
 A YAML configuration file is used to configure Dendrite. A sample configuration file is
 present in the top level of the Dendrite repository:
 
-* [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml)
+* [`dendrite-sample.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.yaml)
 
 You will need to duplicate the sample, calling it `dendrite.yaml` for example, and then
 tailor it to your installation. At a minimum, you will need to populate the following
@@ -86,9 +86,8 @@ that you chose.
 
 ### Global connection pool
 
-If you are running a monolith deployment and want to use a single connection pool to a
-single PostgreSQL database, then you must uncomment and configure the `database` section
-within the `global` section:
+If you want to use a single connection pool to a single PostgreSQL database, then you must
+uncomment and configure the `database` section within the `global` section:
 
 ```yaml
 global:
@@ -102,15 +101,15 @@ global:
 
 **You must then remove or comment out** the `database` sections from other areas of the
 configuration file, e.g. under the `app_service_api`, `federation_api`, `key_server`,
-`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks, otherwise these will
-override the `global` database configuration.
+`media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks, otherwise
+these will override the `global` database configuration.
 
 ### Per-component connections (all other configurations)
 
 If you are are using SQLite databases or separate PostgreSQL
 databases per component, then you must instead configure the `database` sections under each
 of the component blocks ,e.g. under the `app_service_api`, `federation_api`, `key_server`,
-`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks.
+`media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks.
 
 For example, with PostgreSQL:
 

federationapi/api/api.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 
 	"github.com/matrix-org/dendrite/federationapi/types"
 )
@@ -22,8 +23,8 @@ type FederationInternalAPI interface {
 
 	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
 	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
-	MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
-	MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
+	MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res fclient.MSC2836EventRelationshipsResponse, err error)
+	MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error)
 
 	// Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos.
 	PerformBroadcastEDU(
@@ -66,9 +67,9 @@ type RoomserverFederationAPI interface {
 	// containing only the server names (without information for membership events).
 	// The response will include this server if they are joined to the room.
 	QueryJoinedHostServerNamesInRoom(ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse) error
-	GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
+	GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res fclient.RespEventAuth, err error)
 	GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
-	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
+	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error)
 }
 
 type P2PFederationAPI interface {
@@ -98,45 +99,45 @@ type P2PFederationAPI interface {
 // implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
 // this interface are of type FederationClientError
 type KeyserverFederationAPI interface {
-	GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
-	ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
-	QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
+	GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (res fclient.RespUserDevices, err error)
+	ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res fclient.RespClaimKeys, err error)
+	QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (res fclient.RespQueryKeys, err error)
 }
 
 // an interface for gmsl.FederationClient - contains functions called by federationapi only.
 type FederationClient interface {
 	P2PFederationClient
 	gomatrixserverlib.KeyClient
-	SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error)
+	SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res fclient.RespSend, err error)
 
 	// Perform operations
-	LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error)
-	Peek(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error)
-	MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error)
-	SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error)
-	MakeLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error)
+	LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res fclient.RespDirectory, err error)
+	Peek(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res fclient.RespPeek, err error)
+	MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res fclient.RespMakeJoin, err error)
+	SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res fclient.RespSendJoin, err error)
+	MakeLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string) (res fclient.RespMakeLeave, err error)
 	SendLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error)
-	SendInviteV2(ctx context.Context, origin, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error)
+	SendInviteV2(ctx context.Context, origin, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res fclient.RespInviteV2, err error)
 
 	GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
 
-	GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
-	GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error)
-	ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error)
-	QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error)
+	GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res fclient.RespEventAuth, err error)
+	GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (fclient.RespUserDevices, error)
+	ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (fclient.RespClaimKeys, error)
+	QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (fclient.RespQueryKeys, error)
 	Backfill(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
-	MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
-	MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
+	MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res fclient.MSC2836EventRelationshipsResponse, err error)
+	MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error)
 
 	ExchangeThirdPartyInvite(ctx context.Context, origin, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
-	LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error)
-	LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
-	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
+	LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespState, err error)
+	LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res fclient.RespStateIDs, err error)
+	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error)
 }
 
 type P2PFederationClient interface {
-	P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error)
-	P2PGetTransactionFromRelay(ctx context.Context, u gomatrixserverlib.UserID, prev gomatrixserverlib.RelayEntry, relayServer gomatrixserverlib.ServerName) (res gomatrixserverlib.RespGetRelayTransaction, err error)
+	P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res fclient.EmptyResp, err error)
+	P2PGetTransactionFromRelay(ctx context.Context, u gomatrixserverlib.UserID, prev fclient.RelayEntry, relayServer gomatrixserverlib.ServerName) (res fclient.RespGetRelayTransaction, err error)
 }
 
 // FederationClientError is returned from FederationClient methods in the event of a problem.

federationapi/federationapi.go

@@ -17,6 +17,11 @@ package federationapi
 import (
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
@@ -29,7 +34,6 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/internal/caching"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 
@@ -40,17 +44,21 @@ import (
 
 // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.
 func AddPublicRoutes(
-	base *base.BaseDendrite,
+	processContext *process.ProcessContext,
+	routers httputil.Routers,
+	dendriteConfig *config.Dendrite,
+	natsInstance *jetstream.NATSInstance,
 	userAPI userapi.FederationUserAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation *fclient.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	fedAPI federationAPI.FederationInternalAPI,
 	servers federationAPI.ServersInRoomProvider,
+	enableMetrics bool,
 ) {
-	cfg := &base.Cfg.FederationAPI
-	mscCfg := &base.Cfg.MSCs
-	js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	cfg := &dendriteConfig.FederationAPI
+	mscCfg := &dendriteConfig.MSCs
+	js, _ := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream)
 	producer := &producers.SyncAPIProducer{
 		JetStream:              js,
 		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
@@ -75,26 +83,30 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		base,
+		routers,
+		dendriteConfig,
 		rsAPI, f, keyRing,
 		federation, userAPI, mscCfg,
-		servers, producer,
+		servers, producer, enableMetrics,
 	)
 }
 
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *base.BaseDendrite,
+	processContext *process.ProcessContext,
+	dendriteCfg *config.Dendrite,
+	cm sqlutil.Connections,
+	natsInstance *jetstream.NATSInstance,
 	federation api.FederationClient,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	caches *caching.Caches,
 	keyRing *gomatrixserverlib.KeyRing,
 	resetBlacklist bool,
 ) api.FederationInternalAPI {
-	cfg := &base.Cfg.FederationAPI
+	cfg := &dendriteCfg.FederationAPI
 
-	federationDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.IsLocalServerName)
+	federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName)
 	if err != nil {
 		logrus.WithError(err).Panic("failed to connect to federation sender db")
 	}
@@ -108,51 +120,51 @@ func NewInternalAPI(
 		cfg.FederationMaxRetries+1,
 		cfg.P2PFederationRetriesUntilAssumedOffline+1)
 
-	js, nats := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, nats := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream)
 
-	signingInfo := base.Cfg.Global.SigningIdentities()
+	signingInfo := dendriteCfg.Global.SigningIdentities()
 
 	queues := queue.NewOutgoingQueues(
-		federationDB, base.ProcessContext,
+		federationDB, processContext,
 		cfg.Matrix.DisableFederation,
 		cfg.Matrix.ServerName, federation, rsAPI, &stats,
 		signingInfo,
 	)
 
 	rsConsumer := consumers.NewOutputRoomEventConsumer(
-		base.ProcessContext, cfg, js, nats, queues,
+		processContext, cfg, js, nats, queues,
 		federationDB, rsAPI,
 	)
 	if err = rsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start room server consumer")
 	}
 	tsConsumer := consumers.NewOutputSendToDeviceConsumer(
-		base.ProcessContext, cfg, js, queues, federationDB,
+		processContext, cfg, js, queues, federationDB,
 	)
 	if err = tsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start send-to-device consumer")
 	}
 	receiptConsumer := consumers.NewOutputReceiptConsumer(
-		base.ProcessContext, cfg, js, queues, federationDB,
+		processContext, cfg, js, queues, federationDB,
 	)
 	if err = receiptConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start receipt consumer")
 	}
 	typingConsumer := consumers.NewOutputTypingConsumer(
-		base.ProcessContext, cfg, js, queues, federationDB,
+		processContext, cfg, js, queues, federationDB,
 	)
 	if err = typingConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start typing consumer")
 	}
 	keyConsumer := consumers.NewKeyChangeConsumer(
-		base.ProcessContext, &base.Cfg.KeyServer, js, queues, federationDB, rsAPI,
+		processContext, &dendriteCfg.KeyServer, js, queues, federationDB, rsAPI,
 	)
 	if err = keyConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start key server consumer")
 	}
 
 	presenceConsumer := consumers.NewOutputPresenceConsumer(
-		base.ProcessContext, cfg, js, queues, federationDB, rsAPI,
+		processContext, cfg, js, queues, federationDB, rsAPI,
 	)
 	if err = presenceConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start presence consumer")
@@ -161,7 +173,7 @@ func NewInternalAPI(
 	var cleanExpiredEDUs func()
 	cleanExpiredEDUs = func() {
 		logrus.Infof("Cleaning expired EDUs")
-		if err := federationDB.DeleteExpiredEDUs(base.Context()); err != nil {
+		if err := federationDB.DeleteExpiredEDUs(processContext.Context()); err != nil {
 			logrus.WithError(err).Error("Failed to clean expired EDUs")
 		}
 		time.AfterFunc(time.Hour, cleanExpiredEDUs)

federationapi/federationapi_keys_test.go

@@ -12,12 +12,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
@@ -25,7 +28,7 @@ type server struct {
 	name      gomatrixserverlib.ServerName // server name
 	validity  time.Duration                // key validity duration from now
 	config    *config.FederationAPI        // skeleton config, from TestMain
-	fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper
+	fedclient *fclient.FederationClient    // uses MockRoundTripper
 	cache     *caching.Caches              // server-specific cache
 	api       api.FederationInternalAPI    // server-specific server key API
 }
@@ -65,7 +68,7 @@ func TestMain(m *testing.M) {
 
 			// Create a new cache but don't enable prometheus!
 			s.cache = caching.NewRistrettoCache(8*1024*1024, time.Hour, false)
-
+			natsInstance := jetstream.NATSInstance{}
 			// Create a temporary directory for JetStream.
 			d, err := os.MkdirTemp("./", "jetstream*")
 			if err != nil {
@@ -103,14 +106,15 @@ func TestMain(m *testing.M) {
 			transport.RegisterProtocol("matrix", &MockRoundTripper{})
 
 			// Create the federation client.
-			s.fedclient = gomatrixserverlib.NewFederationClient(
+			s.fedclient = fclient.NewFederationClient(
 				s.config.Matrix.SigningIdentities(),
-				gomatrixserverlib.WithTransport(transport),
+				fclient.WithTransport(transport),
 			)
 
 			// Finally, build the server key APIs.
-			sbase := base.NewBaseDendrite(cfg, base.DisableMetrics)
-			s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
+			processCtx := process.NewProcessContext()
+			cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+			s.api = NewInternalAPI(processCtx, cfg, cm, &natsInstance, s.fedclient, nil, s.cache, nil, true)
 		}
 
 		// Now that we have built our server key APIs, start the

federationapi/federationapi_test.go

@@ -10,16 +10,18 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/nats-io/nats.go"
 
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/internal"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
@@ -103,7 +105,7 @@ func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserv
 	return keys, nil
 }
 
-func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) {
+func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res fclient.RespMakeJoin, err error) {
 	for _, r := range f.allowJoins {
 		if r.ID == roomID {
 			res.RoomVersion = r.Version
@@ -127,7 +129,7 @@ func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.Se
 	}
 	return
 }
-func (f *fedClient) SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
+func (f *fedClient) SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res fclient.RespSendJoin, err error) {
 	f.fedClientMutex.Lock()
 	defer f.fedClientMutex.Unlock()
 	for _, r := range f.allowJoins {
@@ -141,7 +143,7 @@ func (f *fedClient) SendJoin(ctx context.Context, origin, s gomatrixserverlib.Se
 	return
 }
 
-func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
+func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res fclient.RespSend, err error) {
 	f.fedClientMutex.Lock()
 	defer f.fedClientMutex.Unlock()
 	for _, edu := range t.EDUs {
@@ -162,21 +164,24 @@ func TestFederationAPIJoinThenKeyUpdate(t *testing.T) {
 }
 
 func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
-	base, close := testrig.CreateBaseDendrite(t, dbType)
-	base.Cfg.FederationAPI.PreferDirectFetch = true
-	base.Cfg.FederationAPI.KeyPerspectives = nil
+	cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	natsInstance := jetstream.NATSInstance{}
+	cfg.FederationAPI.PreferDirectFetch = true
+	cfg.FederationAPI.KeyPerspectives = nil
 	defer close()
-	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
-	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+	jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
 
 	serverA := gomatrixserverlib.ServerName("server.a")
 	serverAKeyID := gomatrixserverlib.KeyID("ed25519:servera")
 	serverAPrivKey := test.PrivateKeyA
 	creator := test.NewUser(t, test.WithSigningServer(serverA, serverAKeyID, serverAPrivKey))
 
-	myServer := base.Cfg.Global.ServerName
-	myServerKeyID := base.Cfg.Global.KeyID
-	myServerPrivKey := base.Cfg.Global.PrivateKey
+	myServer := cfg.Global.ServerName
+	myServerKeyID := cfg.Global.KeyID
+	myServerPrivKey := cfg.Global.PrivateKey
 	joiningUser := test.NewUser(t, test.WithSigningServer(myServer, myServerKeyID, myServerPrivKey))
 	fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID)
 	room := test.NewRoom(t, creator)
@@ -212,7 +217,7 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 			},
 		},
 	}
-	fsapi := federationapi.NewInternalAPI(base, fc, rsapi, base.Caches, nil, false)
+	fsapi := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, fc, rsapi, caches, nil, false)
 
 	var resp api.PerformJoinResponse
 	fsapi.PerformJoin(context.Background(), &api.PerformJoinRequest{
@@ -245,7 +250,7 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 	}
 
 	msg := &nats.Msg{
-		Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
+		Subject: cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
 		Header:  nats.Header{},
 		Data:    b,
 	}
@@ -263,30 +268,6 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 // Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404.
 // Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated.
 func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
-	_, privKey, _ := ed25519.GenerateKey(nil)
-	cfg := &config.Dendrite{}
-	cfg.Defaults(config.DefaultOpts{
-		Generate:       true,
-		SingleDatabase: false,
-	})
-	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
-	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
-	cfg.Global.PrivateKey = privKey
-	cfg.Global.JetStream.InMemory = true
-	b := base.NewBaseDendrite(cfg, base.DisableMetrics)
-	keyRing := &test.NopJSONVerifier{}
-	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
-	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(b, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil)
-	baseURL, cancel := test.ListenAndServe(t, b.PublicFederationAPIMux, true)
-	defer cancel()
-	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
-
-	fedCli := gomatrixserverlib.NewFederationClient(
-		cfg.Global.SigningIdentities(),
-		gomatrixserverlib.WithSkipVerify(true),
-	)
-
 	testCases := []struct {
 		roomVer   gomatrixserverlib.RoomVersion
 		eventJSON string
@@ -315,6 +296,29 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 		},
 	}
 
+	cfg, processCtx, close := testrig.CreateConfig(t, test.DBTypeSQLite)
+	defer close()
+	routers := httputil.NewRouters()
+
+	_, privKey, _ := ed25519.GenerateKey(nil)
+	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
+	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
+	cfg.Global.PrivateKey = privKey
+	cfg.Global.JetStream.InMemory = true
+	keyRing := &test.NopJSONVerifier{}
+	natsInstance := jetstream.NATSInstance{}
+	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
+	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
+	federationapi.AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil, caching.DisableMetrics)
+	baseURL, cancel := test.ListenAndServe(t, routers.Federation, true)
+	defer cancel()
+	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
+
+	fedCli := fclient.NewFederationClient(
+		cfg.Global.SigningIdentities(),
+		fclient.WithSkipVerify(true),
+	)
+
 	for _, tc := range testCases {
 		ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(tc.eventJSON), false, tc.roomVer)
 		if err != nil {

federationapi/internal/federationclient.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 )
 
 // Functions here are "proxying" calls to the gomatrixserverlib federation
@@ -13,56 +14,56 @@ import (
 func (a *FederationInternalAPI) GetEventAuth(
 	ctx context.Context, origin, s gomatrixserverlib.ServerName,
 	roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string,
-) (res gomatrixserverlib.RespEventAuth, err error) {
+) (res fclient.RespEventAuth, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.GetEventAuth(ctx, origin, s, roomVersion, roomID, eventID)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespEventAuth{}, err
+		return fclient.RespEventAuth{}, err
 	}
-	return ires.(gomatrixserverlib.RespEventAuth), nil
+	return ires.(fclient.RespEventAuth), nil
 }
 
 func (a *FederationInternalAPI) GetUserDevices(
 	ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string,
-) (gomatrixserverlib.RespUserDevices, error) {
+) (fclient.RespUserDevices, error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.GetUserDevices(ctx, origin, s, userID)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespUserDevices{}, err
+		return fclient.RespUserDevices{}, err
 	}
-	return ires.(gomatrixserverlib.RespUserDevices), nil
+	return ires.(fclient.RespUserDevices), nil
 }
 
 func (a *FederationInternalAPI) ClaimKeys(
 	ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string,
-) (gomatrixserverlib.RespClaimKeys, error) {
+) (fclient.RespClaimKeys, error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.ClaimKeys(ctx, origin, s, oneTimeKeys)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespClaimKeys{}, err
+		return fclient.RespClaimKeys{}, err
 	}
-	return ires.(gomatrixserverlib.RespClaimKeys), nil
+	return ires.(fclient.RespClaimKeys), nil
 }
 
 func (a *FederationInternalAPI) QueryKeys(
 	ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string,
-) (gomatrixserverlib.RespQueryKeys, error) {
+) (fclient.RespQueryKeys, error) {
 	ires, err := a.doRequestIfNotBackingOffOrBlacklisted(s, func() (interface{}, error) {
 		return a.federation.QueryKeys(ctx, origin, s, keys)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespQueryKeys{}, err
+		return fclient.RespQueryKeys{}, err
 	}
-	return ires.(gomatrixserverlib.RespQueryKeys), nil
+	return ires.(fclient.RespQueryKeys), nil
 }
 
 func (a *FederationInternalAPI) Backfill(
@@ -81,45 +82,46 @@ func (a *FederationInternalAPI) Backfill(
 
 func (a *FederationInternalAPI) LookupState(
 	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
-) (res gomatrixserverlib.RespState, err error) {
+) (res gomatrixserverlib.StateResponse, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.LookupState(ctx, origin, s, roomID, eventID, roomVersion)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespState{}, err
+		return &fclient.RespState{}, err
 	}
-	return ires.(gomatrixserverlib.RespState), nil
+	r := ires.(fclient.RespState)
+	return &r, nil
 }
 
 func (a *FederationInternalAPI) LookupStateIDs(
 	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string,
-) (res gomatrixserverlib.RespStateIDs, err error) {
+) (res gomatrixserverlib.StateIDResponse, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.LookupStateIDs(ctx, origin, s, roomID, eventID)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespStateIDs{}, err
+		return fclient.RespStateIDs{}, err
 	}
-	return ires.(gomatrixserverlib.RespStateIDs), nil
+	return ires.(fclient.RespStateIDs), nil
 }
 
 func (a *FederationInternalAPI) LookupMissingEvents(
 	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string,
-	missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion,
-) (res gomatrixserverlib.RespMissingEvents, err error) {
+	missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion,
+) (res fclient.RespMissingEvents, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.LookupMissingEvents(ctx, origin, s, roomID, missing, roomVersion)
 	})
 	if err != nil {
-		return gomatrixserverlib.RespMissingEvents{}, err
+		return fclient.RespMissingEvents{}, err
 	}
-	return ires.(gomatrixserverlib.RespMissingEvents), nil
+	return ires.(fclient.RespMissingEvents), nil
 }
 
 func (a *FederationInternalAPI) GetEvent(
@@ -151,9 +153,9 @@ func (a *FederationInternalAPI) LookupServerKeys(
 }
 
 func (a *FederationInternalAPI) MSC2836EventRelationships(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest,
+	ctx context.Context, origin, s gomatrixserverlib.ServerName, r fclient.MSC2836EventRelationshipsRequest,
 	roomVersion gomatrixserverlib.RoomVersion,
-) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) {
+) (res fclient.MSC2836EventRelationshipsResponse, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
@@ -162,12 +164,12 @@ func (a *FederationInternalAPI) MSC2836EventRelationships(
 	if err != nil {
 		return res, err
 	}
-	return ires.(gomatrixserverlib.MSC2836EventRelationshipsResponse), nil
+	return ires.(fclient.MSC2836EventRelationshipsResponse), nil
 }
 
 func (a *FederationInternalAPI) MSC2946Spaces(
 	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, suggestedOnly bool,
-) (res gomatrixserverlib.MSC2946SpacesResponse, err error) {
+) (res fclient.MSC2946SpacesResponse, err error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)
 	defer cancel()
 	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
@@ -176,5 +178,5 @@ func (a *FederationInternalAPI) MSC2946Spaces(
 	if err != nil {
 		return res, err
 	}
-	return ires.(gomatrixserverlib.MSC2946SpacesResponse), nil
+	return ires.(fclient.MSC2946SpacesResponse), nil
 }

federationapi/internal/federationclient_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -33,20 +34,20 @@ const (
 	FailuresUntilBlacklist      = 8
 )
 
-func (t *testFedClient) QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error) {
+func (t *testFedClient) QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (fclient.RespQueryKeys, error) {
 	t.queryKeysCalled = true
 	if t.shouldFail {
-		return gomatrixserverlib.RespQueryKeys{}, fmt.Errorf("Failure")
+		return fclient.RespQueryKeys{}, fmt.Errorf("Failure")
 	}
-	return gomatrixserverlib.RespQueryKeys{}, nil
+	return fclient.RespQueryKeys{}, nil
 }
 
-func (t *testFedClient) ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error) {
+func (t *testFedClient) ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (fclient.RespClaimKeys, error) {
 	t.claimKeysCalled = true
 	if t.shouldFail {
-		return gomatrixserverlib.RespClaimKeys{}, fmt.Errorf("Failure")
+		return fclient.RespClaimKeys{}, fmt.Errorf("Failure")
 	}
-	return gomatrixserverlib.RespClaimKeys{}, nil
+	return fclient.RespClaimKeys{}, nil
 }
 
 func TestFederationClientQueryKeys(t *testing.T) {
@@ -54,7 +55,7 @@ func TestFederationClientQueryKeys(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},
@@ -85,7 +86,7 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},
@@ -115,7 +116,7 @@ func TestFederationClientQueryKeysFailure(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},
@@ -145,7 +146,7 @@ func TestFederationClientClaimKeys(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},
@@ -176,7 +177,7 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "server",
 			},
 		},

federationapi/internal/perform.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 
@@ -255,7 +256,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	// waste the effort.
 	// TODO: Can we expand Check here to return a list of missing auth
 	// events rather than failing one at a time?
-	var respState *gomatrixserverlib.RespState
+	var respState *fclient.RespState
 	respState, err = respSendJoin.Check(
 		context.Background(),
 		respMakeJoin.RoomVersion,

federationapi/internal/perform_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -35,8 +36,8 @@ type testFedClient struct {
 	shouldFail      bool
 }
 
-func (t *testFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) {
-	return gomatrixserverlib.RespDirectory{}, nil
+func (t *testFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res fclient.RespDirectory, err error) {
+	return fclient.RespDirectory{}, nil
 }
 
 func TestPerformWakeupServers(t *testing.T) {
@@ -54,7 +55,7 @@ func TestPerformWakeupServers(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
 			},
 		},
@@ -96,7 +97,7 @@ func TestQueryRelayServers(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
 			},
 		},
@@ -133,7 +134,7 @@ func TestRemoveRelayServers(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
 			},
 		},
@@ -169,7 +170,7 @@ func TestPerformDirectoryLookup(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
 			},
 		},
@@ -204,7 +205,7 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
 
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+			SigningIdentity: fclient.SigningIdentity{
 				ServerName: server,
 			},
 		},

federationapi/queue/destinationqueue.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/sirupsen/logrus"
 	"go.uber.org/atomic"
 
@@ -50,7 +51,7 @@ type destinationQueue struct {
 	queues             *OutgoingQueues
 	db                 storage.Database
 	process            *process.ProcessContext
-	signing            map[gomatrixserverlib.ServerName]*gomatrixserverlib.SigningIdentity
+	signing            map[gomatrixserverlib.ServerName]*fclient.SigningIdentity
 	rsAPI              api.FederationRoomserverAPI
 	client             fedapi.FederationClient         // federation client
 	origin             gomatrixserverlib.ServerName    // origin of requests

federationapi/queue/queue.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
@@ -45,7 +46,7 @@ type OutgoingQueues struct {
 	origin      gomatrixserverlib.ServerName
 	client      fedapi.FederationClient
 	statistics  *statistics.Statistics
-	signing     map[gomatrixserverlib.ServerName]*gomatrixserverlib.SigningIdentity
+	signing     map[gomatrixserverlib.ServerName]*fclient.SigningIdentity
 	queuesMutex sync.Mutex // protects the below
 	queues      map[gomatrixserverlib.ServerName]*destinationQueue
 }
@@ -90,7 +91,7 @@ func NewOutgoingQueues(
 	client fedapi.FederationClient,
 	rsAPI api.FederationRoomserverAPI,
 	statistics *statistics.Statistics,
-	signing []*gomatrixserverlib.SigningIdentity,
+	signing []*fclient.SigningIdentity,
 ) *OutgoingQueues {
 	queues := &OutgoingQueues{
 		disabled:   disabled,
@@ -100,7 +101,7 @@ func NewOutgoingQueues(
 		origin:     origin,
 		client:     client,
 		statistics: statistics,
-		signing:    map[gomatrixserverlib.ServerName]*gomatrixserverlib.SigningIdentity{},
+		signing:    map[gomatrixserverlib.ServerName]*fclient.SigningIdentity{},
 		queues:     map[gomatrixserverlib.ServerName]*destinationQueue{},
 	}
 	for _, identity := range signing {

federationapi/queue/queue_test.go

@@ -21,6 +21,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"go.uber.org/atomic"
 	"gotest.tools/v3/poll"
 
@@ -34,31 +38,29 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/dendrite/test/testrig"
 )
 
 func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *process.ProcessContext, func()) {
 	if realDatabase {
 		// Real Database/s
-		b, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		connStr, dbClose := test.PrepareDBConnectionString(t, dbType)
-		db, err := storage.NewDatabase(b, &config.DatabaseOptions{
+		db, err := storage.NewDatabase(processCtx.Context(), cm, &config.DatabaseOptions{
 			ConnectionString: config.DataSource(connStr),
-		}, b.Caches, b.Cfg.Global.IsLocalServerName)
+		}, caches, cfg.Global.IsLocalServerName)
 		if err != nil {
 			t.Fatalf("NewDatabase returned %s", err)
 		}
-		return db, b.ProcessContext, func() {
+		return db, processCtx, func() {
+			close()
 			dbClose()
-			baseClose()
 		}
 	} else {
 		// Fake Database
 		db := test.NewInMemoryFederationDatabase()
-		b := struct {
-			ProcessContext *process.ProcessContext
-		}{ProcessContext: process.NewProcessContext()}
-		return db, b.ProcessContext, func() {}
+		return db, process.NewProcessContext(), func() {}
 	}
 }
 
@@ -79,24 +81,24 @@ type stubFederationClient struct {
 	txRelayCount         atomic.Uint32
 }
 
-func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
+func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res fclient.RespSend, err error) {
 	var result error
 	if !f.shouldTxSucceed {
 		result = fmt.Errorf("transaction failed")
 	}
 
 	f.txCount.Add(1)
-	return gomatrixserverlib.RespSend{}, result
+	return fclient.RespSend{}, result
 }
 
-func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error) {
+func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res fclient.EmptyResp, err error) {
 	var result error
 	if !f.shouldTxRelaySucceed {
 		result = fmt.Errorf("relay transaction failed")
 	}
 
 	f.txRelayCount.Add(1)
-	return gomatrixserverlib.EmptyResp{}, result
+	return fclient.EmptyResp{}, result
 }
 
 func mustCreatePDU(t *testing.T) *gomatrixserverlib.HeaderedEvent {
@@ -126,7 +128,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
 	rs := &stubFederationRoomServerAPI{}
 
 	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline)
-	signingInfo := []*gomatrixserverlib.SigningIdentity{
+	signingInfo := []*fclient.SigningIdentity{
 		{
 			KeyID:      "ed21019:auto",
 			PrivateKey: test.PrivateKeyA,

federationapi/routing/devices.go

@@ -19,6 +19,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 	"github.com/tidwall/gjson"
 )
@@ -53,10 +54,10 @@ func GetUserDevices(
 		return jsonerror.InternalAPIError(req.Context(), err)
 	}
 
-	response := gomatrixserverlib.RespUserDevices{
+	response := fclient.RespUserDevices{
 		UserID:   userID,
 		StreamID: res.StreamID,
-		Devices:  []gomatrixserverlib.RespUserDevice{},
+		Devices:  []fclient.RespUserDevice{},
 	}
 
 	if masterKey, ok := sigRes.MasterKeys[userID]; ok {
@@ -67,7 +68,7 @@ func GetUserDevices(
 	}
 
 	for _, dev := range res.Devices {
-		var key gomatrixserverlib.RespUserDeviceKeys
+		var key fclient.RespUserDeviceKeys
 		err := json.Unmarshal(dev.DeviceKeys.KeyJSON, &key)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Warnf("malformed device key: %s", string(dev.DeviceKeys.KeyJSON))
@@ -79,7 +80,7 @@ func GetUserDevices(
 			displayName = gjson.GetBytes(dev.DeviceKeys.KeyJSON, "unsigned.device_display_name").Str
 		}
 
-		device := gomatrixserverlib.RespUserDevice{
+		device := fclient.RespUserDevice{
 			DeviceID:    dev.DeviceID,
 			DisplayName: displayName,
 			Keys:        key,

federationapi/routing/eventauth.go

@@ -19,6 +19,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -70,7 +71,7 @@ func GetEventAuth(
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: gomatrixserverlib.RespEventAuth{
+		JSON: fclient.RespEventAuth{
 			AuthEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(response.AuthChainEvents),
 		},
 	}

federationapi/routing/invite.go

@@ -25,6 +25,7 @@ import (
 	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -218,12 +219,12 @@ func processInvite(
 	if isInviteV2 {
 		return util.JSONResponse{
 			Code: http.StatusOK,
-			JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent.JSON()},
+			JSON: fclient.RespInviteV2{Event: signedEvent.JSON()},
 		}
 	} else {
 		return util.JSONResponse{
 			Code: http.StatusOK,
-			JSON: gomatrixserverlib.RespInvite{Event: signedEvent.JSON()},
+			JSON: fclient.RespInvite{Event: signedEvent.JSON()},
 		}
 	}
 }

federationapi/routing/join.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 
@@ -433,7 +434,7 @@ func SendJoin(
 	// https://matrix.org/docs/spec/server_server/latest#put-matrix-federation-v1-send-join-roomid-eventid
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: gomatrixserverlib.RespSendJoin{
+		JSON: fclient.RespSendJoin{
 			StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.StateEvents),
 			AuthEvents:  gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.AuthChainEvents),
 			Origin:      cfg.Matrix.ServerName,

federationapi/routing/keys.go

@@ -25,6 +25,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/ed25519"
@@ -144,7 +145,7 @@ func LocalKeys(cfg *config.FederationAPI, serverName gomatrixserverlib.ServerNam
 
 func localKeys(cfg *config.FederationAPI, serverName gomatrixserverlib.ServerName) (*gomatrixserverlib.ServerKeys, error) {
 	var keys gomatrixserverlib.ServerKeys
-	var identity *gomatrixserverlib.SigningIdentity
+	var identity *fclient.SigningIdentity
 	var err error
 	if virtualHost := cfg.Matrix.VirtualHostForHTTPHost(serverName); virtualHost == nil {
 		if identity, err = cfg.Matrix.SigningIdentityFor(cfg.Matrix.ServerName); err != nil {

federationapi/routing/missingevents.go

@@ -19,6 +19,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -67,7 +68,7 @@ func GetMissingEvents(
 
 	eventsResponse.Events = filterEvents(eventsResponse.Events, roomID)
 
-	resp := gomatrixserverlib.RespMissingEvents{
+	resp := fclient.RespMissingEvents{
 		Events: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(eventsResponse.Events),
 	}
 

federationapi/routing/peek.go

@@ -21,6 +21,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -87,7 +88,7 @@ func Peek(
 		return util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
 	}
 
-	respPeek := gomatrixserverlib.RespPeek{
+	respPeek := fclient.RespPeek{
 		StateEvents:     gomatrixserverlib.NewEventJSONsFromHeaderedEvents(response.StateEvents),
 		AuthEvents:      gomatrixserverlib.NewEventJSONsFromHeaderedEvents(response.AuthChainEvents),
 		RoomVersion:     response.RoomVersion,

federationapi/routing/profile.go

@@ -50,10 +50,7 @@ func GetProfile(
 		}
 	}
 
-	var profileRes userapi.QueryProfileResponse
-	err = userAPI.QueryProfile(httpReq.Context(), &userapi.QueryProfileRequest{
-		UserID: userID,
-	}, &profileRes)
+	profile, err := userAPI.QueryProfile(httpReq.Context(), userID)
 	if err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("userAPI.QueryProfile failed")
 		return jsonerror.InternalServerError()
@@ -65,21 +62,21 @@ func GetProfile(
 	if field != "" {
 		switch field {
 		case "displayname":
-			res = eventutil.DisplayName{
-				DisplayName: profileRes.DisplayName,
+			res = eventutil.UserProfile{
+				DisplayName: profile.DisplayName,
 			}
 		case "avatar_url":
-			res = eventutil.AvatarURL{
-				AvatarURL: profileRes.AvatarURL,
+			res = eventutil.UserProfile{
+				AvatarURL: profile.AvatarURL,
 			}
 		default:
 			code = http.StatusBadRequest
 			res = jsonerror.InvalidArgumentValue("The request body did not contain an allowed value of argument 'field'. Allowed values are either: 'avatar_url', 'displayname'.")
 		}
 	} else {
-		res = eventutil.ProfileResponse{
-			AvatarURL:   profileRes.AvatarURL,
-			DisplayName: profileRes.DisplayName,
+		res = eventutil.UserProfile{
+			AvatarURL:   profile.AvatarURL,
+			DisplayName: profile.DisplayName,
 		}
 	}
 

federationapi/routing/profile_test.go

@@ -23,11 +23,15 @@ import (
 	"testing"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
 	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	userAPI "github.com/matrix-org/dendrite/userapi/api"
@@ -40,29 +44,32 @@ type fakeUserAPI struct {
 	userAPI.FederationUserAPI
 }
 
-func (u *fakeUserAPI) QueryProfile(ctx context.Context, req *userAPI.QueryProfileRequest, res *userAPI.QueryProfileResponse) error {
-	return nil
+func (u *fakeUserAPI) QueryProfile(ctx context.Context, userID string) (*authtypes.Profile, error) {
+	return &authtypes.Profile{}, nil
 }
 
 func TestHandleQueryProfile(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, close := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		routers := httputil.NewRouters()
 		defer close()
 
 		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
-		base.PublicFederationAPIMux = fedMux
-		base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
-		base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
+		natsInstance := jetstream.NATSInstance{}
+		routers.Federation = fedMux
+		cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
+		cfg.FederationAPI.Matrix.Metrics.Enabled = false
 		fedClient := fakeFedClient{}
 		serverKeyAPI := &signing.YggdrasilKeys{}
 		keyRing := serverKeyAPI.KeyRing()
-		fedapi := fedAPI.NewInternalAPI(base, &fedClient, nil, nil, keyRing, true)
+		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true)
 		userapi := fakeUserAPI{}
 		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
 		if !ok {
 			panic("This is a programming error.")
 		}
-		routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, &base.Cfg.MSCs, nil, nil)
+		routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)

federationapi/routing/publicrooms.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -48,9 +49,9 @@ func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.FederationRoomser
 
 func publicRooms(
 	ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.FederationRoomserverAPI,
-) (*gomatrixserverlib.RespPublicRooms, error) {
+) (*fclient.RespPublicRooms, error) {
 
-	var response gomatrixserverlib.RespPublicRooms
+	var response fclient.RespPublicRooms
 	var limit int16
 	var offset int64
 	limit = request.Limit
@@ -122,7 +123,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 }
 
 // due to lots of switches
-func fillInRooms(ctx context.Context, roomIDs []string, rsAPI roomserverAPI.FederationRoomserverAPI) ([]gomatrixserverlib.PublicRoom, error) {
+func fillInRooms(ctx context.Context, roomIDs []string, rsAPI roomserverAPI.FederationRoomserverAPI) ([]fclient.PublicRoom, error) {
 	avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
 	nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
 	canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
@@ -144,10 +145,10 @@ func fillInRooms(ctx context.Context, roomIDs []string, rsAPI roomserverAPI.Fede
 		util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed")
 		return nil, err
 	}
-	chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs))
+	chunk := make([]fclient.PublicRoom, len(roomIDs))
 	i := 0
 	for roomID, data := range stateRes.Rooms {
-		pub := gomatrixserverlib.PublicRoom{
+		pub := fclient.PublicRoom{
 			RoomID: roomID,
 		}
 		joinCount := 0

federationapi/routing/query.go

@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -50,7 +51,7 @@ func RoomAliasToID(
 		}
 	}
 
-	var resp gomatrixserverlib.RespDirectory
+	var resp fclient.RespDirectory
 
 	if domain == cfg.Matrix.ServerName {
 		queryReq := &roomserverAPI.GetRoomIDForAliasRequest{
@@ -71,7 +72,7 @@ func RoomAliasToID(
 				return jsonerror.InternalServerError()
 			}
 
-			resp = gomatrixserverlib.RespDirectory{
+			resp = fclient.RespDirectory{
 				RoomID:  queryRes.RoomID,
 				Servers: serverQueryRes.ServerNames,
 			}

federationapi/routing/query_test.go

@@ -28,10 +28,14 @@ import (
 	fedclient "github.com/matrix-org/dendrite/federationapi/api"
 	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/crypto/ed25519"
 )
@@ -40,29 +44,32 @@ type fakeFedClient struct {
 	fedclient.FederationClient
 }
 
-func (f *fakeFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) {
+func (f *fakeFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res fclient.RespDirectory, err error) {
 	return
 }
 
 func TestHandleQueryDirectory(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, close := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		routers := httputil.NewRouters()
 		defer close()
 
 		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
-		base.PublicFederationAPIMux = fedMux
-		base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
-		base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
+		natsInstance := jetstream.NATSInstance{}
+		routers.Federation = fedMux
+		cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
+		cfg.FederationAPI.Matrix.Metrics.Enabled = false
 		fedClient := fakeFedClient{}
 		serverKeyAPI := &signing.YggdrasilKeys{}
 		keyRing := serverKeyAPI.KeyRing()
-		fedapi := fedAPI.NewInternalAPI(base, &fedClient, nil, nil, keyRing, true)
+		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true)
 		userapi := fakeUserAPI{}
 		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
 		if !ok {
 			panic("This is a programming error.")
 		}
-		routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, &base.Cfg.MSCs, nil, nil)
+		routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)

federationapi/routing/routing.go

@@ -31,7 +31,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -55,7 +54,8 @@ const (
 // applied:
 // nolint: gocyclo
 func Setup(
-	base *base.BaseDendrite,
+	routers httputil.Routers,
+	dendriteCfg *config.Dendrite,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	fsAPI *fedInternal.FederationInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
@@ -63,14 +63,14 @@ func Setup(
 	userAPI userapi.FederationUserAPI,
 	mscCfg *config.MSCs,
 	servers federationAPI.ServersInRoomProvider,
-	producer *producers.SyncAPIProducer,
+	producer *producers.SyncAPIProducer, enableMetrics bool,
 ) {
-	fedMux := base.PublicFederationAPIMux
-	keyMux := base.PublicKeyAPIMux
-	wkMux := base.PublicWellKnownAPIMux
-	cfg := &base.Cfg.FederationAPI
+	fedMux := routers.Federation
+	keyMux := routers.Keys
+	wkMux := routers.WellKnown
+	cfg := &dendriteCfg.FederationAPI
 
-	if base.EnableMetrics {
+	if enableMetrics {
 		prometheus.MustRegister(
 			internal.PDUCountTotal, internal.EDUCountTotal,
 		)

federationapi/routing/send_test.go

@@ -25,7 +25,10 @@ import (
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
 	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -44,21 +47,24 @@ type sendContent struct {
 
 func TestHandleSend(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		base, close := testrig.CreateBaseDendrite(t, dbType)
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		routers := httputil.NewRouters()
 		defer close()
 
 		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
-		base.PublicFederationAPIMux = fedMux
-		base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
-		base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
-		fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true)
+		natsInstance := jetstream.NATSInstance{}
+		routers.Federation = fedMux
+		cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
+		cfg.FederationAPI.Matrix.Metrics.Enabled = false
+		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, nil, nil, nil, true)
 		serverKeyAPI := &signing.YggdrasilKeys{}
 		keyRing := serverKeyAPI.KeyRing()
 		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
 		if !ok {
 			panic("This is a programming error.")
 		}
-		routing.Setup(base, nil, r, keyRing, nil, nil, &base.Cfg.MSCs, nil, nil)
+		routing.Setup(routers, cfg, nil, r, keyRing, nil, nil, &cfg.MSCs, nil, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)

federationapi/routing/state.go

@@ -20,6 +20,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/util"
 )
 
@@ -40,7 +41,7 @@ func GetState(
 		return *err
 	}
 
-	return util.JSONResponse{Code: http.StatusOK, JSON: &gomatrixserverlib.RespState{
+	return util.JSONResponse{Code: http.StatusOK, JSON: &fclient.RespState{
 		AuthEvents:  gomatrixserverlib.NewEventJSONsFromHeaderedEvents(authChain),
 		StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateEvents),
 	}}
@@ -66,7 +67,7 @@ func GetStateIDs(
 	stateEventIDs := getIDsFromEvent(stateEvents)
 	authEventIDs := getIDsFromEvent(authEvents)
 
-	return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.RespStateIDs{
+	return util.JSONResponse{Code: http.StatusOK, JSON: fclient.RespStateIDs{
 		StateEventIDs: stateEventIDs,
 		AuthEventIDs:  authEventIDs,
 	},

federationapi/routing/threepid.go

@@ -256,17 +256,14 @@ func createInviteFrom3PIDInvite(
 		StateKey: &inv.MXID,
 	}
 
-	var res userapi.QueryProfileResponse
-	err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{
-		UserID: inv.MXID,
-	}, &res)
+	profile, err := userAPI.QueryProfile(ctx, inv.MXID)
 	if err != nil {
 		return nil, err
 	}
 
 	content := gomatrixserverlib.MemberContent{
-		AvatarURL:   res.AvatarURL,
-		DisplayName: res.DisplayName,
+		AvatarURL:   profile.AvatarURL,
+		DisplayName: profile.DisplayName,
 		Membership:  gomatrixserverlib.Invite,
 		ThirdPartyInvite: &gomatrixserverlib.MemberThirdPartyInvite{
 			Signed: inv.Signed,

federationapi/storage/postgres/blacklist_table.go

@@ -60,19 +60,12 @@ func NewPostgresBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) {
 		return
 	}
 
-	if s.insertBlacklistStmt, err = db.Prepare(insertBlacklistSQL); err != nil {
-		return
-	}
-	if s.selectBlacklistStmt, err = db.Prepare(selectBlacklistSQL); err != nil {
-		return
-	}
-	if s.deleteBlacklistStmt, err = db.Prepare(deleteBlacklistSQL); err != nil {
-		return
-	}
-	if s.deleteAllBlacklistStmt, err = db.Prepare(deleteAllBlacklistSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertBlacklistStmt, insertBlacklistSQL},
+		{&s.selectBlacklistStmt, selectBlacklistSQL},
+		{&s.deleteBlacklistStmt, deleteBlacklistSQL},
+		{&s.deleteAllBlacklistStmt, deleteAllBlacklistSQL},
+	}.Prepare(db)
 }
 
 func (s *blacklistStatements) InsertBlacklist(

federationapi/storage/postgres/joined_hosts_table.go

@@ -90,28 +90,15 @@ func NewPostgresJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err erro
 	if err != nil {
 		return
 	}
-	if s.insertJoinedHostsStmt, err = s.db.Prepare(insertJoinedHostsSQL); err != nil {
-		return
-	}
-	if s.deleteJoinedHostsStmt, err = s.db.Prepare(deleteJoinedHostsSQL); err != nil {
-		return
-	}
-	if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil {
-		return
-	}
-	if s.selectJoinedHostsStmt, err = s.db.Prepare(selectJoinedHostsSQL); err != nil {
-		return
-	}
-	if s.selectAllJoinedHostsStmt, err = s.db.Prepare(selectAllJoinedHostsSQL); err != nil {
-		return
-	}
-	if s.selectJoinedHostsForRoomsStmt, err = s.db.Prepare(selectJoinedHostsForRoomsSQL); err != nil {
-		return
-	}
-	if s.selectJoinedHostsForRoomsExcludingBlacklistedStmt, err = s.db.Prepare(selectJoinedHostsForRoomsExcludingBlacklistedSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertJoinedHostsStmt, insertJoinedHostsSQL},
+		{&s.deleteJoinedHostsStmt, deleteJoinedHostsSQL},
+		{&s.deleteJoinedHostsForRoomStmt, deleteJoinedHostsForRoomSQL},
+		{&s.selectJoinedHostsStmt, selectJoinedHostsSQL},
+		{&s.selectAllJoinedHostsStmt, selectAllJoinedHostsSQL},
+		{&s.selectJoinedHostsForRoomsStmt, selectJoinedHostsForRoomsSQL},
+		{&s.selectJoinedHostsForRoomsExcludingBlacklistedStmt, selectJoinedHostsForRoomsExcludingBlacklistedSQL},
+	}.Prepare(db)
 }
 
 func (s *joinedHostsStatements) InsertJoinedHosts(

federationapi/storage/postgres/notary_server_keys_json_table.go

@@ -19,6 +19,7 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/federationapi/storage/tables"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -50,10 +51,9 @@ func NewPostgresNotaryServerKeysTable(db *sql.DB) (s *notaryServerKeysStatements
 		return
 	}
 
-	if s.insertServerKeysJSONStmt, err = db.Prepare(insertServerKeysJSONSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertServerKeysJSONStmt, insertServerKeysJSONSQL},
+	}.Prepare(db)
 }
 
 func (s *notaryServerKeysStatements) InsertJSONResponse(

federationapi/storage/postgres/notary_server_keys_metadata_table.go

@@ -22,6 +22,7 @@ import (
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/federationapi/storage/tables"
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -91,22 +92,13 @@ func NewPostgresNotaryServerKeysMetadataTable(db *sql.DB) (s *notaryServerKeysMe
 		return
 	}
 
-	if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil {
-		return
-	}
-	if s.selectNotaryKeyResponsesStmt, err = db.Prepare(selectNotaryKeyResponsesSQL); err != nil {
-		return
-	}
-	if s.selectNotaryKeyResponsesWithKeyIDsStmt, err = db.Prepare(selectNotaryKeyResponsesWithKeyIDsSQL); err != nil {
-		return
-	}
-	if s.selectNotaryKeyMetadataStmt, err = db.Prepare(selectNotaryKeyMetadataSQL); err != nil {
-		return
-	}
-	if s.deleteUnusedServerKeysJSONStmt, err = db.Prepare(deleteUnusedServerKeysJSONSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.upsertServerKeysStmt, upsertServerKeysSQL},
+		{&s.selectNotaryKeyResponsesStmt, selectNotaryKeyResponsesSQL},
+		{&s.selectNotaryKeyResponsesWithKeyIDsStmt, selectNotaryKeyResponsesWithKeyIDsSQL},
+		{&s.selectNotaryKeyMetadataStmt, selectNotaryKeyMetadataSQL},
+		{&s.deleteUnusedServerKeysJSONStmt, deleteUnusedServerKeysJSONSQL},
+	}.Prepare(db)
 }
 
 func (s *notaryServerKeysMetadataStatements) UpsertKey(

federationapi/storage/postgres/queue_json_table.go

@@ -65,16 +65,11 @@ func NewPostgresQueueJSONTable(db *sql.DB) (s *queueJSONStatements, err error) {
 	if err != nil {
 		return
 	}
-	if s.insertJSONStmt, err = s.db.Prepare(insertJSONSQL); err != nil {
-		return
-	}
-	if s.deleteJSONStmt, err = s.db.Prepare(deleteJSONSQL); err != nil {
-		return
-	}
-	if s.selectJSONStmt, err = s.db.Prepare(selectJSONSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertJSONStmt, insertJSONSQL},
+		{&s.deleteJSONStmt, deleteJSONSQL},
+		{&s.selectJSONStmt, selectJSONSQL},
+	}.Prepare(db)
 }
 
 func (s *queueJSONStatements) InsertQueueJSON(

federationapi/storage/postgres/queue_pdus_table.go

@@ -78,22 +78,13 @@ func NewPostgresQueuePDUsTable(db *sql.DB) (s *queuePDUsStatements, err error) {
 	if err != nil {
 		return
 	}
-	if s.insertQueuePDUStmt, err = s.db.Prepare(insertQueuePDUSQL); err != nil {
-		return
-	}
-	if s.deleteQueuePDUsStmt, err = s.db.Prepare(deleteQueuePDUSQL); err != nil {
-		return
-	}
-	if s.selectQueuePDUsStmt, err = s.db.Prepare(selectQueuePDUsSQL); err != nil {
-		return
-	}
-	if s.selectQueuePDUReferenceJSONCountStmt, err = s.db.Prepare(selectQueuePDUReferenceJSONCountSQL); err != nil {
-		return
-	}
-	if s.selectQueuePDUServerNamesStmt, err = s.db.Prepare(selectQueuePDUServerNamesSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertQueuePDUStmt, insertQueuePDUSQL},
+		{&s.deleteQueuePDUsStmt, deleteQueuePDUSQL},
+		{&s.selectQueuePDUsStmt, selectQueuePDUsSQL},
+		{&s.selectQueuePDUReferenceJSONCountStmt, selectQueuePDUReferenceJSONCountSQL},
+		{&s.selectQueuePDUServerNamesStmt, selectQueuePDUServerNamesSQL},
+	}.Prepare(db)
 }
 
 func (s *queuePDUsStatements) InsertQueuePDU(

federationapi/storage/postgres/server_key_table.go

@@ -72,13 +72,10 @@ func NewPostgresServerSigningKeysTable(db *sql.DB) (s *serverSigningKeyStatement
 	if err != nil {
 		return
 	}
-	if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerSigningKeysSQL); err != nil {
-		return
-	}
-	if s.upsertServerKeysStmt, err = db.Prepare(upsertServerSigningKeysSQL); err != nil {
-		return
-	}
-	return s, nil
+	return s, sqlutil.StatementList{
+		{&s.bulkSelectServerKeysStmt, bulkSelectServerSigningKeysSQL},
+		{&s.upsertServerKeysStmt, upsertServerSigningKeysSQL},
+	}.Prepare(db)
 }
 
 func (s *serverSigningKeyStatements) BulkSelectServerKeys(

federationapi/storage/postgres/storage.go

@@ -16,6 +16,7 @@
 package postgres
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 
@@ -23,7 +24,6 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -36,10 +36,10 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) {
+func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
+	if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil {
 		return nil, err
 	}
 	blacklist, err := NewPostgresBlacklistTable(d.db)
@@ -95,7 +95,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 		Version: "federationsender: drop federationsender_rooms",
 		Up:      deltas.UpRemoveRoomsTable,
 	})
-	err = m.Up(base.Context())
+	err = m.Up(ctx)
 	if err != nil {
 		return nil, err
 	}

federationapi/storage/sqlite3/blacklist_table.go

@@ -60,19 +60,12 @@ func NewSQLiteBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) {
 		return
 	}
 
-	if s.insertBlacklistStmt, err = db.Prepare(insertBlacklistSQL); err != nil {
-		return
-	}
-	if s.selectBlacklistStmt, err = db.Prepare(selectBlacklistSQL); err != nil {
-		return
-	}
-	if s.deleteBlacklistStmt, err = db.Prepare(deleteBlacklistSQL); err != nil {
-		return
-	}
-	if s.deleteAllBlacklistStmt, err = db.Prepare(deleteAllBlacklistSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertBlacklistStmt, insertBlacklistSQL},
+		{&s.selectBlacklistStmt, selectBlacklistSQL},
+		{&s.deleteBlacklistStmt, deleteBlacklistSQL},
+		{&s.deleteAllBlacklistStmt, deleteAllBlacklistSQL},
+	}.Prepare(db)
 }
 
 func (s *blacklistStatements) InsertBlacklist(

federationapi/storage/sqlite3/joined_hosts_table.go

@@ -90,22 +90,14 @@ func NewSQLiteJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err error)
 	if err != nil {
 		return
 	}
-	if s.insertJoinedHostsStmt, err = db.Prepare(insertJoinedHostsSQL); err != nil {
-		return
-	}
-	if s.deleteJoinedHostsStmt, err = db.Prepare(deleteJoinedHostsSQL); err != nil {
-		return
-	}
-	if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil {
-		return
-	}
-	if s.selectJoinedHostsStmt, err = db.Prepare(selectJoinedHostsSQL); err != nil {
-		return
-	}
-	if s.selectAllJoinedHostsStmt, err = db.Prepare(selectAllJoinedHostsSQL); err != nil {
-		return
-	}
-	return
+
+	return s, sqlutil.StatementList{
+		{&s.insertJoinedHostsStmt, insertJoinedHostsSQL},
+		{&s.deleteJoinedHostsStmt, deleteJoinedHostsSQL},
+		{&s.deleteJoinedHostsForRoomStmt, deleteJoinedHostsForRoomSQL},
+		{&s.selectJoinedHostsStmt, selectJoinedHostsSQL},
+		{&s.selectAllJoinedHostsStmt, selectAllJoinedHostsSQL},
+	}.Prepare(db)
 }
 
 func (s *joinedHostsStatements) InsertJoinedHosts(

federationapi/storage/sqlite3/notary_server_keys_json_table.go

@@ -19,6 +19,7 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/federationapi/storage/tables"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -49,10 +50,9 @@ func NewSQLiteNotaryServerKeysTable(db *sql.DB) (s *notaryServerKeysStatements,
 		return
 	}
 
-	if s.insertServerKeysJSONStmt, err = db.Prepare(insertServerKeysJSONSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertServerKeysJSONStmt, insertServerKeysJSONSQL},
+	}.Prepare(db)
 }
 
 func (s *notaryServerKeysStatements) InsertJSONResponse(

federationapi/storage/sqlite3/notary_server_keys_metadata_table.go

@@ -92,19 +92,12 @@ func NewSQLiteNotaryServerKeysMetadataTable(db *sql.DB) (s *notaryServerKeysMeta
 		return
 	}
 
-	if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil {
-		return
-	}
-	if s.selectNotaryKeyResponsesStmt, err = db.Prepare(selectNotaryKeyResponsesSQL); err != nil {
-		return
-	}
-	if s.selectNotaryKeyMetadataStmt, err = db.Prepare(selectNotaryKeyMetadataSQL); err != nil {
-		return
-	}
-	if s.deleteUnusedServerKeysJSONStmt, err = db.Prepare(deleteUnusedServerKeysJSONSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.upsertServerKeysStmt, upsertServerKeysSQL},
+		{&s.selectNotaryKeyResponsesStmt, selectNotaryKeyResponsesSQL},
+		{&s.selectNotaryKeyMetadataStmt, selectNotaryKeyMetadataSQL},
+		{&s.deleteUnusedServerKeysJSONStmt, deleteUnusedServerKeysJSONSQL},
+	}.Prepare(db)
 }
 
 func (s *notaryServerKeysMetadataStatements) UpsertKey(

federationapi/storage/sqlite3/queue_json_table.go

@@ -66,10 +66,10 @@ func NewSQLiteQueueJSONTable(db *sql.DB) (s *queueJSONStatements, err error) {
 	if err != nil {
 		return
 	}
-	if s.insertJSONStmt, err = db.Prepare(insertJSONSQL); err != nil {
-		return
-	}
-	return
+
+	return s, sqlutil.StatementList{
+		{&s.insertJSONStmt, insertJSONSQL},
+	}.Prepare(db)
 }
 
 func (s *queueJSONStatements) InsertQueueJSON(