Merge branch 'main' into s7evink/guestaccess

.github/codecov.yaml

@@ -0,0 +1,20 @@
+flag_management:
+  default_rules:
+    carryforward: true
+
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 0%
+        base: auto
+        flags:
+          - unittests
+    patch:
+      default:
+        target: 75%
+        threshold: 0%
+        base: auto
+        flags:
+          - unittests

.github/workflows/dendrite.yml

@@ -68,7 +68,7 @@ jobs:
 
   # run go test with different go versions
   test:
-    timeout-minutes: 5
+    timeout-minutes: 10
     name: Unit tests (Go ${{ matrix.go }})
     runs-on: ubuntu-latest
     # Service containers to run with `container-job`
@@ -94,14 +94,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.18", "1.19"]
+        go: ["1.19"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
         uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
-          cache: true
+      - uses: actions/cache@v3
+        # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go${{ matrix.go }}-unit-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go${{ matrix.go }}-unit-
       - name: Set up gotestfmt
         uses: gotesttools/gotestfmt-action@v2
         with:
@@ -194,6 +202,66 @@ jobs:
         with:
           jobs: ${{ toJSON(needs) }}
 
+  # run go test with different go versions
+  integration:
+    timeout-minutes: 20
+    needs: initial-tests-done
+    name: Integration tests (Go ${{ matrix.go }})
+    runs-on: ubuntu-latest
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:13-alpine
+        # Provide the password for postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.19"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Set up gotestfmt
+        uses: gotesttools/gotestfmt-action@v2
+        with:
+          # Optional: pass GITHUB_TOKEN to avoid rate limiting.
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go${{ matrix.go }}-test-race-
+      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: unittests
+
   # run database upgrade tests
   upgrade_test:
     name: Upgrade tests
@@ -404,6 +472,7 @@ jobs:
         upgrade_test_direct,
         sytest,
         complement,
+        integration
       ]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped

.github/workflows/docker.yml

@@ -68,18 +68,6 @@ jobs:
             ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
       - name: Build release monolith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_monolith_release
@@ -98,6 +86,18 @@ jobs:
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
 
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results.sarif"
+
   polylith:
     name: Polylith image
     runs-on: ubuntu-latest
@@ -148,18 +148,6 @@ jobs:
             ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
       - name: Build release polylith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_polylith_release
@@ -178,6 +166,18 @@ jobs:
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
 
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results.sarif"
+
   demo-pinecone:
     name: Pinecone demo image
     runs-on: ubuntu-latest

.github/workflows/schedules.yaml

@@ -10,79 +10,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # run go test with different go versions
-  test:
-    timeout-minutes: 20
-    name: Unit tests (Go ${{ matrix.go }})
-    runs-on: ubuntu-latest
-    # Service containers to run with `container-job`
-    services:
-      # Label used to access the service container
-      postgres:
-        # Docker Hub image
-        image: postgres:13-alpine
-        # Provide the password for postgres
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: dendrite
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    strategy:
-      fail-fast: false
-      matrix:
-        go: ["1.18", "1.19"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ matrix.go }}
-      - name: Set up gotestfmt
-        uses: gotesttools/gotestfmt-action@v2
-        with:
-          # Optional: pass GITHUB_TOKEN to avoid rate limiting.
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go${{ matrix.go }}-test-race-
-      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
-        env:
-          POSTGRES_HOST: localhost
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: dendrite
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-
-  # Dummy step to gate other tests on without repeating the whole list
-  initial-tests-done:
-    name: Initial tests passed
-    needs: [test]
-    runs-on: ubuntu-latest
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    steps:
-      - name: Check initial tests passed
-        uses: re-actors/alls-green@release/v1
-        with:
-          jobs: ${{ toJSON(needs) }}
-
   # run Sytest in different variations
   sytest:
     timeout-minutes: 60
-    needs: initial-tests-done
     name: "Sytest (${{ matrix.label }})"
     runs-on: ubuntu-latest
     strategy:
@@ -104,13 +34,23 @@ jobs:
       image: matrixdotorg/sytest-dendrite:latest
       volumes:
         - ${{ github.workspace }}:/src
+        - /root/.cache/go-build:/github/home/.cache/go-build
+        - /root/.cache/go-mod:/gopath/pkg/mod
       env:
         POSTGRES: ${{ matrix.postgres && 1}}
         API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
         RACE_DETECTION: 1
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            /gopath/pkg/mod
+          key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-sytest-
       - name: Run Sytest
         run: /bootstrap.sh dendrite
         working-directory: /src

CHANGES.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## Dendrite 0.10.8 (2022-11-29)
+
+### Features
+
+* The built-in NATS Server has been updated to version 2.9.8
+* A number of under-the-hood changes have been merged for future virtual hosting support in Dendrite (running multiple domain names on the same Dendrite deployment)
+
+### Fixes
+
+* Event auth handling of invites has been refactored, which should fix some edge cases being handled incorrectly
+* Fix a bug when returning an empty protocol list, which could cause Element to display "The homeserver may be too old to support third party networks" when opening the public room directory
+* The sync API will no longer filter out the user's own membership when using lazy-loading
+* Dendrite will now correctly detect JetStream consumers being deleted, stopping the consumer goroutine as needed
+* A panic in the federation API where the server list could go out of bounds has been fixed
+* Blacklisted servers will now be excluded when querying joined servers, which improves CPU usage and performs less unnecessary outbound requests
+* A database writer will now be used to assign state key NIDs when requesting NIDs that may not exist yet
+* Dendrite will now correctly move local aliases for an upgraded room when the room is upgraded remotely
+* Dendrite will now correctly move account data for an upgraded room when the room is upgraded remotely
+* Missing state key NIDs will now be allocated on request rather than returning an error
+* Guest access is now correctly denied on a number of endpoints
+* Presence information will now be correctly sent for new private chats
+* A number of unspecced fields have been removed from outbound `/send` transactions
+
 ## Dendrite 0.10.7 (2022-11-04)
 
 ### Features

appservice/appservice.go

@@ -24,6 +24,8 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/appservice/consumers"
 	"github.com/matrix-org/dendrite/appservice/inthttp"
@@ -32,12 +34,11 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddInternalRoutes registers HTTP handlers for internal API calls
-func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI) {
+func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI, enableMetrics bool) {
-	inthttp.AddRoutes(queryAPI, router)
+	inthttp.AddRoutes(queryAPI, router, enableMetrics)
 }
 
 // NewInternalAPI returns a concerete implementation of the internal API. Callers

appservice/appservice_test.go

@@ -0,0 +1,223 @@
+package appservice_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/gorilla/mux"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/appservice/inthttp"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/userapi"
+
+	"github.com/matrix-org/dendrite/test/testrig"
+)
+
+func TestAppserviceInternalAPI(t *testing.T) {
+
+	// Set expected results
+	existingProtocol := "irc"
+	wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}}
+	wantProtocolResult := map[string]api.ASProtocolResponse{
+		existingProtocol: wantProtocolResponse,
+	}
+
+	// create a dummy AS url, handling some cases
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.Contains(r.URL.Path, "location"):
+			// Check if we've got an existing protocol, if so, return a proper response.
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "user"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "protocol"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode(nil); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		default:
+			t.Logf("hit location: %s", r.URL.Path)
+		}
+	}))
+
+	// The test cases to run
+	runCases := func(t *testing.T, testAPI api.AppServiceInternalAPI) {
+		t.Run("UserIDExists", func(t *testing.T) {
+			testUserIDExists(t, testAPI, "@as-testing:test", true)
+			testUserIDExists(t, testAPI, "@as1-testing:test", false)
+		})
+
+		t.Run("AliasExists", func(t *testing.T) {
+			testAliasExists(t, testAPI, "@asroom-testing:test", true)
+			testAliasExists(t, testAPI, "@asroom1-testing:test", false)
+		})
+
+		t.Run("Locations", func(t *testing.T) {
+			testLocations(t, testAPI, existingProtocol, wantLocationResponse)
+			testLocations(t, testAPI, "abc", nil)
+		})
+
+		t.Run("User", func(t *testing.T) {
+			testUser(t, testAPI, existingProtocol, wantUserResponse)
+			testUser(t, testAPI, "abc", nil)
+		})
+
+		t.Run("Protocols", func(t *testing.T) {
+			testProtocol(t, testAPI, existingProtocol, wantProtocolResult)
+			testProtocol(t, testAPI, existingProtocol, wantProtocolResult) // tests the cache
+			testProtocol(t, testAPI, "", wantProtocolResult)               // tests getting all protocols
+			testProtocol(t, testAPI, "abc", nil)
+		})
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, closeBase := testrig.CreateBaseDendrite(t, dbType)
+		defer closeBase()
+
+		// Create a dummy application service
+		base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
+			{
+				ID:              "someID",
+				URL:             srv.URL,
+				ASToken:         "",
+				HSToken:         "",
+				SenderLocalpart: "senderLocalPart",
+				NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+					"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
+					"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
+				},
+				Protocols: []string{existingProtocol},
+			},
+		}
+
+		// Create required internal APIs
+		rsAPI := roomserver.NewInternalAPI(base)
+		usrAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, nil, rsAPI, nil)
+		asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
+
+		// Finally execute the tests
+		t.Run("HTTP API", func(t *testing.T) {
+			router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
+			appservice.AddInternalRoutes(router, asAPI, base.EnableMetrics)
+			apiURL, cancel := test.ListenAndServe(t, router, false)
+			defer cancel()
+
+			asHTTPApi, err := inthttp.NewAppserviceClient(apiURL, &http.Client{})
+			if err != nil {
+				t.Fatalf("failed to create HTTP client: %s", err)
+			}
+			runCases(t, asHTTPApi)
+		})
+
+		t.Run("Monolith", func(t *testing.T) {
+			runCases(t, asAPI)
+		})
+	})
+}
+
+func testUserIDExists(t *testing.T, asAPI api.AppServiceInternalAPI, userID string, wantExists bool) {
+	ctx := context.Background()
+	userResp := &api.UserIDExistsResponse{}
+
+	if err := asAPI.UserIDExists(ctx, &api.UserIDExistsRequest{
+		UserID: userID,
+	}, userResp); err != nil {
+		t.Errorf("failed to get userID: %s", err)
+	}
+	if userResp.UserIDExists != wantExists {
+		t.Errorf("unexpected result for UserIDExists(%s): %v, expected %v", userID, userResp.UserIDExists, wantExists)
+	}
+}
+
+func testAliasExists(t *testing.T, asAPI api.AppServiceInternalAPI, alias string, wantExists bool) {
+	ctx := context.Background()
+	aliasResp := &api.RoomAliasExistsResponse{}
+
+	if err := asAPI.RoomAliasExists(ctx, &api.RoomAliasExistsRequest{
+		Alias: alias,
+	}, aliasResp); err != nil {
+		t.Errorf("failed to get alias: %s", err)
+	}
+	if aliasResp.AliasExists != wantExists {
+		t.Errorf("unexpected result for RoomAliasExists(%s): %v, expected %v", alias, aliasResp.AliasExists, wantExists)
+	}
+}
+
+func testLocations(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASLocationResponse) {
+	ctx := context.Background()
+	locationResp := &api.LocationResponse{}
+
+	if err := asAPI.Locations(ctx, &api.LocationRequest{
+		Protocol: proto,
+	}, locationResp); err != nil {
+		t.Errorf("failed to get locations: %s", err)
+	}
+	if !reflect.DeepEqual(locationResp.Locations, wantResult) {
+		t.Errorf("unexpected result for Locations(%s): %+v, expected %+v", proto, locationResp.Locations, wantResult)
+	}
+}
+
+func testUser(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASUserResponse) {
+	ctx := context.Background()
+	userResp := &api.UserResponse{}
+
+	if err := asAPI.User(ctx, &api.UserRequest{
+		Protocol: proto,
+	}, userResp); err != nil {
+		t.Errorf("failed to get user: %s", err)
+	}
+	if !reflect.DeepEqual(userResp.Users, wantResult) {
+		t.Errorf("unexpected result for User(%s): %+v, expected %+v", proto, userResp.Users, wantResult)
+	}
+}
+
+func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult map[string]api.ASProtocolResponse) {
+	ctx := context.Background()
+	protoResp := &api.ProtocolResponse{}
+
+	if err := asAPI.Protocols(ctx, &api.ProtocolRequest{
+		Protocol: proto,
+	}, protoResp); err != nil {
+		t.Errorf("failed to get Protocols: %s", err)
+	}
+	if !reflect.DeepEqual(protoResp.Protocols, wantResult) {
+		t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
+	}
+}

appservice/inthttp/server.go

@@ -8,29 +8,29 @@ import (
 )
 
 // AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
-func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
+func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
 	internalAPIMux.Handle(
 		AppServiceRoomAliasExistsPath,
-		httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", a.RoomAliasExists),
+		httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", enableMetrics, a.RoomAliasExists),
 	)
 
 	internalAPIMux.Handle(
 		AppServiceUserIDExistsPath,
-		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
+		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", enableMetrics, a.UserIDExists),
 	)
 
 	internalAPIMux.Handle(
 		AppServiceProtocolsPath,
-		httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
+		httputil.MakeInternalRPCAPI("AppserviceProtocols", enableMetrics, a.Protocols),
 	)
 
 	internalAPIMux.Handle(
 		AppServiceLocationsPath,
-		httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
+		httputil.MakeInternalRPCAPI("AppserviceLocations", enableMetrics, a.Locations),
 	)
 
 	internalAPIMux.Handle(
 		AppServiceUserPath,
-		httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
+		httputil.MakeInternalRPCAPI("AppserviceUser", enableMetrics, a.User),
 	)
 }

build/gobind-pinecone/monolith.go

@@ -336,6 +336,7 @@ func (m *DendriteMonolith) Start() {
 	}
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
+	base.ConfigureAdminEndpoints()
 	defer base.Close() // nolint: errcheck
 
 	federation := conn.CreateFederationClient(base, m.PineconeQUIC)
@@ -382,6 +383,8 @@ func (m *DendriteMonolith) Start() {
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 	httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
 
 	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()

build/gobind-yggdrasil/monolith.go

@@ -150,6 +150,7 @@ func (m *DendriteMonolith) Start() {
 	}
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
+	base.ConfigureAdminEndpoints()
 	m.processContext = base.ProcessContext
 	defer base.Close() // nolint: errcheck
 
@@ -196,6 +197,8 @@ func (m *DendriteMonolith) Start() {
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 
 	yggRouter := mux.NewRouter()
 	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)

cmd/create-account/main.go

@@ -177,7 +177,7 @@ func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, a
 	defer regResp.Body.Close() // nolint: errcheck
 	if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
 		body, _ = io.ReadAll(regResp.Body)
-		return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
+		return "", fmt.Errorf("got HTTP %d error from server: %s", regResp.StatusCode, string(body))
 	}
 	r, err := io.ReadAll(regResp.Body)
 	if err != nil {

cmd/dendrite-demo-pinecone/main.go

@@ -155,6 +155,7 @@ func main() {
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
+	base.ConfigureAdminEndpoints()
 	defer base.Close() // nolint: errcheck
 
 	pineconeEventChannel := make(chan pineconeEvents.Event)
@@ -248,6 +249,8 @@ func main() {
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 	httpRouter.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
 		c, err := wsUpgrader.Upgrade(w, r, nil)
 		if err != nil {

cmd/dendrite-demo-yggdrasil/main.go

@@ -144,6 +144,7 @@ func main() {
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
+	base.ConfigureAdminEndpoints()
 	defer base.Close() // nolint: errcheck
 
 	ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
@@ -198,6 +199,8 @@ func main() {
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 	embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
 
 	yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()

cmd/dendrite-monolith-server/main.go

@@ -18,6 +18,8 @@ import (
 	"flag"
 	"os"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/keyserver"
@@ -29,7 +31,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/userapi"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -75,7 +76,7 @@ func main() {
 	// call functions directly on the impl unless running in HTTP mode
 	rsAPI := rsImpl
 	if base.UseHTTPAPIs {
-		roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl)
+		roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl, base.EnableMetrics)
 		rsAPI = base.RoomserverHTTPClient()
 	}
 	if traceInternal {
@@ -89,7 +90,7 @@ func main() {
 	)
 	fsImplAPI := fsAPI
 	if base.UseHTTPAPIs {
-		federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
+		federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
 		fsAPI = base.FederationAPIHTTPClient()
 	}
 	keyRing := fsAPI.KeyRing()
@@ -97,7 +98,7 @@ func main() {
 	keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
 	keyAPI := keyImpl
 	if base.UseHTTPAPIs {
-		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
+		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI, base.EnableMetrics)
 		keyAPI = base.KeyServerHTTPClient()
 	}
 
@@ -105,7 +106,7 @@ func main() {
 	userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
 	userAPI := userImpl
 	if base.UseHTTPAPIs {
-		userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
+		userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
 		userAPI = base.UserAPIClient()
 	}
 	if traceInternal {
@@ -119,7 +120,7 @@ func main() {
 	// before the listeners are up.
 	asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI)
 	if base.UseHTTPAPIs {
-		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
+		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI, base.EnableMetrics)
 		asAPI = base.AppserviceHTTPClient()
 	}
 

cmd/dendrite-polylith-multi/personalities/appservice.go

@@ -26,7 +26,7 @@ func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) {
 	rsAPI := base.RoomserverHTTPClient()
 
 	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
+	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -34,7 +34,7 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 		rsAPI, fsAPI, keyAPI, nil,
 	)
 
-	federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
+	federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.FederationAPI.InternalAPI.Listen,

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -25,7 +25,7 @@ func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
 	intAPI.SetUserAPI(base.UserAPIClient())
 
-	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
+	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.KeyServer.InternalAPI.Listen, // internal listener

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -26,7 +26,7 @@ func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	rsAPI := roomserver.NewInternalAPI(base)
 	rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing())
 	rsAPI.SetAppserviceAPI(asAPI)
-	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
+	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.RoomServer.InternalAPI.Listen, // internal listener

cmd/dendrite-polylith-multi/personalities/userapi.go

@@ -27,7 +27,7 @@ func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 		base.PushGatewayHTTPClient(),
 	)
 
-	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
+	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.UserAPI.InternalAPI.Listen, // internal listener

cmd/dendrite-upgrade-tests/main.go

@@ -7,6 +7,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -61,6 +62,7 @@ COPY . .
 RUN go build ./cmd/dendrite-monolith-server
 RUN go build ./cmd/generate-keys
 RUN go build ./cmd/generate-config
+RUN go build ./cmd/create-account
 RUN ./generate-config --ci > dendrite.yaml
 RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
 
@@ -104,6 +106,7 @@ COPY . .
 RUN go build ./cmd/dendrite-monolith-server
 RUN go build ./cmd/generate-keys
 RUN go build ./cmd/generate-config
+RUN go build ./cmd/create-account
 RUN ./generate-config --ci > dendrite.yaml
 RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
 
@@ -458,6 +461,46 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo
 	if err = runTests(csAPIURL, v); err != nil {
 		return fmt.Errorf("failed to run tests on version %s: %s", v, err)
 	}
+
+	err = testCreateAccount(dockerClient, v, containerID)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// test that create-account is working
+func testCreateAccount(dockerClient *client.Client, v string, containerID string) error {
+	createUser := strings.ToLower("createaccountuser-" + v)
+	log.Printf("%s: Creating account %s with create-account\n", v, createUser)
+
+	respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{
+		AttachStderr: true,
+		AttachStdout: true,
+		Cmd: []string{
+			"/build/create-account",
+			"-username", createUser,
+			"-password", "someRandomPassword",
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to ContainerExecCreate: %w", err)
+	}
+
+	response, err := dockerClient.ContainerExecAttach(context.Background(), respID.ID, types.ExecStartCheck{})
+	if err != nil {
+		return fmt.Errorf("failed to attach to container: %w", err)
+	}
+	defer response.Close()
+
+	data, err := ioutil.ReadAll(response.Reader)
+	if err != nil {
+		return err
+	}
+
+	if !bytes.Contains(data, []byte("AccessToken")) {
+		return fmt.Errorf("failed to create-account: %s", string(data))
+	}
 	return nil
 }
 

docs/CONTRIBUTING.md

@@ -9,6 +9,28 @@ permalink: /development/contributing
 Everyone is welcome to contribute to Dendrite! We aim to make it as easy as
 possible to get started.
 
+ ## Contribution types
+
+We are a small team maintaining a large project. As a result, we cannot merge every feature, even if it
+is bug-free and useful, because we then commit to maintaining it indefinitely. We will always accept:
+ - bug fixes
+ - security fixes (please responsibly disclose via security@matrix.org *before* creating pull requests)
+
+We will accept the following with caveats:
+ - documentation fixes, provided they do not add additional instructions which can end up going out-of-date,
+   e.g example configs, shell commands.
+ - performance fixes, provided they do not add significantly more maintenance burden.
+ - additional functionality on existing features, provided the functionality is small and maintainable.
+ - additional functionality that, in its absence, would impact the ecosystem e.g spam and abuse mitigations
+ - test-only changes, provided they help improve coverage or test tricky code.
+
+The following items are at risk of not being accepted:
+ - Configuration or CLI changes, particularly ones which increase the overall configuration surface.
+
+The following items are unlikely to be accepted into a main Dendrite release for now:
+ - New MSC implementations.
+ - New features which are not in the specification.
+
 ## Sign off
 
 We require that everyone who contributes to the project signs off their contributions
@@ -75,7 +97,20 @@ comment. Please avoid doing this if you can.
 We also have unit tests which we run via:
 
 ```bash
-go test --race ./...
+DENDRITE_TEST_SKIP_NODB=1 go test --race ./...
+```
+
+This only runs SQLite database tests. If you wish to execute Postgres tests as well, you'll either need to 
+have Postgres installed locally (`createdb` will be used) or have a remote/containerized Postgres instance
+available.
+
+To configure the connection to a remote Postgres, you can use the following enviroment variables:
+
+```bash
+POSTGRES_USER=postgres
+POSTGERS_PASSWORD=yourPostgresPassword
+POSTGRES_HOST=localhost
+POSTGRES_DB=postgres # the superuser database to use
 ```
 
 In general, we like submissions that come with tests. Anything that proves that the

federationapi/consumers/roomserver.go

@@ -232,7 +232,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 }
 
 func (s *OutputRoomEventConsumer) sendPresence(roomID string, addedJoined []types.JoinedHost) {
-	joined := make([]gomatrixserverlib.ServerName, len(addedJoined))
+	joined := make([]gomatrixserverlib.ServerName, 0, len(addedJoined))
 	for _, added := range addedJoined {
 		joined = append(joined, added.ServerName)
 	}

federationapi/federationapi.go

@@ -43,8 +43,8 @@ import (
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
 // on the given input API.
-func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI) {
+func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI, enableMetrics bool) {
-	inthttp.AddRoutes(intAPI, router)
+	inthttp.AddRoutes(intAPI, router, enableMetrics)
 }
 
 // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.

federationapi/inthttp/server.go

@@ -17,41 +17,41 @@ import (
 
 // AddRoutes adds the FederationInternalAPI handlers to the http.ServeMux.
 // nolint:gocyclo
-func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
+func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
 	internalAPIMux.Handle(
 		FederationAPIQueryJoinedHostServerNamesInRoomPath,
-		httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", intAPI.QueryJoinedHostServerNamesInRoom),
+		httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", enableMetrics, intAPI.QueryJoinedHostServerNamesInRoom),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformInviteRequestPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", intAPI.PerformInvite),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", enableMetrics, intAPI.PerformInvite),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformLeaveRequestPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", intAPI.PerformLeave),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", enableMetrics, intAPI.PerformLeave),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformDirectoryLookupRequestPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", intAPI.PerformDirectoryLookup),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", enableMetrics, intAPI.PerformDirectoryLookup),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformBroadcastEDUPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", intAPI.PerformBroadcastEDU),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", enableMetrics, intAPI.PerformBroadcastEDU),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformWakeupServers,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformWakeupServers", intAPI.PerformWakeupServers),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformWakeupServers", enableMetrics, intAPI.PerformWakeupServers),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformJoinRequestPath,
 		httputil.MakeInternalRPCAPI(
-			"FederationAPIPerformJoinRequest",
+			"FederationAPIPerformJoinRequest", enableMetrics,
 			func(ctx context.Context, req *api.PerformJoinRequest, res *api.PerformJoinResponse) error {
 				intAPI.PerformJoin(ctx, req, res)
 				return nil
@@ -62,7 +62,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIGetUserDevicesPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIGetUserDevices",
+			"FederationAPIGetUserDevices", enableMetrics,
 			func(ctx context.Context, req *getUserDevices) (*gomatrixserverlib.RespUserDevices, error) {
 				res, err := intAPI.GetUserDevices(ctx, req.Origin, req.S, req.UserID)
 				return &res, federationClientError(err)
@@ -73,7 +73,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIClaimKeysPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIClaimKeys",
+			"FederationAPIClaimKeys", enableMetrics,
 			func(ctx context.Context, req *claimKeys) (*gomatrixserverlib.RespClaimKeys, error) {
 				res, err := intAPI.ClaimKeys(ctx, req.Origin, req.S, req.OneTimeKeys)
 				return &res, federationClientError(err)
@@ -84,7 +84,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIQueryKeysPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIQueryKeys",
+			"FederationAPIQueryKeys", enableMetrics,
 			func(ctx context.Context, req *queryKeys) (*gomatrixserverlib.RespQueryKeys, error) {
 				res, err := intAPI.QueryKeys(ctx, req.Origin, req.S, req.Keys)
 				return &res, federationClientError(err)
@@ -95,7 +95,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIBackfillPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIBackfill",
+			"FederationAPIBackfill", enableMetrics,
 			func(ctx context.Context, req *backfill) (*gomatrixserverlib.Transaction, error) {
 				res, err := intAPI.Backfill(ctx, req.Origin, req.S, req.RoomID, req.Limit, req.EventIDs)
 				return &res, federationClientError(err)
@@ -106,7 +106,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPILookupStatePath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupState",
+			"FederationAPILookupState", enableMetrics,
 			func(ctx context.Context, req *lookupState) (*gomatrixserverlib.RespState, error) {
 				res, err := intAPI.LookupState(ctx, req.Origin, req.S, req.RoomID, req.EventID, req.RoomVersion)
 				return &res, federationClientError(err)
@@ -117,7 +117,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPILookupStateIDsPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupStateIDs",
+			"FederationAPILookupStateIDs", enableMetrics,
 			func(ctx context.Context, req *lookupStateIDs) (*gomatrixserverlib.RespStateIDs, error) {
 				res, err := intAPI.LookupStateIDs(ctx, req.Origin, req.S, req.RoomID, req.EventID)
 				return &res, federationClientError(err)
@@ -128,7 +128,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPILookupMissingEventsPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupMissingEvents",
+			"FederationAPILookupMissingEvents", enableMetrics,
 			func(ctx context.Context, req *lookupMissingEvents) (*gomatrixserverlib.RespMissingEvents, error) {
 				res, err := intAPI.LookupMissingEvents(ctx, req.Origin, req.S, req.RoomID, req.Missing, req.RoomVersion)
 				return &res, federationClientError(err)
@@ -139,7 +139,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIGetEventPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIGetEvent",
+			"FederationAPIGetEvent", enableMetrics,
 			func(ctx context.Context, req *getEvent) (*gomatrixserverlib.Transaction, error) {
 				res, err := intAPI.GetEvent(ctx, req.Origin, req.S, req.EventID)
 				return &res, federationClientError(err)
@@ -150,7 +150,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIGetEventAuthPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIGetEventAuth",
+			"FederationAPIGetEventAuth", enableMetrics,
 			func(ctx context.Context, req *getEventAuth) (*gomatrixserverlib.RespEventAuth, error) {
 				res, err := intAPI.GetEventAuth(ctx, req.Origin, req.S, req.RoomVersion, req.RoomID, req.EventID)
 				return &res, federationClientError(err)
@@ -160,13 +160,13 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 
 	internalAPIMux.Handle(
 		FederationAPIQueryServerKeysPath,
-		httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", intAPI.QueryServerKeys),
+		httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", enableMetrics, intAPI.QueryServerKeys),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPILookupServerKeysPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupServerKeys",
+			"FederationAPILookupServerKeys", enableMetrics,
 			func(ctx context.Context, req *lookupServerKeys) (*[]gomatrixserverlib.ServerKeys, error) {
 				res, err := intAPI.LookupServerKeys(ctx, req.S, req.KeyRequests)
 				return &res, federationClientError(err)
@@ -177,7 +177,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIEventRelationshipsPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIMSC2836EventRelationships",
+			"FederationAPIMSC2836EventRelationships", enableMetrics,
 			func(ctx context.Context, req *eventRelationships) (*gomatrixserverlib.MSC2836EventRelationshipsResponse, error) {
 				res, err := intAPI.MSC2836EventRelationships(ctx, req.Origin, req.S, req.Req, req.RoomVer)
 				return &res, federationClientError(err)
@@ -188,7 +188,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPISpacesSummaryPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIMSC2946SpacesSummary",
+			"FederationAPIMSC2946SpacesSummary", enableMetrics,
 			func(ctx context.Context, req *spacesReq) (*gomatrixserverlib.MSC2946SpacesResponse, error) {
 				res, err := intAPI.MSC2946Spaces(ctx, req.Origin, req.S, req.RoomID, req.SuggestedOnly)
 				return &res, federationClientError(err)
@@ -198,7 +198,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 
 	// TODO: Look at this shape
 	internalAPIMux.Handle(FederationAPIQueryPublicKeyPath,
-		httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse {
 			request := api.QueryPublicKeysRequest{}
 			response := api.QueryPublicKeysResponse{}
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -215,7 +215,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 
 	// TODO: Look at this shape
 	internalAPIMux.Handle(FederationAPIInputPublicKeyPath,
-		httputil.MakeInternalAPI("FederationAPIInputPublicKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("FederationAPIInputPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse {
 			request := api.InputPublicKeysRequest{}
 			response := api.InputPublicKeysResponse{}
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {

internal/httputil/httpapi.go

@@ -24,16 +24,17 @@ import (
 	"strings"
 
 	"github.com/getsentry/sentry-go"
-	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
-	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 // BasicAuth is used for authorization on /metrics handlers
@@ -227,7 +228,7 @@ func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request)
 // This is used for APIs that are internal to dendrite.
 // If we are passed a tracing context in the request headers then we use that
 // as the parent of any tracing spans we create.
-func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse) http.Handler {
+func MakeInternalAPI(metricsName string, enableMetrics bool, f func(*http.Request) util.JSONResponse) http.Handler {
 	h := util.MakeJSONAPI(util.NewJSONRequestHandler(f))
 	withSpan := func(w http.ResponseWriter, req *http.Request) {
 		carrier := opentracing.HTTPHeadersCarrier(req.Header)
@@ -246,6 +247,10 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse
 		h.ServeHTTP(w, req)
 	}
 
+	if !enableMetrics {
+		return http.HandlerFunc(withSpan)
+	}
+
 	return promhttp.InstrumentHandlerCounter(
 		promauto.NewCounterVec(
 			prometheus.CounterOpts{

internal/httputil/internalapi.go

@@ -22,7 +22,7 @@ import (
 	"reflect"
 
 	"github.com/matrix-org/util"
-	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go"
 )
 
 type InternalAPIError struct {
@@ -34,8 +34,8 @@ func (e InternalAPIError) Error() string {
 	return fmt.Sprintf("internal API returned %q error: %s", e.Type, e.Message)
 }
 
-func MakeInternalRPCAPI[reqtype, restype any](metricsName string, f func(context.Context, *reqtype, *restype) error) http.Handler {
+func MakeInternalRPCAPI[reqtype, restype any](metricsName string, enableMetrics bool, f func(context.Context, *reqtype, *restype) error) http.Handler {
-	return MakeInternalAPI(metricsName, func(req *http.Request) util.JSONResponse {
+	return MakeInternalAPI(metricsName, enableMetrics, func(req *http.Request) util.JSONResponse {
 		var request reqtype
 		var response restype
 		if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -57,8 +57,8 @@ func MakeInternalRPCAPI[reqtype, restype any](metricsName string, f func(context
 	})
 }
 
-func MakeInternalProxyAPI[reqtype, restype any](metricsName string, f func(context.Context, *reqtype) (*restype, error)) http.Handler {
+func MakeInternalProxyAPI[reqtype, restype any](metricsName string, enableMetrics bool, f func(context.Context, *reqtype) (*restype, error)) http.Handler {
-	return MakeInternalAPI(metricsName, func(req *http.Request) util.JSONResponse {
+	return MakeInternalAPI(metricsName, enableMetrics, func(req *http.Request) util.JSONResponse {
 		var request reqtype
 		if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
 			return util.MessageResponse(http.StatusBadRequest, err.Error())

internal/pushgateway/client.go

@@ -9,6 +9,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal"
+
 	"github.com/opentracing/opentracing-go"
 )
 
@@ -50,8 +52,7 @@ func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest,
 		return err
 	}
 
-	//nolint:errcheck
+	defer internal.CloseAndLogIfError(ctx, hresp.Body, "failed to close response body")
-	defer hresp.Body.Close()
 
 	if hresp.StatusCode == http.StatusOK {
 		return json.NewDecoder(hresp.Body).Decode(resp)

internal/pushgateway/client_test.go

@@ -0,0 +1,54 @@
+package pushgateway
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+)
+
+func TestNotify(t *testing.T) {
+	wantResponse := NotifyResponse{
+		Rejected: []string{"testing"},
+	}
+
+	var i = 0
+
+	svr := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// /notify only accepts POST requests
+		if r.Method != http.MethodPost {
+			w.WriteHeader(http.StatusNotImplemented)
+			return
+		}
+
+		if i != 0 { // error path
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		// happy path
+		json.NewEncoder(w).Encode(wantResponse)
+	}))
+	defer svr.Close()
+
+	cl := NewHTTPClient(true)
+	gotResponse := NotifyResponse{}
+
+	// Test happy path
+	err := cl.Notify(context.Background(), svr.URL, &NotifyRequest{}, &gotResponse)
+	if err != nil {
+		t.Errorf("failed to notify client")
+	}
+	if !reflect.DeepEqual(gotResponse, wantResponse) {
+		t.Errorf("expected response %+v, got %+v", wantResponse, gotResponse)
+	}
+
+	// Test error path
+	i++
+	err = cl.Notify(context.Background(), svr.URL, &NotifyRequest{}, &gotResponse)
+	if err == nil {
+		t.Errorf("expected notifying the pushgateway to fail, but it succeeded")
+	}
+}

internal/pushrules/evaluate.go

@@ -145,6 +145,11 @@ func conditionMatches(cond *Condition, event *gomatrixserverlib.Event, ec Evalua
 }
 
 func patternMatches(key, pattern string, event *gomatrixserverlib.Event) (bool, error) {
+	// It doesn't make sense for an empty pattern to match anything.
+	if pattern == "" {
+		return false, nil
+	}
+
 	re, err := globToRegexp(pattern)
 	if err != nil {
 		return false, err
@@ -154,12 +159,20 @@ func patternMatches(key, pattern string, event *gomatrixserverlib.Event) (bool,
 	if err = json.Unmarshal(event.JSON(), &eventMap); err != nil {
 		return false, fmt.Errorf("parsing event: %w", err)
 	}
+	// From the spec:
+	// "If the property specified by key is completely absent from
+	// the event, or does not have a string value, then the condition
+	// will not match, even if pattern is *."
 	v, err := lookupMapPath(strings.Split(key, "."), eventMap)
 	if err != nil {
 		// An unknown path is a benign error that shouldn't stop rule
 		// processing. It's just a non-match.
 		return false, nil
 	}
+	if _, ok := v.(string); !ok {
+		// A non-string never matches.
+		return false, nil
+	}
 
 	return re.MatchString(fmt.Sprint(v)), nil
 }

internal/pushrules/evaluate_test.go

@@ -111,7 +111,10 @@ func TestConditionMatches(t *testing.T) {
 		{"empty", Condition{}, `{}`, false},
 		{"empty", Condition{Kind: "unknownstring"}, `{}`, false},
 
-		{"eventMatch", Condition{Kind: EventMatchCondition, Key: "content"}, `{"content":{}}`, true},
+		// Neither of these should match because `content` is not a full string match,
+		// and `content.body` is not a string value.
+		{"eventMatch", Condition{Kind: EventMatchCondition, Key: "content"}, `{"content":{}}`, false},
+		{"eventBodyMatch", Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3"}, `{"content":{"body": 3}}`, false},
 
 		{"displayNameNoMatch", Condition{Kind: ContainsDisplayNameCondition}, `{"content":{"body":"something without displayname"}}`, false},
 		{"displayNameMatch", Condition{Kind: ContainsDisplayNameCondition}, `{"content":{"body":"hello Dear User, how are you?"}}`, true},
@@ -137,7 +140,7 @@ func TestConditionMatches(t *testing.T) {
 				t.Fatalf("conditionMatches failed: %v", err)
 			}
 			if got != tst.Want {
-				t.Errorf("conditionMatches: got %v, want %v", got, tst.Want)
+				t.Errorf("conditionMatches: got %v, want %v on %s", got, tst.Want, tst.Name)
 			}
 		})
 	}
@@ -161,9 +164,7 @@ func TestPatternMatches(t *testing.T) {
 	}{
 		{"empty", "", "", `{}`, false},
 
-		// Note that an empty pattern contains no wildcard characters,
+		{"patternEmpty", "content", "", `{"content":{}}`, false},
-		// which implicitly means "*".
-		{"patternEmpty", "content", "", `{"content":{}}`, true},
 
 		{"literal", "content.creator", "acreator", `{"content":{"creator":"acreator"}}`, true},
 		{"substring", "content.creator", "reat", `{"content":{"creator":"acreator"}}`, true},
@@ -178,7 +179,7 @@ func TestPatternMatches(t *testing.T) {
 				t.Fatalf("patternMatches failed: %v", err)
 			}
 			if got != tst.Want {
-				t.Errorf("patternMatches: got %v, want %v", got, tst.Want)
+				t.Errorf("patternMatches: got %v, want %v on %s", got, tst.Want, tst.Name)
 			}
 		})
 	}

internal/pushrules/util.go

@@ -11,22 +11,27 @@ import (
 // kind and a tweaks map. Returns a nil map if it would have been
 // empty.
 func ActionsToTweaks(as []*Action) (ActionKind, map[string]interface{}, error) {
-	kind := UnknownAction
+	var kind ActionKind
-	tweaks := map[string]interface{}{}
+	var tweaks map[string]interface{}
 
 	for _, a := range as {
-		if a.Kind == SetTweakAction {
+		switch a.Kind {
-			tweaks[string(a.Tweak)] = a.Value
+		case DontNotifyAction:
-			continue
+			// Don't bother processing any further
-		}
+			return DontNotifyAction, nil, nil
-		if kind != UnknownAction {
-			return UnknownAction, nil, fmt.Errorf("got multiple primary actions: already had %q, got %s", kind, a.Kind)
-		}
-		kind = a.Kind
-	}
 
-	if len(tweaks) == 0 {
+		case SetTweakAction:
-		tweaks = nil
+			if tweaks == nil {
+				tweaks = map[string]interface{}{}
+			}
+			tweaks[string(a.Tweak)] = a.Value
+
+		default:
+			if kind != UnknownAction {
+				return UnknownAction, nil, fmt.Errorf("got multiple primary actions: already had %q, got %s", kind, a.Kind)
+			}
+			kind = a.Kind
+		}
 	}
 
 	return kind, tweaks, nil

internal/pushrules/util_test.go

@@ -17,6 +17,7 @@ func TestActionsToTweaks(t *testing.T) {
 		{"empty", nil, UnknownAction, nil},
 		{"zero", []*Action{{}}, UnknownAction, nil},
 		{"onlyPrimary", []*Action{{Kind: NotifyAction}}, NotifyAction, nil},
+		{"onlyPrimaryDontNotify", []*Action{{Kind: DontNotifyAction}}, DontNotifyAction, nil},
 		{"onlyTweak", []*Action{{Kind: SetTweakAction, Tweak: HighlightTweak}}, UnknownAction, map[string]interface{}{"highlight": nil}},
 		{"onlyTweakWithValue", []*Action{{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, UnknownAction, map[string]interface{}{"sound": "default"}},
 		{

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 10
-	VersionPatch = 7
+	VersionPatch = 8
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/inthttp/server.go

@@ -21,59 +21,59 @@ import (
 	"github.com/matrix-org/dendrite/keyserver/api"
 )
 
-func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) {
+func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI, enableMetrics bool) {
 	internalAPIMux.Handle(
 		PerformClaimKeysPath,
-		httputil.MakeInternalRPCAPI("KeyserverPerformClaimKeys", s.PerformClaimKeys),
+		httputil.MakeInternalRPCAPI("KeyserverPerformClaimKeys", enableMetrics, s.PerformClaimKeys),
 	)
 
 	internalAPIMux.Handle(
 		PerformDeleteKeysPath,
-		httputil.MakeInternalRPCAPI("KeyserverPerformDeleteKeys", s.PerformDeleteKeys),
+		httputil.MakeInternalRPCAPI("KeyserverPerformDeleteKeys", enableMetrics, s.PerformDeleteKeys),
 	)
 
 	internalAPIMux.Handle(
 		PerformUploadKeysPath,
-		httputil.MakeInternalRPCAPI("KeyserverPerformUploadKeys", s.PerformUploadKeys),
+		httputil.MakeInternalRPCAPI("KeyserverPerformUploadKeys", enableMetrics, s.PerformUploadKeys),
 	)
 
 	internalAPIMux.Handle(
 		PerformUploadDeviceKeysPath,
-		httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceKeys", s.PerformUploadDeviceKeys),
+		httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceKeys", enableMetrics, s.PerformUploadDeviceKeys),
 	)
 
 	internalAPIMux.Handle(
 		PerformUploadDeviceSignaturesPath,
-		httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceSignatures", s.PerformUploadDeviceSignatures),
+		httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceSignatures", enableMetrics, s.PerformUploadDeviceSignatures),
 	)
 
 	internalAPIMux.Handle(
 		QueryKeysPath,
-		httputil.MakeInternalRPCAPI("KeyserverQueryKeys", s.QueryKeys),
+		httputil.MakeInternalRPCAPI("KeyserverQueryKeys", enableMetrics, s.QueryKeys),
 	)
 
 	internalAPIMux.Handle(
 		QueryOneTimeKeysPath,
-		httputil.MakeInternalRPCAPI("KeyserverQueryOneTimeKeys", s.QueryOneTimeKeys),
+		httputil.MakeInternalRPCAPI("KeyserverQueryOneTimeKeys", enableMetrics, s.QueryOneTimeKeys),
 	)
 
 	internalAPIMux.Handle(
 		QueryDeviceMessagesPath,
-		httputil.MakeInternalRPCAPI("KeyserverQueryDeviceMessages", s.QueryDeviceMessages),
+		httputil.MakeInternalRPCAPI("KeyserverQueryDeviceMessages", enableMetrics, s.QueryDeviceMessages),
 	)
 
 	internalAPIMux.Handle(
 		QueryKeyChangesPath,
-		httputil.MakeInternalRPCAPI("KeyserverQueryKeyChanges", s.QueryKeyChanges),
+		httputil.MakeInternalRPCAPI("KeyserverQueryKeyChanges", enableMetrics, s.QueryKeyChanges),
 	)
 
 	internalAPIMux.Handle(
 		QuerySignaturesPath,
-		httputil.MakeInternalRPCAPI("KeyserverQuerySignatures", s.QuerySignatures),
+		httputil.MakeInternalRPCAPI("KeyserverQuerySignatures", enableMetrics, s.QuerySignatures),
 	)
 
 	internalAPIMux.Handle(
 		PerformMarkAsStalePath,
-		httputil.MakeInternalRPCAPI("KeyserverMarkAsStale", s.PerformMarkAsStaleIfNeeded),
+		httputil.MakeInternalRPCAPI("KeyserverMarkAsStale", enableMetrics, s.PerformMarkAsStaleIfNeeded),
 	)
 }

keyserver/keyserver.go

@@ -32,8 +32,8 @@ import (
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
 // on the given input API.
-func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) {
+func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI, enableMetrics bool) {
-	inthttp.AddRoutes(router, intAPI)
+	inthttp.AddRoutes(router, intAPI, enableMetrics)
 }
 
 // NewInternalAPI returns a concerete implementation of the internal API. Callers

roomserver/inthttp/server.go

@@ -9,198 +9,198 @@ import (
 
 // AddRoutes adds the RoomserverInternalAPI handlers to the http.ServeMux.
 // nolint: gocyclo
-func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
+func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
 	internalAPIMux.Handle(
 		RoomserverInputRoomEventsPath,
-		httputil.MakeInternalRPCAPI("RoomserverInputRoomEvents", r.InputRoomEvents),
+		httputil.MakeInternalRPCAPI("RoomserverInputRoomEvents", enableMetrics, r.InputRoomEvents),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformInvitePath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformInvite", r.PerformInvite),
+		httputil.MakeInternalRPCAPI("RoomserverPerformInvite", enableMetrics, r.PerformInvite),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformJoinPath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformJoin", r.PerformJoin),
+		httputil.MakeInternalRPCAPI("RoomserverPerformJoin", enableMetrics, r.PerformJoin),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformLeavePath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformLeave", r.PerformLeave),
+		httputil.MakeInternalRPCAPI("RoomserverPerformLeave", enableMetrics, r.PerformLeave),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformPeekPath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformPeek", r.PerformPeek),
+		httputil.MakeInternalRPCAPI("RoomserverPerformPeek", enableMetrics, r.PerformPeek),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformInboundPeekPath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformInboundPeek", r.PerformInboundPeek),
+		httputil.MakeInternalRPCAPI("RoomserverPerformInboundPeek", enableMetrics, r.PerformInboundPeek),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformUnpeekPath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformUnpeek", r.PerformUnpeek),
+		httputil.MakeInternalRPCAPI("RoomserverPerformUnpeek", enableMetrics, r.PerformUnpeek),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformRoomUpgradePath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformRoomUpgrade", r.PerformRoomUpgrade),
+		httputil.MakeInternalRPCAPI("RoomserverPerformRoomUpgrade", enableMetrics, r.PerformRoomUpgrade),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformPublishPath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformPublish", r.PerformPublish),
+		httputil.MakeInternalRPCAPI("RoomserverPerformPublish", enableMetrics, r.PerformPublish),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformAdminEvacuateRoomPath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateRoom", r.PerformAdminEvacuateRoom),
+		httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateRoom", enableMetrics, r.PerformAdminEvacuateRoom),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformAdminEvacuateUserPath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateUser", r.PerformAdminEvacuateUser),
+		httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateUser", enableMetrics, r.PerformAdminEvacuateUser),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformAdminDownloadStatePath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformAdminDownloadState", r.PerformAdminDownloadState),
+		httputil.MakeInternalRPCAPI("RoomserverPerformAdminDownloadState", enableMetrics, r.PerformAdminDownloadState),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryPublishedRoomsPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryPublishedRooms", r.QueryPublishedRooms),
+		httputil.MakeInternalRPCAPI("RoomserverQueryPublishedRooms", enableMetrics, r.QueryPublishedRooms),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryLatestEventsAndStatePath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryLatestEventsAndState", r.QueryLatestEventsAndState),
+		httputil.MakeInternalRPCAPI("RoomserverQueryLatestEventsAndState", enableMetrics, r.QueryLatestEventsAndState),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryStateAfterEventsPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryStateAfterEvents", r.QueryStateAfterEvents),
+		httputil.MakeInternalRPCAPI("RoomserverQueryStateAfterEvents", enableMetrics, r.QueryStateAfterEvents),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryEventsByIDPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryEventsByID", r.QueryEventsByID),
+		httputil.MakeInternalRPCAPI("RoomserverQueryEventsByID", enableMetrics, r.QueryEventsByID),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryMembershipForUserPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryMembershipForUser", r.QueryMembershipForUser),
+		httputil.MakeInternalRPCAPI("RoomserverQueryMembershipForUser", enableMetrics, r.QueryMembershipForUser),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryMembershipsForRoomPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryMembershipsForRoom", r.QueryMembershipsForRoom),
+		httputil.MakeInternalRPCAPI("RoomserverQueryMembershipsForRoom", enableMetrics, r.QueryMembershipsForRoom),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryServerJoinedToRoomPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryServerJoinedToRoom", r.QueryServerJoinedToRoom),
+		httputil.MakeInternalRPCAPI("RoomserverQueryServerJoinedToRoom", enableMetrics, r.QueryServerJoinedToRoom),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryServerAllowedToSeeEventPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryServerAllowedToSeeEvent", r.QueryServerAllowedToSeeEvent),
+		httputil.MakeInternalRPCAPI("RoomserverQueryServerAllowedToSeeEvent", enableMetrics, r.QueryServerAllowedToSeeEvent),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryMissingEventsPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryMissingEvents", r.QueryMissingEvents),
+		httputil.MakeInternalRPCAPI("RoomserverQueryMissingEvents", enableMetrics, r.QueryMissingEvents),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryStateAndAuthChainPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryStateAndAuthChain", r.QueryStateAndAuthChain),
+		httputil.MakeInternalRPCAPI("RoomserverQueryStateAndAuthChain", enableMetrics, r.QueryStateAndAuthChain),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformBackfillPath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformBackfill", r.PerformBackfill),
+		httputil.MakeInternalRPCAPI("RoomserverPerformBackfill", enableMetrics, r.PerformBackfill),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverPerformForgetPath,
-		httputil.MakeInternalRPCAPI("RoomserverPerformForget", r.PerformForget),
+		httputil.MakeInternalRPCAPI("RoomserverPerformForget", enableMetrics, r.PerformForget),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryRoomVersionCapabilitiesPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionCapabilities", r.QueryRoomVersionCapabilities),
+		httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionCapabilities", enableMetrics, r.QueryRoomVersionCapabilities),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryRoomVersionForRoomPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionForRoom", r.QueryRoomVersionForRoom),
+		httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionForRoom", enableMetrics, r.QueryRoomVersionForRoom),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverSetRoomAliasPath,
-		httputil.MakeInternalRPCAPI("RoomserverSetRoomAlias", r.SetRoomAlias),
+		httputil.MakeInternalRPCAPI("RoomserverSetRoomAlias", enableMetrics, r.SetRoomAlias),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverGetRoomIDForAliasPath,
-		httputil.MakeInternalRPCAPI("RoomserverGetRoomIDForAlias", r.GetRoomIDForAlias),
+		httputil.MakeInternalRPCAPI("RoomserverGetRoomIDForAlias", enableMetrics, r.GetRoomIDForAlias),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverGetAliasesForRoomIDPath,
-		httputil.MakeInternalRPCAPI("RoomserverGetAliasesForRoomID", r.GetAliasesForRoomID),
+		httputil.MakeInternalRPCAPI("RoomserverGetAliasesForRoomID", enableMetrics, r.GetAliasesForRoomID),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverRemoveRoomAliasPath,
-		httputil.MakeInternalRPCAPI("RoomserverRemoveRoomAlias", r.RemoveRoomAlias),
+		httputil.MakeInternalRPCAPI("RoomserverRemoveRoomAlias", enableMetrics, r.RemoveRoomAlias),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryCurrentStatePath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryCurrentState", r.QueryCurrentState),
+		httputil.MakeInternalRPCAPI("RoomserverQueryCurrentState", enableMetrics, r.QueryCurrentState),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryRoomsForUserPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryRoomsForUser", r.QueryRoomsForUser),
+		httputil.MakeInternalRPCAPI("RoomserverQueryRoomsForUser", enableMetrics, r.QueryRoomsForUser),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryBulkStateContentPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryBulkStateContent", r.QueryBulkStateContent),
+		httputil.MakeInternalRPCAPI("RoomserverQueryBulkStateContent", enableMetrics, r.QueryBulkStateContent),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQuerySharedUsersPath,
-		httputil.MakeInternalRPCAPI("RoomserverQuerySharedUsers", r.QuerySharedUsers),
+		httputil.MakeInternalRPCAPI("RoomserverQuerySharedUsers", enableMetrics, r.QuerySharedUsers),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryKnownUsersPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryKnownUsers", r.QueryKnownUsers),
+		httputil.MakeInternalRPCAPI("RoomserverQueryKnownUsers", enableMetrics, r.QueryKnownUsers),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryServerBannedFromRoomPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryServerBannedFromRoom", r.QueryServerBannedFromRoom),
+		httputil.MakeInternalRPCAPI("RoomserverQueryServerBannedFromRoom", enableMetrics, r.QueryServerBannedFromRoom),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryAuthChainPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryAuthChain", r.QueryAuthChain),
+		httputil.MakeInternalRPCAPI("RoomserverQueryAuthChain", enableMetrics, r.QueryAuthChain),
 	)
 
 	internalAPIMux.Handle(
 		RoomserverQueryRestrictedJoinAllowed,
-		httputil.MakeInternalRPCAPI("RoomserverQueryRestrictedJoinAllowed", r.QueryRestrictedJoinAllowed),
+		httputil.MakeInternalRPCAPI("RoomserverQueryRestrictedJoinAllowed", enableMetrics, r.QueryRestrictedJoinAllowed),
 	)
 	internalAPIMux.Handle(
 		RoomserverQueryMembershipAtEventPath,
-		httputil.MakeInternalRPCAPI("RoomserverQueryMembershipAtEventPath", r.QueryMembershipAtEvent),
+		httputil.MakeInternalRPCAPI("RoomserverQueryMembershipAtEventPath", enableMetrics, r.QueryMembershipAtEvent),
 	)
 }

roomserver/roomserver.go

@@ -16,18 +16,19 @@ package roomserver
 
 import (
 	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/internal"
 	"github.com/matrix-org/dendrite/roomserver/inthttp"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/sirupsen/logrus"
 )
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
 // on the given input API.
-func AddInternalRoutes(router *mux.Router, intAPI api.RoomserverInternalAPI) {
+func AddInternalRoutes(router *mux.Router, intAPI api.RoomserverInternalAPI, enableMetrics bool) {
-	inthttp.AddRoutes(intAPI, router)
+	inthttp.AddRoutes(intAPI, router, enableMetrics)
 }
 
 // NewInternalAPI returns a concerete implementation of the internal API. Callers

setup/base/base.go

@@ -413,6 +413,24 @@ func (b *BaseDendrite) configureHTTPErrors() {
 	b.PublicClientAPIMux.MethodNotAllowedHandler = http.HandlerFunc(clientNotFoundHandler)
 }
 
+func (b *BaseDendrite) ConfigureAdminEndpoints() {
+	b.DendriteAdminMux.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+	})
+	b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) {
+		if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded {
+			w.WriteHeader(503)
+			_ = json.NewEncoder(w).Encode(struct {
+				Warnings []string `json:"warnings"`
+			}{
+				Warnings: reasons,
+			})
+			return
+		}
+		w.WriteHeader(200)
+	})
+}
+
 // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on
 // ApiMux under /api/ and adds a prometheus handler under /metrics.
 func (b *BaseDendrite) SetupAndServeHTTP(
@@ -463,21 +481,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		internalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth))
 	}
 
-	b.DendriteAdminMux.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) {
+	b.ConfigureAdminEndpoints()
-		w.WriteHeader(200)
-	})
-	b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) {
-		if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded {
-			w.WriteHeader(503)
-			_ = json.NewEncoder(w).Encode(struct {
-				Warnings []string `json:"warnings"`
-			}{
-				Warnings: reasons,
-			})
-			return
-		}
-		w.WriteHeader(200)
-	})
 
 	var clientHandler http.Handler
 	clientHandler = b.PublicClientAPIMux

setup/config/config_global.go

@@ -17,7 +17,7 @@ type Global struct {
 	gomatrixserverlib.SigningIdentity `yaml:",inline"`
 
 	// The secondary server names, used for virtual hosting.
-	VirtualHosts []*VirtualHost `yaml:"virtual_hosts"`
+	VirtualHosts []*VirtualHost `yaml:"-"`
 
 	// Path to the private key which will be used to sign requests and events.
 	PrivateKeyPath Path `yaml:"private_key"`

syncapi/storage/shared/storage_consumer.go

@@ -57,31 +57,23 @@ type Database struct {
 }
 
 func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) {
-	return d.NewDatabaseTransaction(ctx)
+	txn, err := d.DB.BeginTx(ctx, &sql.TxOptions{
-
+		// Set the isolation level so that we see a snapshot of the database.
-	/*
+		// In PostgreSQL repeatable read transactions will see a snapshot taken
-		TODO: Repeatable read is probably the right thing to do here,
+		// at the first query, and since the transaction is read-only it can't
-		but it seems to cause some problems with the invite tests, so
+		// run into any serialisation errors.
-		need to investigate that further.
+		// https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ
-
+		Isolation: sql.LevelRepeatableRead,
-		txn, err := d.DB.BeginTx(ctx, &sql.TxOptions{
+		ReadOnly:  true,
-			// Set the isolation level so that we see a snapshot of the database.
+	})
-			// In PostgreSQL repeatable read transactions will see a snapshot taken
+	if err != nil {
-			// at the first query, and since the transaction is read-only it can't
+		return nil, err
-			// run into any serialisation errors.
+	}
-			// https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ
+	return &DatabaseTransaction{
-			Isolation: sql.LevelRepeatableRead,
+		Database: d,
-			ReadOnly:  true,
+		ctx:      ctx,
-		})
+		txn:      txn,
-		if err != nil {
+	}, nil
-			return nil, err
-		}
-		return &DatabaseTransaction{
-			Database: d,
-			ctx:      ctx,
-			txn:      txn,
-		}, nil
-	*/
 }
 
 func (d *Database) NewDatabaseTransaction(ctx context.Context) (*DatabaseTransaction, error) {

syncapi/streams/stream_receipt.go

@@ -87,8 +87,7 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 		}
 
 		ev := gomatrixserverlib.ClientEvent{
-			Type:   gomatrixserverlib.MReceipt,
+			Type: gomatrixserverlib.MReceipt,
-			RoomID: roomID,
 		}
 		content := make(map[string]ReceiptMRead)
 		for _, receipt := range receipts {

syncapi/types/types.go

@@ -480,6 +480,13 @@ func (jr JoinResponse) MarshalJSON() ([]byte, error) {
 	if jr.Ephemeral != nil && len(jr.Ephemeral.Events) == 0 {
 		a.Ephemeral = nil
 	}
+	if jr.Ephemeral != nil {
+		// Remove the room_id from EDUs, as this seems to cause Element Web
+		// to trigger notifications - https://github.com/vector-im/element-web/issues/17263
+		for i := range jr.Ephemeral.Events {
+			jr.Ephemeral.Events[i].RoomID = ""
+		}
+	}
 	if jr.AccountData != nil && len(jr.AccountData.Events) == 0 {
 		a.AccountData = nil
 	}

syncapi/types/types_test.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"encoding/json"
+	"reflect"
 	"testing"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -63,3 +64,102 @@ func TestNewInviteResponse(t *testing.T) {
 		t.Fatalf("Invite response didn't contain correct info")
 	}
 }
+
+func TestJoinResponse_MarshalJSON(t *testing.T) {
+	type fields struct {
+		Summary             *Summary
+		State               *ClientEvents
+		Timeline            *Timeline
+		Ephemeral           *ClientEvents
+		AccountData         *ClientEvents
+		UnreadNotifications *UnreadNotifications
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		want    []byte
+		wantErr bool
+	}{
+		{
+			name: "empty state is removed",
+			fields: fields{
+				State: &ClientEvents{},
+			},
+			want: []byte("{}"),
+		},
+		{
+			name: "empty accountdata is removed",
+			fields: fields{
+				AccountData: &ClientEvents{},
+			},
+			want: []byte("{}"),
+		},
+		{
+			name: "empty ephemeral is removed",
+			fields: fields{
+				Ephemeral: &ClientEvents{},
+			},
+			want: []byte("{}"),
+		},
+		{
+			name: "empty timeline is removed",
+			fields: fields{
+				Timeline: &Timeline{},
+			},
+			want: []byte("{}"),
+		},
+		{
+			name: "empty summary is removed",
+			fields: fields{
+				Summary: &Summary{},
+			},
+			want: []byte("{}"),
+		},
+		{
+			name: "unread notifications are removed, if everything else is empty",
+			fields: fields{
+				UnreadNotifications: &UnreadNotifications{},
+			},
+			want: []byte("{}"),
+		},
+		{
+			name: "unread notifications are NOT removed, if state is set",
+			fields: fields{
+				State:               &ClientEvents{Events: []gomatrixserverlib.ClientEvent{{Content: []byte("{}")}}},
+				UnreadNotifications: &UnreadNotifications{NotificationCount: 1},
+			},
+			want: []byte(`{"state":{"events":[{"content":{},"type":""}]},"unread_notifications":{"highlight_count":0,"notification_count":1}}`),
+		},
+		{
+			name: "roomID is removed from EDUs",
+			fields: fields{
+				Ephemeral: &ClientEvents{
+					Events: []gomatrixserverlib.ClientEvent{
+						{RoomID: "!someRandomRoomID:test", Content: []byte("{}")},
+					},
+				},
+			},
+			want: []byte(`{"ephemeral":{"events":[{"content":{},"type":""}]}}`),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			jr := JoinResponse{
+				Summary:             tt.fields.Summary,
+				State:               tt.fields.State,
+				Timeline:            tt.fields.Timeline,
+				Ephemeral:           tt.fields.Ephemeral,
+				AccountData:         tt.fields.AccountData,
+				UnreadNotifications: tt.fields.UnreadNotifications,
+			}
+			got, err := jr.MarshalJSON()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("MarshalJSON() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("MarshalJSON() got = %v, want %v", string(got), string(tt.want))
+			}
+		})
+	}
+}

userapi/consumers/roomserver.go

@@ -385,7 +385,6 @@ func (s *OutputRoomEventConsumer) localRoomMembers(ctx context.Context, roomID s
 	req := &rsapi.QueryMembershipsForRoomRequest{
 		RoomID:     roomID,
 		JoinedOnly: true,
-		LocalOnly:  true,
 	}
 	var res rsapi.QueryMembershipsForRoomResponse
 
@@ -396,8 +395,23 @@ func (s *OutputRoomEventConsumer) localRoomMembers(ctx context.Context, roomID s
 	}
 
 	var members []*localMembership
-	var ntotal int
 	for _, event := range res.JoinEvents {
+		// Filter out invalid join events
+		if event.StateKey == nil {
+			continue
+		}
+		if *event.StateKey == "" {
+			continue
+		}
+		_, serverName, err := gomatrixserverlib.SplitID('@', *event.StateKey)
+		if err != nil {
+			log.WithError(err).Error("failed to get servername from statekey")
+			continue
+		}
+		// Only get memberships for our server
+		if serverName != s.serverName {
+			continue
+		}
 		member, err := newLocalMembership(&event)
 		if err != nil {
 			log.WithError(err).Errorf("Parsing MemberContent")
@@ -410,11 +424,10 @@ func (s *OutputRoomEventConsumer) localRoomMembers(ctx context.Context, roomID s
 			continue
 		}
 
-		ntotal++
 		members = append(members, member)
 	}
 
-	return members, ntotal, nil
+	return members, len(res.JoinEvents), nil
 }
 
 // roomName returns the name in the event (if type==m.room.name), or
@@ -641,7 +654,7 @@ func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event *
 	if rule == nil {
 		// SPEC: If no rules match an event, the homeserver MUST NOT
 		// notify the Push Gateway for that event.
-		return nil, err
+		return nil, nil
 	}
 
 	log.WithFields(log.Fields{

userapi/inthttp/server.go

@@ -16,177 +16,178 @@ package inthttp
 
 import (
 	"github.com/gorilla/mux"
+
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/userapi/api"
 )
 
 // nolint: gocyclo
-func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
+func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI, enableMetrics bool) {
-	addRoutesLoginToken(internalAPIMux, s)
+	addRoutesLoginToken(internalAPIMux, s, enableMetrics)
 
 	internalAPIMux.Handle(
 		PerformAccountCreationPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformAccountCreation", s.PerformAccountCreation),
+		httputil.MakeInternalRPCAPI("UserAPIPerformAccountCreation", enableMetrics, s.PerformAccountCreation),
 	)
 
 	internalAPIMux.Handle(
 		PerformPasswordUpdatePath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformPasswordUpdate", s.PerformPasswordUpdate),
+		httputil.MakeInternalRPCAPI("UserAPIPerformPasswordUpdate", enableMetrics, s.PerformPasswordUpdate),
 	)
 
 	internalAPIMux.Handle(
 		PerformDeviceCreationPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformDeviceCreation", s.PerformDeviceCreation),
+		httputil.MakeInternalRPCAPI("UserAPIPerformDeviceCreation", enableMetrics, s.PerformDeviceCreation),
 	)
 
 	internalAPIMux.Handle(
 		PerformLastSeenUpdatePath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformLastSeenUpdate", s.PerformLastSeenUpdate),
+		httputil.MakeInternalRPCAPI("UserAPIPerformLastSeenUpdate", enableMetrics, s.PerformLastSeenUpdate),
 	)
 
 	internalAPIMux.Handle(
 		PerformDeviceUpdatePath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformDeviceUpdate", s.PerformDeviceUpdate),
+		httputil.MakeInternalRPCAPI("UserAPIPerformDeviceUpdate", enableMetrics, s.PerformDeviceUpdate),
 	)
 
 	internalAPIMux.Handle(
 		PerformDeviceDeletionPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformDeviceDeletion", s.PerformDeviceDeletion),
+		httputil.MakeInternalRPCAPI("UserAPIPerformDeviceDeletion", enableMetrics, s.PerformDeviceDeletion),
 	)
 
 	internalAPIMux.Handle(
 		PerformAccountDeactivationPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformAccountDeactivation", s.PerformAccountDeactivation),
+		httputil.MakeInternalRPCAPI("UserAPIPerformAccountDeactivation", enableMetrics, s.PerformAccountDeactivation),
 	)
 
 	internalAPIMux.Handle(
 		PerformOpenIDTokenCreationPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformOpenIDTokenCreation", s.PerformOpenIDTokenCreation),
+		httputil.MakeInternalRPCAPI("UserAPIPerformOpenIDTokenCreation", enableMetrics, s.PerformOpenIDTokenCreation),
 	)
 
 	internalAPIMux.Handle(
 		QueryProfilePath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryProfile", s.QueryProfile),
+		httputil.MakeInternalRPCAPI("UserAPIQueryProfile", enableMetrics, s.QueryProfile),
 	)
 
 	internalAPIMux.Handle(
 		QueryAccessTokenPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryAccessToken", s.QueryAccessToken),
+		httputil.MakeInternalRPCAPI("UserAPIQueryAccessToken", enableMetrics, s.QueryAccessToken),
 	)
 
 	internalAPIMux.Handle(
 		QueryDevicesPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryDevices", s.QueryDevices),
+		httputil.MakeInternalRPCAPI("UserAPIQueryDevices", enableMetrics, s.QueryDevices),
 	)
 
 	internalAPIMux.Handle(
 		QueryAccountDataPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryAccountData", s.QueryAccountData),
+		httputil.MakeInternalRPCAPI("UserAPIQueryAccountData", enableMetrics, s.QueryAccountData),
 	)
 
 	internalAPIMux.Handle(
 		QueryDeviceInfosPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryDeviceInfos", s.QueryDeviceInfos),
+		httputil.MakeInternalRPCAPI("UserAPIQueryDeviceInfos", enableMetrics, s.QueryDeviceInfos),
 	)
 
 	internalAPIMux.Handle(
 		QuerySearchProfilesPath,
-		httputil.MakeInternalRPCAPI("UserAPIQuerySearchProfiles", s.QuerySearchProfiles),
+		httputil.MakeInternalRPCAPI("UserAPIQuerySearchProfiles", enableMetrics, s.QuerySearchProfiles),
 	)
 
 	internalAPIMux.Handle(
 		QueryOpenIDTokenPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryOpenIDToken", s.QueryOpenIDToken),
+		httputil.MakeInternalRPCAPI("UserAPIQueryOpenIDToken", enableMetrics, s.QueryOpenIDToken),
 	)
 
 	internalAPIMux.Handle(
 		InputAccountDataPath,
-		httputil.MakeInternalRPCAPI("UserAPIInputAccountData", s.InputAccountData),
+		httputil.MakeInternalRPCAPI("UserAPIInputAccountData", enableMetrics, s.InputAccountData),
 	)
 
 	internalAPIMux.Handle(
 		QueryKeyBackupPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryKeyBackup", s.QueryKeyBackup),
+		httputil.MakeInternalRPCAPI("UserAPIQueryKeyBackup", enableMetrics, s.QueryKeyBackup),
 	)
 
 	internalAPIMux.Handle(
 		PerformKeyBackupPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformKeyBackup", s.PerformKeyBackup),
+		httputil.MakeInternalRPCAPI("UserAPIPerformKeyBackup", enableMetrics, s.PerformKeyBackup),
 	)
 
 	internalAPIMux.Handle(
 		QueryNotificationsPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryNotifications", s.QueryNotifications),
+		httputil.MakeInternalRPCAPI("UserAPIQueryNotifications", enableMetrics, s.QueryNotifications),
 	)
 
 	internalAPIMux.Handle(
 		PerformPusherSetPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformPusherSet", s.PerformPusherSet),
+		httputil.MakeInternalRPCAPI("UserAPIPerformPusherSet", enableMetrics, s.PerformPusherSet),
 	)
 
 	internalAPIMux.Handle(
 		PerformPusherDeletionPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformPusherDeletion", s.PerformPusherDeletion),
+		httputil.MakeInternalRPCAPI("UserAPIPerformPusherDeletion", enableMetrics, s.PerformPusherDeletion),
 	)
 
 	internalAPIMux.Handle(
 		QueryPushersPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryPushers", s.QueryPushers),
+		httputil.MakeInternalRPCAPI("UserAPIQueryPushers", enableMetrics, s.QueryPushers),
 	)
 
 	internalAPIMux.Handle(
 		PerformPushRulesPutPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformPushRulesPut", s.PerformPushRulesPut),
+		httputil.MakeInternalRPCAPI("UserAPIPerformPushRulesPut", enableMetrics, s.PerformPushRulesPut),
 	)
 
 	internalAPIMux.Handle(
 		QueryPushRulesPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryPushRules", s.QueryPushRules),
+		httputil.MakeInternalRPCAPI("UserAPIQueryPushRules", enableMetrics, s.QueryPushRules),
 	)
 
 	internalAPIMux.Handle(
 		PerformSetAvatarURLPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformSetAvatarURL", s.SetAvatarURL),
+		httputil.MakeInternalRPCAPI("UserAPIPerformSetAvatarURL", enableMetrics, s.SetAvatarURL),
 	)
 
 	internalAPIMux.Handle(
 		QueryNumericLocalpartPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryNumericLocalpart", s.QueryNumericLocalpart),
+		httputil.MakeInternalRPCAPI("UserAPIQueryNumericLocalpart", enableMetrics, s.QueryNumericLocalpart),
 	)
 
 	internalAPIMux.Handle(
 		QueryAccountAvailabilityPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryAccountAvailability", s.QueryAccountAvailability),
+		httputil.MakeInternalRPCAPI("UserAPIQueryAccountAvailability", enableMetrics, s.QueryAccountAvailability),
 	)
 
 	internalAPIMux.Handle(
 		QueryAccountByPasswordPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryAccountByPassword", s.QueryAccountByPassword),
+		httputil.MakeInternalRPCAPI("UserAPIQueryAccountByPassword", enableMetrics, s.QueryAccountByPassword),
 	)
 
 	internalAPIMux.Handle(
 		PerformSetDisplayNamePath,
-		httputil.MakeInternalRPCAPI("UserAPISetDisplayName", s.SetDisplayName),
+		httputil.MakeInternalRPCAPI("UserAPISetDisplayName", enableMetrics, s.SetDisplayName),
 	)
 
 	internalAPIMux.Handle(
 		QueryLocalpartForThreePIDPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryLocalpartForThreePID", s.QueryLocalpartForThreePID),
+		httputil.MakeInternalRPCAPI("UserAPIQueryLocalpartForThreePID", enableMetrics, s.QueryLocalpartForThreePID),
 	)
 
 	internalAPIMux.Handle(
 		QueryThreePIDsForLocalpartPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryThreePIDsForLocalpart", s.QueryThreePIDsForLocalpart),
+		httputil.MakeInternalRPCAPI("UserAPIQueryThreePIDsForLocalpart", enableMetrics, s.QueryThreePIDsForLocalpart),
 	)
 
 	internalAPIMux.Handle(
 		PerformForgetThreePIDPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformForgetThreePID", s.PerformForgetThreePID),
+		httputil.MakeInternalRPCAPI("UserAPIPerformForgetThreePID", enableMetrics, s.PerformForgetThreePID),
 	)
 
 	internalAPIMux.Handle(
 		PerformSaveThreePIDAssociationPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformSaveThreePIDAssociation", s.PerformSaveThreePIDAssociation),
+		httputil.MakeInternalRPCAPI("UserAPIPerformSaveThreePIDAssociation", enableMetrics, s.PerformSaveThreePIDAssociation),
 	)
 
 	internalAPIMux.Handle(

userapi/inthttp/server_logintoken.go

@@ -16,24 +16,25 @@ package inthttp
 
 import (
 	"github.com/gorilla/mux"
+
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/userapi/api"
 )
 
 // addRoutesLoginToken adds routes for all login token API calls.
-func addRoutesLoginToken(internalAPIMux *mux.Router, s api.UserInternalAPI) {
+func addRoutesLoginToken(internalAPIMux *mux.Router, s api.UserInternalAPI, enableMetrics bool) {
 	internalAPIMux.Handle(
 		PerformLoginTokenCreationPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenCreation", s.PerformLoginTokenCreation),
+		httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenCreation", enableMetrics, s.PerformLoginTokenCreation),
 	)
 
 	internalAPIMux.Handle(
 		PerformLoginTokenDeletionPath,
-		httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenDeletion", s.PerformLoginTokenDeletion),
+		httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenDeletion", enableMetrics, s.PerformLoginTokenDeletion),
 	)
 
 	internalAPIMux.Handle(
 		QueryLoginTokenPath,
-		httputil.MakeInternalRPCAPI("UserAPIQueryLoginToken", s.QueryLoginToken),
+		httputil.MakeInternalRPCAPI("UserAPIQueryLoginToken", enableMetrics, s.QueryLoginToken),
 	)
 }

userapi/userapi.go

@@ -37,8 +37,8 @@ import (
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
 // on the given input API.
-func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI) {
+func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI, enableMetrics bool) {
-	inthttp.AddRoutes(router, intAPI)
+	inthttp.AddRoutes(router, intAPI, enableMetrics)
 }
 
 // NewInternalAPI returns a concerete implementation of the internal API. Callers

userapi/userapi_test.go

@@ -27,14 +27,13 @@ import (
 	"golang.org/x/crypto/bcrypt"
 
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/dendrite/userapi/inthttp"
-
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/internal"
+	"github.com/matrix-org/dendrite/userapi/inthttp"
 	"github.com/matrix-org/dendrite/userapi/storage"
 )
 
@@ -79,19 +78,6 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType) (ap
 func TestQueryProfile(t *testing.T) {
 	aliceAvatarURL := "mxc://example.com/alice"
 	aliceDisplayName := "Alice"
-	// only one DBType, since userapi.AddInternalRoutes complains about multiple prometheus counters added
-	userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, test.DBTypeSQLite)
-	defer close()
-	_, err := accountDB.CreateAccount(context.TODO(), "alice", serverName, "foobar", "", api.AccountTypeUser)
-	if err != nil {
-		t.Fatalf("failed to make account: %s", err)
-	}
-	if _, _, err := accountDB.SetAvatarURL(context.TODO(), "alice", serverName, aliceAvatarURL); err != nil {
-		t.Fatalf("failed to set avatar url: %s", err)
-	}
-	if _, _, err := accountDB.SetDisplayName(context.TODO(), "alice", serverName, aliceDisplayName); err != nil {
-		t.Fatalf("failed to set display name: %s", err)
-	}
 
 	testCases := []struct {
 		req     api.QueryProfileRequest
@@ -142,19 +128,34 @@ func TestQueryProfile(t *testing.T) {
 		}
 	}
 
-	t.Run("HTTP API", func(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
+		userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType)
-		userapi.AddInternalRoutes(router, userAPI)
+		defer close()
-		apiURL, cancel := test.ListenAndServe(t, router, false)
+		_, err := accountDB.CreateAccount(context.TODO(), "alice", serverName, "foobar", "", api.AccountTypeUser)
-		defer cancel()
-		httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{})
 		if err != nil {
-			t.Fatalf("failed to create HTTP client")
+			t.Fatalf("failed to make account: %s", err)
 		}
-		runCases(httpAPI, true)
+		if _, _, err := accountDB.SetAvatarURL(context.TODO(), "alice", serverName, aliceAvatarURL); err != nil {
-	})
+			t.Fatalf("failed to set avatar url: %s", err)
-	t.Run("Monolith", func(t *testing.T) {
+		}
-		runCases(userAPI, false)
+		if _, _, err := accountDB.SetDisplayName(context.TODO(), "alice", serverName, aliceDisplayName); err != nil {
+			t.Fatalf("failed to set display name: %s", err)
+		}
+
+		t.Run("HTTP API", func(t *testing.T) {
+			router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
+			userapi.AddInternalRoutes(router, userAPI, false)
+			apiURL, cancel := test.ListenAndServe(t, router, false)
+			defer cancel()
+			httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{})
+			if err != nil {
+				t.Fatalf("failed to create HTTP client")
+			}
+			runCases(httpAPI, true)
+		})
+		t.Run("Monolith", func(t *testing.T) {
+			runCases(userAPI, false)
+		})
 	})
 }