Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/usagestats

.github/workflows/schedules.yaml

@@ -0,0 +1,128 @@
+name: Scheduled
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # every day at midnight
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # run go test with different go versions
+  test:
+    timeout-minutes: 20
+    name: Unit tests (Go ${{ matrix.go }})
+    runs-on: ubuntu-latest
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:13-alpine
+        # Provide the password for postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.18", "1.19"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go }}
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go${{ matrix.go }}-test-race-
+      - run: go test -race ./...
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+
+  # Dummy step to gate other tests on without repeating the whole list
+  initial-tests-done:
+    name: Initial tests passed
+    needs: [test]
+    runs-on: ubuntu-latest
+    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
+    steps:
+      - name: Check initial tests passed
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}
+
+  # run Sytest in different variations
+  sytest:
+    timeout-minutes: 60
+    needs: initial-tests-done
+    name: "Sytest (${{ matrix.label }})"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: SQLite
+
+          - label: SQLite, full HTTP APIs
+            api: full-http
+
+          - label: PostgreSQL
+            postgres: postgres
+
+          - label: PostgreSQL, full HTTP APIs
+            postgres: postgres
+            api: full-http
+    container:
+      image: matrixdotorg/sytest-dendrite:latest
+      volumes:
+        - ${{ github.workspace }}:/src
+      env:
+        POSTGRES: ${{ matrix.postgres && 1}}
+        API: ${{ matrix.api && 1 }}
+        SYTEST_BRANCH: ${{ github.head_ref }}
+        RACE_DETECTION: 1
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run Sytest
+        run: /bootstrap.sh dendrite
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Sytest List Maintenance
+        if: ${{ always() }}
+        run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
+        continue-on-error: true # not fatal
+      - name: Are We Synapse Yet?
+        if: ${{ always() }}
+        run: /src/are-we-synapse-yet.py /logs/results.tap -v
+        continue-on-error: true # not fatal
+      - name: Upload Sytest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*

CHANGES.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## Dendrite 0.10.5 (2022-10-31)
+
+### Features
+
+* It is now possible to use hCaptcha instead of reCAPTCHA for protecting registration
+* A new `auto_join_rooms` configuration option has been added for automatically joining new users to a set of rooms
+* A new `/_dendrite/admin/downloadState/{serverName}/{roomID}` endpoint has been added, which allows a server administrator to attempt to repair a room with broken room state by downloading a state snapshot from another federated server in the room
+
+### Fixes
+
+* Querying cross-signing keys for users should now be considerably faster
+* A bug in state resolution where some events were not correctly selected for third-party invites has been fixed
+* A bug in state resolution which could result in `not in room` event rejections has been fixed
+* When accepting a DM invite, it should now be possible to see messages that were sent before the invite was accepted
+* Claiming remote E2EE one-time keys has been refactored and should be more reliable now
+* Various fixes have been made to the `/members` endpoint, which may help with E2EE reliability and clients rendering memberships
+* A race condition in the federation API destination queues has been fixed when associating queued events with remote server destinations
+* A bug in the sync API where too many events were selected resulting in high CPU usage has been fixed
+* Configuring the avatar URL for the Server Notices user should work correctly now
+
 ## Dendrite 0.10.4 (2022-10-21)
 
 ### Features

clientapi/routing/admin.go

@@ -191,3 +191,43 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien
 		JSON: struct{}{},
 	}
 }
+
+func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	roomID, ok := vars["roomID"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting room ID."),
+		}
+	}
+	serverName, ok := vars["serverName"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting remote server name."),
+		}
+	}
+	res := &roomserverAPI.PerformAdminDownloadStateResponse{}
+	if err := rsAPI.PerformAdminDownloadState(
+		req.Context(),
+		&roomserverAPI.PerformAdminDownloadStateRequest{
+			UserID:     device.UserID,
+			RoomID:     roomID,
+			ServerName: gomatrixserverlib.ServerName(serverName),
+		},
+		res,
+	); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{},
+	}
+}

clientapi/routing/auth_fallback.go

@@ -31,8 +31,7 @@ const recaptchaTemplate = `
 <title>Authentication</title>
 <meta name='viewport' content='width=device-width, initial-scale=1,
     user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
-<script src="https://www.google.com/recaptcha/api.js"
+<script src="{{.apiJsUrl}}" async defer></script>
-    async defer></script>
 <script src="//code.jquery.com/jquery-1.11.2.min.js"></script>
 <script>
 function captchaDone() {
@@ -51,8 +50,8 @@ function captchaDone() {
         Please verify that you're not a robot.
         </p>
 		<input type="hidden" name="session" value="{{.session}}" />
-        <div class="g-recaptcha"
+        <div class="{{.sitekeyClass}}"
-            data-sitekey="{{.siteKey}}"
+            data-sitekey="{{.sitekey}}"
             data-callback="captchaDone">
         </div>
         <noscript>
@@ -114,9 +113,12 @@ func AuthFallback(
 
 	serveRecaptcha := func() {
 		data := map[string]string{
-			"myUrl":   req.URL.String(),
+			"myUrl":        req.URL.String(),
-			"session": sessionID,
+			"session":      sessionID,
-			"siteKey": cfg.RecaptchaPublicKey,
+			"apiJsUrl":     cfg.RecaptchaApiJsUrl,
+			"sitekey":      cfg.RecaptchaPublicKey,
+			"sitekeyClass": cfg.RecaptchaSitekeyClass,
+			"formField":    cfg.RecaptchaFormField,
 		}
 		serveTemplate(w, recaptchaTemplate, data)
 	}
@@ -155,7 +157,7 @@ func AuthFallback(
 				return &res
 			}
 
-			response := req.Form.Get("g-recaptcha-response")
+			response := req.Form.Get(cfg.RecaptchaFormField)
 			if err := validateRecaptcha(cfg, response, clientIP); err != nil {
 				util.GetLogger(req.Context()).Error(err)
 				return err

clientapi/routing/directory.go

@@ -18,14 +18,15 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
 type roomDirectoryResponse struct {
@@ -318,3 +319,43 @@ func SetVisibility(
 		JSON: struct{}{},
 	}
 }
+
+func SetVisibilityAS(
+	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, dev *userapi.Device,
+	networkID, roomID string,
+) util.JSONResponse {
+	if dev.AccountType != userapi.AccountTypeAppService {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Only appservice may use this endpoint"),
+		}
+	}
+	var v roomVisibility
+
+	// If the method is delete, we simply mark the visibility as private
+	if req.Method == http.MethodDelete {
+		v.Visibility = "private"
+	} else {
+		if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil {
+			return *reqErr
+		}
+	}
+	var publishRes roomserverAPI.PerformPublishResponse
+	if err := rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
+		RoomID:       roomID,
+		Visibility:   v.Visibility,
+		NetworkID:    networkID,
+		AppserviceID: dev.AppserviceID,
+	}, &publishRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
+	if publishRes.Error != nil {
+		util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
+		return publishRes.Error.JSONResponse()
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/directory_public.go

@@ -39,14 +39,17 @@ var (
 )
 
 type PublicRoomReq struct {
-	Since  string `json:"since,omitempty"`
+	Since              string `json:"since,omitempty"`
-	Limit  int16  `json:"limit,omitempty"`
+	Limit              int64  `json:"limit,omitempty"`
-	Filter filter `json:"filter,omitempty"`
+	Filter             filter `json:"filter,omitempty"`
-	Server string `json:"server,omitempty"`
+	Server             string `json:"server,omitempty"`
+	IncludeAllNetworks bool   `json:"include_all_networks,omitempty"`
+	NetworkID          string `json:"third_party_instance_id,omitempty"`
 }
 
 type filter struct {
-	SearchTerms string `json:"generic_search_term,omitempty"`
+	SearchTerms string   `json:"generic_search_term,omitempty"`
+	RoomTypes   []string `json:"room_types,omitempty"` // TODO: Implement filter on this
 }
 
 // GetPostPublicRooms implements GET and POST /publicRooms
@@ -61,6 +64,13 @@ func GetPostPublicRooms(
 		return *fillErr
 	}
 
+	if request.IncludeAllNetworks && request.NetworkID != "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidParam("include_all_networks and third_party_instance_id can not be used together"),
+		}
+	}
+
 	serverName := gomatrixserverlib.ServerName(request.Server)
 	if serverName != "" && !cfg.Matrix.IsLocalServerName(serverName) {
 		res, err := federation.GetPublicRoomsFiltered(
@@ -97,7 +107,7 @@ func publicRooms(
 	response := gomatrixserverlib.RespPublicRooms{
 		Chunk: []gomatrixserverlib.PublicRoom{},
 	}
-	var limit int16
+	var limit int64
 	var offset int64
 	limit = request.Limit
 	if limit == 0 {
@@ -114,7 +124,7 @@ func publicRooms(
 
 	var rooms []gomatrixserverlib.PublicRoom
 	if request.Since == "" {
-		rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider)
+		rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider, request)
 	} else {
 		rooms = getPublicRoomsFromCache()
 	}
@@ -176,7 +186,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 				JSON: jsonerror.BadJSON("limit param is not a number"),
 			}
 		}
-		request.Limit = int16(limit)
+		request.Limit = int64(limit)
 		request.Since = httpReq.FormValue("since")
 		request.Server = httpReq.FormValue("server")
 	} else {
@@ -204,7 +214,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 //	 limit=3&since=6  => G     (prev='3', next='')
 //
 //	A value of '-1' for prev/next indicates no position.
-func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
+func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
 	prev = -1
 	next = -1
 
@@ -230,6 +240,7 @@ func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (
 
 func refreshPublicRoomCache(
 	ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
+	request PublicRoomReq,
 ) []gomatrixserverlib.PublicRoom {
 	cacheMu.Lock()
 	defer cacheMu.Unlock()
@@ -238,8 +249,17 @@ func refreshPublicRoomCache(
 		extraRooms = extRoomsProvider.Rooms()
 	}
 
+	// TODO: this is only here to make Sytest happy, for now.
+	ns := strings.Split(request.NetworkID, "|")
+	if len(ns) == 2 {
+		request.NetworkID = ns[1]
+	}
+
 	var queryRes roomserverAPI.QueryPublishedRoomsResponse
-	err := rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
+	err := rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{
+		NetworkID:          request.NetworkID,
+		IncludeAllNetworks: request.IncludeAllNetworks,
+	}, &queryRes)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
 		return publicRoomsCache

clientapi/routing/directory_public_test.go

@@ -17,7 +17,7 @@ func TestSliceInto(t *testing.T) {
 	slice := []gomatrixserverlib.PublicRoom{
 		pubRoom("a"), pubRoom("b"), pubRoom("c"), pubRoom("d"), pubRoom("e"), pubRoom("f"), pubRoom("g"),
 	}
-	limit := int16(3)
+	limit := int64(3)
 	testCases := []struct {
 		since      int64
 		wantPrev   int

clientapi/routing/register.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -336,6 +337,7 @@ func validateRecaptcha(
 	response string,
 	clientip string,
 ) *util.JSONResponse {
+	ip, _, _ := net.SplitHostPort(clientip)
 	if !cfg.RecaptchaEnabled {
 		return &util.JSONResponse{
 			Code: http.StatusConflict,
@@ -355,7 +357,7 @@ func validateRecaptcha(
 		url.Values{
 			"secret":   {cfg.RecaptchaPrivateKey},
 			"response": {response},
-			"remoteip": {clientip},
+			"remoteip": {ip},
 		},
 	)
 

clientapi/routing/routing.go

@@ -163,6 +163,12 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/downloadState/{serverName}/{roomID}",
+		httputil.MakeAdminAPI("admin_download_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminDownloadState(req, cfg, device, rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	dendriteAdminRouter.Handle("/admin/fulltext/reindex",
 		httputil.MakeAdminAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return AdminReindex(req, cfg, device, natsClient)
@@ -480,7 +486,7 @@ func Setup(
 			return GetVisibility(req, rsAPI, vars["roomID"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
-	// TODO: Add AS support
+
 	v3mux.Handle("/directory/list/room/{roomID}",
 		httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -490,6 +496,27 @@ func Setup(
 			return SetVisibility(req, rsAPI, device, vars["roomID"])
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
+	v3mux.Handle("/directory/list/appservice/{networkID}/{roomID}",
+		httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SetVisibilityAS(req, rsAPI, device, vars["networkID"], vars["roomID"])
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	// Undocumented endpoint
+	v3mux.Handle("/directory/list/appservice/{networkID}/{roomID}",
+		httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SetVisibilityAS(req, rsAPI, device, vars["networkID"], vars["roomID"])
+		}),
+	).Methods(http.MethodDelete, http.MethodOptions)
+
 	v3mux.Handle("/publicRooms",
 		httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse {
 			return GetPostPublicRooms(req, rsAPI, extRoomsProvider, federation, cfg)

dendrite-sample.monolith.yaml

@@ -179,7 +179,13 @@ client_api:
   recaptcha_public_key: ""
   recaptcha_private_key: ""
   recaptcha_bypass_secret: ""
-  recaptcha_siteverify_api: ""
+  
+  # To use hcaptcha.com instead of ReCAPTCHA, set the following parameters, otherwise just keep them empty.
+  # recaptcha_siteverify_api: "https://hcaptcha.com/siteverify"
+  # recaptcha_api_js_url: "https://js.hcaptcha.com/1/api.js"
+  # recaptcha_form_field: "h-captcha-response"
+  # recaptcha_sitekey_class: "h-captcha"
+
 
   # TURN server information that this homeserver should send to clients.
   turn:

dendrite-sample.polylith.yaml

@@ -175,7 +175,13 @@ client_api:
   recaptcha_public_key: ""
   recaptcha_private_key: ""
   recaptcha_bypass_secret: ""
-  recaptcha_siteverify_api: ""
+
+  # To use hcaptcha.com instead of ReCAPTCHA, set the following parameters, otherwise just keep them empty.
+  # recaptcha_siteverify_api: "https://hcaptcha.com/siteverify"
+  # recaptcha_api_js_url: "https://js.hcaptcha.com/1/api.js"
+  # recaptcha_form_field: "h-captcha-response"
+  # recaptcha_sitekey_class: "h-captcha"
+
 
   # TURN server information that this homeserver should send to clients.
   turn:

federationapi/internal/federationclient.go

@@ -44,7 +44,7 @@ func (a *FederationInternalAPI) ClaimKeys(
 ) (gomatrixserverlib.RespClaimKeys, error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
-	ires, err := a.doRequestIfNotBackingOffOrBlacklisted(s, func() (interface{}, error) {
+	ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
 		return a.federation.ClaimKeys(ctx, s, oneTimeKeys)
 	})
 	if err != nil {

federationapi/routing/publicrooms.go

@@ -2,24 +2,29 @@ package routing
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"strconv"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
 type PublicRoomReq struct {
-	Since  string `json:"since,omitempty"`
+	Since              string `json:"since,omitempty"`
-	Limit  int16  `json:"limit,omitempty"`
+	Limit              int16  `json:"limit,omitempty"`
-	Filter filter `json:"filter,omitempty"`
+	Filter             filter `json:"filter,omitempty"`
+	IncludeAllNetworks bool   `json:"include_all_networks,omitempty"`
+	NetworkID          string `json:"third_party_instance_id,omitempty"`
 }
 
 type filter struct {
-	SearchTerms string `json:"generic_search_term,omitempty"`
+	SearchTerms string   `json:"generic_search_term,omitempty"`
+	RoomTypes   []string `json:"room_types,omitempty"`
 }
 
 // GetPostPublicRooms implements GET and POST /publicRooms
@@ -57,8 +62,14 @@ func publicRooms(
 		return nil, err
 	}
 
+	if request.IncludeAllNetworks && request.NetworkID != "" {
+		return nil, fmt.Errorf("include_all_networks and third_party_instance_id can not be used together")
+	}
+
 	var queryRes roomserverAPI.QueryPublishedRoomsResponse
-	err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
+	err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{
+		NetworkID: request.NetworkID,
+	}, &queryRes)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
 		return nil, err

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74
 	github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15

go.sum

@@ -387,8 +387,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa h1:S98DShDv3sn7O4n4HjtJOejypseYVpv1R/XPg+cDnfI=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74 h1:I4LUlFqxZ72m3s9wIvUIV2FpprsxW28dO/0lAgepCZY=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 h1:nAT5w41Q9uWTSnpKW55/hBwP91j2IFYPDRs0jJ8TyFI=
 github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 10
-	VersionPatch = 4
+	VersionPatch = 5
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/internal/internal.go

@@ -128,58 +128,49 @@ func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformC
 func (a *KeyInternalAPI) claimRemoteKeys(
 	ctx context.Context, timeout time.Duration, res *api.PerformClaimKeysResponse, domainToDeviceKeys map[string]map[string]map[string]string,
 ) {
-	resultCh := make(chan *gomatrixserverlib.RespClaimKeys, len(domainToDeviceKeys))
+	var wg sync.WaitGroup // Wait for fan-out goroutines to finish
-	// allows us to wait until all federation servers have been poked
+	var mu sync.Mutex     // Protects the response struct
-	var wg sync.WaitGroup
+	var claimed int       // Number of keys claimed in total
-	wg.Add(len(domainToDeviceKeys))
+	var failures int      // Number of servers we failed to ask
-	// mutex for failures
+
-	var failMu sync.Mutex
+	util.GetLogger(ctx).Infof("Claiming remote keys from %d server(s)", len(domainToDeviceKeys))
-	util.GetLogger(ctx).WithField("num_servers", len(domainToDeviceKeys)).Info("Claiming remote keys from servers")
+	wg.Add(len(domainToDeviceKeys))
 
-	// fan out
 	for d, k := range domainToDeviceKeys {
 		go func(domain string, keysToClaim map[string]map[string]string) {
-			defer wg.Done()
 			fedCtx, cancel := context.WithTimeout(ctx, timeout)
 			defer cancel()
+			defer wg.Done()
+
 			claimKeyRes, err := a.FedClient.ClaimKeys(fedCtx, gomatrixserverlib.ServerName(domain), keysToClaim)
+
+			mu.Lock()
+			defer mu.Unlock()
+
 			if err != nil {
 				util.GetLogger(ctx).WithError(err).WithField("server", domain).Error("ClaimKeys failed")
-				failMu.Lock()
 				res.Failures[domain] = map[string]interface{}{
 					"message": err.Error(),
 				}
-				failMu.Unlock()
+				failures++
 				return
 			}
-			resultCh <- &claimKeyRes
+
+			for userID, deviceIDToKeys := range claimKeyRes.OneTimeKeys {
+				res.OneTimeKeys[userID] = make(map[string]map[string]json.RawMessage)
+				for deviceID, keys := range deviceIDToKeys {
+					res.OneTimeKeys[userID][deviceID] = keys
+					claimed += len(keys)
+				}
+			}
 		}(d, k)
 	}
 
-	// Close the result channel when the goroutines have quit so the for .. range exits
+	wg.Wait()
-	go func() {
+	util.GetLogger(ctx).WithFields(logrus.Fields{
-		wg.Wait()
+		"num_keys":     claimed,
-		close(resultCh)
+		"num_failures": failures,
-	}()
+	}).Infof("Claimed remote keys from %d server(s)", len(domainToDeviceKeys))
-
-	keysClaimed := 0
-	for result := range resultCh {
-		for userID, nest := range result.OneTimeKeys {
-			res.OneTimeKeys[userID] = make(map[string]map[string]json.RawMessage)
-			for deviceID, nest2 := range nest {
-				res.OneTimeKeys[userID][deviceID] = make(map[string]json.RawMessage)
-				for keyIDWithAlgo, otk := range nest2 {
-					keyJSON, err := json.Marshal(otk)
-					if err != nil {
-						continue
-					}
-					res.OneTimeKeys[userID][deviceID][keyIDWithAlgo] = keyJSON
-					keysClaimed++
-				}
-			}
-		}
-	}
-	util.GetLogger(ctx).WithField("num_keys", keysClaimed).Info("Claimed remote keys")
 }
 
 func (a *KeyInternalAPI) PerformDeleteKeys(ctx context.Context, req *api.PerformDeleteKeysRequest, res *api.PerformDeleteKeysResponse) error {

roomserver/api/api.go

@@ -150,6 +150,7 @@ type ClientRoomserverAPI interface {
 	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse) error
 	PerformAdminEvacuateRoom(ctx context.Context, req *PerformAdminEvacuateRoomRequest, res *PerformAdminEvacuateRoomResponse) error
 	PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse) error
+	PerformAdminDownloadState(ctx context.Context, req *PerformAdminDownloadStateRequest, res *PerformAdminDownloadStateResponse) error
 	PerformPeek(ctx context.Context, req *PerformPeekRequest, res *PerformPeekResponse) error
 	PerformUnpeek(ctx context.Context, req *PerformUnpeekRequest, res *PerformUnpeekResponse) error
 	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error

roomserver/api/api_trace.go

@@ -131,6 +131,16 @@ func (t *RoomserverInternalAPITrace) PerformAdminEvacuateUser(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) PerformAdminDownloadState(
+	ctx context.Context,
+	req *PerformAdminDownloadStateRequest,
+	res *PerformAdminDownloadStateResponse,
+) error {
+	err := t.Impl.PerformAdminDownloadState(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformAdminDownloadState req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
 func (t *RoomserverInternalAPITrace) PerformInboundPeek(
 	ctx context.Context,
 	req *PerformInboundPeekRequest,

roomserver/api/perform.go

@@ -168,8 +168,10 @@ type PerformBackfillResponse struct {
 }
 
 type PerformPublishRequest struct {
-	RoomID     string
+	RoomID       string
-	Visibility string
+	Visibility   string
+	AppserviceID string
+	NetworkID    string
 }
 
 type PerformPublishResponse struct {
@@ -235,3 +237,13 @@ type PerformAdminEvacuateUserResponse struct {
 	Affected []string `json:"affected"`
 	Error    *PerformError
 }
+
+type PerformAdminDownloadStateRequest struct {
+	RoomID     string                       `json:"room_id"`
+	UserID     string                       `json:"user_id"`
+	ServerName gomatrixserverlib.ServerName `json:"server_name"`
+}
+
+type PerformAdminDownloadStateResponse struct {
+	Error *PerformError `json:"error,omitempty"`
+}

roomserver/api/query.go

@@ -21,8 +21,9 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 )
 
 // QueryLatestEventsAndStateRequest is a request to QueryLatestEventsAndState
@@ -257,7 +258,9 @@ type QueryRoomVersionForRoomResponse struct {
 
 type QueryPublishedRoomsRequest struct {
 	// Optional. If specified, returns whether this room is published or not.
-	RoomID string
+	RoomID             string
+	NetworkID          string
+	IncludeAllNetworks bool
 }
 
 type QueryPublishedRoomsResponse struct {

roomserver/internal/perform/perform_admin.go

@@ -236,3 +236,145 @@ func (r *Admin) PerformAdminEvacuateUser(
 	}
 	return nil
 }
+
+func (r *Admin) PerformAdminDownloadState(
+	ctx context.Context,
+	req *api.PerformAdminDownloadStateRequest,
+	res *api.PerformAdminDownloadStateResponse,
+) error {
+	roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.DB.RoomInfo: %s", err),
+		}
+		return nil
+	}
+
+	if roomInfo == nil || roomInfo.IsStub() {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("room %q not found", req.RoomID),
+		}
+		return nil
+	}
+
+	fwdExtremities, _, depth, err := r.DB.LatestEventIDs(ctx, roomInfo.RoomNID)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.DB.LatestEventIDs: %s", err),
+		}
+		return nil
+	}
+
+	authEventMap := map[string]*gomatrixserverlib.Event{}
+	stateEventMap := map[string]*gomatrixserverlib.Event{}
+
+	for _, fwdExtremity := range fwdExtremities {
+		var state gomatrixserverlib.RespState
+		state, err = r.Inputer.FSAPI.LookupState(ctx, req.ServerName, req.RoomID, fwdExtremity.EventID, roomInfo.RoomVersion)
+		if err != nil {
+			res.Error = &api.PerformError{
+				Code: api.PerformErrorBadRequest,
+				Msg:  fmt.Sprintf("r.Inputer.FSAPI.LookupState (%q): %s", fwdExtremity.EventID, err),
+			}
+			return nil
+		}
+		for _, authEvent := range state.AuthEvents.UntrustedEvents(roomInfo.RoomVersion) {
+			if err = authEvent.VerifyEventSignatures(ctx, r.Inputer.KeyRing); err != nil {
+				continue
+			}
+			authEventMap[authEvent.EventID()] = authEvent
+		}
+		for _, stateEvent := range state.StateEvents.UntrustedEvents(roomInfo.RoomVersion) {
+			if err = stateEvent.VerifyEventSignatures(ctx, r.Inputer.KeyRing); err != nil {
+				continue
+			}
+			stateEventMap[stateEvent.EventID()] = stateEvent
+		}
+	}
+
+	authEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(authEventMap))
+	stateEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(stateEventMap))
+	stateIDs := make([]string, 0, len(stateEventMap))
+
+	for _, authEvent := range authEventMap {
+		authEvents = append(authEvents, authEvent.Headered(roomInfo.RoomVersion))
+	}
+	for _, stateEvent := range stateEventMap {
+		stateEvents = append(stateEvents, stateEvent.Headered(roomInfo.RoomVersion))
+		stateIDs = append(stateIDs, stateEvent.EventID())
+	}
+
+	builder := &gomatrixserverlib.EventBuilder{
+		Type:    "org.matrix.dendrite.state_download",
+		Sender:  req.UserID,
+		RoomID:  req.RoomID,
+		Content: gomatrixserverlib.RawJSON("{}"),
+	}
+
+	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("gomatrixserverlib.StateNeededForEventBuilder: %s", err),
+		}
+		return nil
+	}
+
+	queryRes := &api.QueryLatestEventsAndStateResponse{
+		RoomExists:   true,
+		RoomVersion:  roomInfo.RoomVersion,
+		LatestEvents: fwdExtremities,
+		StateEvents:  stateEvents,
+		Depth:        depth,
+	}
+
+	ev, err := eventutil.BuildEvent(ctx, builder, r.Cfg.Matrix, time.Now(), &eventsNeeded, queryRes)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("eventutil.BuildEvent: %s", err),
+		}
+		return nil
+	}
+
+	inputReq := &api.InputRoomEventsRequest{
+		Asynchronous: false,
+	}
+	inputRes := &api.InputRoomEventsResponse{}
+
+	for _, authEvent := range append(authEvents, stateEvents...) {
+		inputReq.InputRoomEvents = append(inputReq.InputRoomEvents, api.InputRoomEvent{
+			Kind:  api.KindOutlier,
+			Event: authEvent,
+		})
+	}
+
+	inputReq.InputRoomEvents = append(inputReq.InputRoomEvents, api.InputRoomEvent{
+		Kind:          api.KindNew,
+		Event:         ev,
+		Origin:        r.Cfg.Matrix.ServerName,
+		HasState:      true,
+		StateEventIDs: stateIDs,
+		SendAsServer:  string(r.Cfg.Matrix.ServerName),
+	})
+
+	if err := r.Inputer.InputRoomEvents(ctx, inputReq, inputRes); err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.Inputer.InputRoomEvents: %s", err),
+		}
+		return nil
+	}
+
+	if inputRes.ErrMsg != "" {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  inputRes.ErrMsg,
+		}
+	}
+
+	return nil
+}

roomserver/internal/perform/perform_publish.go

@@ -30,7 +30,7 @@ func (r *Publisher) PerformPublish(
 	req *api.PerformPublishRequest,
 	res *api.PerformPublishResponse,
 ) error {
-	err := r.DB.PublishRoom(ctx, req.RoomID, req.Visibility == "public")
+	err := r.DB.PublishRoom(ctx, req.RoomID, req.AppserviceID, req.NetworkID, req.Visibility == "public")
 	if err != nil {
 		res.Error = &api.PerformError{
 			Msg: err.Error(),

roomserver/internal/query/query.go

@@ -702,7 +702,7 @@ func (r *Queryer) QueryPublishedRooms(
 		}
 		return err
 	}
-	rooms, err := r.DB.GetPublishedRooms(ctx)
+	rooms, err := r.DB.GetPublishedRooms(ctx, req.NetworkID, req.IncludeAllNetworks)
 	if err != nil {
 		return err
 	}

roomserver/inthttp/client.go

@@ -27,18 +27,19 @@ const (
 	RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents"
 
 	// Perform operations
-	RoomserverPerformInvitePath            = "/roomserver/performInvite"
+	RoomserverPerformInvitePath             = "/roomserver/performInvite"
-	RoomserverPerformPeekPath              = "/roomserver/performPeek"
+	RoomserverPerformPeekPath               = "/roomserver/performPeek"
-	RoomserverPerformUnpeekPath            = "/roomserver/performUnpeek"
+	RoomserverPerformUnpeekPath             = "/roomserver/performUnpeek"
-	RoomserverPerformRoomUpgradePath       = "/roomserver/performRoomUpgrade"
+	RoomserverPerformRoomUpgradePath        = "/roomserver/performRoomUpgrade"
-	RoomserverPerformJoinPath              = "/roomserver/performJoin"
+	RoomserverPerformJoinPath               = "/roomserver/performJoin"
-	RoomserverPerformLeavePath             = "/roomserver/performLeave"
+	RoomserverPerformLeavePath              = "/roomserver/performLeave"
-	RoomserverPerformBackfillPath          = "/roomserver/performBackfill"
+	RoomserverPerformBackfillPath           = "/roomserver/performBackfill"
-	RoomserverPerformPublishPath           = "/roomserver/performPublish"
+	RoomserverPerformPublishPath            = "/roomserver/performPublish"
-	RoomserverPerformInboundPeekPath       = "/roomserver/performInboundPeek"
+	RoomserverPerformInboundPeekPath        = "/roomserver/performInboundPeek"
-	RoomserverPerformForgetPath            = "/roomserver/performForget"
+	RoomserverPerformForgetPath             = "/roomserver/performForget"
-	RoomserverPerformAdminEvacuateRoomPath = "/roomserver/performAdminEvacuateRoom"
+	RoomserverPerformAdminEvacuateRoomPath  = "/roomserver/performAdminEvacuateRoom"
-	RoomserverPerformAdminEvacuateUserPath = "/roomserver/performAdminEvacuateUser"
+	RoomserverPerformAdminEvacuateUserPath  = "/roomserver/performAdminEvacuateUser"
+	RoomserverPerformAdminDownloadStatePath = "/roomserver/performAdminDownloadState"
 
 	// Query operations
 	RoomserverQueryLatestEventsAndStatePath    = "/roomserver/queryLatestEventsAndState"
@@ -261,6 +262,17 @@ func (h *httpRoomserverInternalAPI) PerformAdminEvacuateRoom(
 	)
 }
 
+func (h *httpRoomserverInternalAPI) PerformAdminDownloadState(
+	ctx context.Context,
+	request *api.PerformAdminDownloadStateRequest,
+	response *api.PerformAdminDownloadStateResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformAdminDownloadState", h.roomserverURL+RoomserverPerformAdminDownloadStatePath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
 func (h *httpRoomserverInternalAPI) PerformAdminEvacuateUser(
 	ctx context.Context,
 	request *api.PerformAdminEvacuateUserRequest,

roomserver/inthttp/server.go

@@ -65,6 +65,11 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 		httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateUser", r.PerformAdminEvacuateUser),
 	)
 
+	internalAPIMux.Handle(
+		RoomserverPerformAdminDownloadStatePath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformAdminDownloadState", r.PerformAdminDownloadState),
+	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryPublishedRoomsPath,
 		httputil.MakeInternalRPCAPI("RoomserverQueryPublishedRooms", r.QueryPublishedRooms),

roomserver/storage/interface.go

@@ -139,9 +139,9 @@ type Database interface {
 	// Returns an error if the retrieval went wrong.
 	EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error)
 	// Publish or unpublish a room from the room directory.
-	PublishRoom(ctx context.Context, roomID string, publish bool) error
+	PublishRoom(ctx context.Context, roomID, appserviceID, networkID string, publish bool) error
 	// Returns a list of room IDs for rooms which are published.
-	GetPublishedRooms(ctx context.Context) ([]string, error)
+	GetPublishedRooms(ctx context.Context, networkID string, includeAllNetworks bool) ([]string, error)
 	// Returns whether a given room is published or not.
 	GetPublishedRoom(ctx context.Context, roomID string) (bool, error)
 

roomserver/storage/postgres/deltas/20221027084407_published_appservice.go

@@ -0,0 +1,45 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+)
+
+func UpPulishedAppservice(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_published ADD COLUMN IF NOT EXISTS appservice_id TEXT NOT NULL DEFAULT '';`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_published ADD COLUMN IF NOT EXISTS network_id TEXT NOT NULL DEFAULT '';`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownPublishedAppservice(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_published DROP COLUMN IF EXISTS appservice_id;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_published DROP COLUMN IF EXISTS network_id;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}

roomserver/storage/postgres/published_table.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 )
 
@@ -27,31 +28,48 @@ const publishedSchema = `
 -- Stores which rooms are published in the room directory
 CREATE TABLE IF NOT EXISTS roomserver_published (
     -- The room ID of the room
-    room_id TEXT NOT NULL PRIMARY KEY,
+    room_id TEXT NOT NULL,
+    -- The appservice ID of the room
+    appservice_id TEXT NOT NULL,
+    -- The network_id of the room
+    network_id TEXT NOT NULL,
     -- Whether it is published or not
-    published BOOLEAN NOT NULL DEFAULT false
+    published BOOLEAN NOT NULL DEFAULT false,
+    PRIMARY KEY (room_id, appservice_id, network_id)
 );
 `
 
 const upsertPublishedSQL = "" +
-	"INSERT INTO roomserver_published (room_id, published) VALUES ($1, $2) " +
+	"INSERT INTO roomserver_published (room_id, appservice_id, network_id, published) VALUES ($1, $2, $3, $4) " +
-	"ON CONFLICT (room_id) DO UPDATE SET published=$2"
+	"ON CONFLICT (room_id, appservice_id, network_id) DO UPDATE SET published=$4"
 
 const selectAllPublishedSQL = "" +
-	"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
+	"SELECT room_id FROM roomserver_published WHERE published = $1 AND CASE WHEN $2 THEN 1=1 ELSE network_id = '' END ORDER BY room_id ASC"
+
+const selectNetworkPublishedSQL = "" +
+	"SELECT room_id FROM roomserver_published WHERE published = $1 AND network_id = $2 ORDER BY room_id ASC"
 
 const selectPublishedSQL = "" +
 	"SELECT published FROM roomserver_published WHERE room_id = $1"
 
 type publishedStatements struct {
-	upsertPublishedStmt    *sql.Stmt
+	upsertPublishedStmt        *sql.Stmt
-	selectAllPublishedStmt *sql.Stmt
+	selectAllPublishedStmt     *sql.Stmt
-	selectPublishedStmt    *sql.Stmt
+	selectPublishedStmt        *sql.Stmt
+	selectNetworkPublishedStmt *sql.Stmt
 }
 
 func CreatePublishedTable(db *sql.DB) error {
 	_, err := db.Exec(publishedSchema)
-	return err
+	if err != nil {
+		return err
+	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "roomserver: published appservice",
+		Up:      deltas.UpPulishedAppservice,
+	})
+	return m.Up(context.Background())
 }
 
 func PreparePublishedTable(db *sql.DB) (tables.Published, error) {
@@ -61,14 +79,15 @@ func PreparePublishedTable(db *sql.DB) (tables.Published, error) {
 		{&s.upsertPublishedStmt, upsertPublishedSQL},
 		{&s.selectAllPublishedStmt, selectAllPublishedSQL},
 		{&s.selectPublishedStmt, selectPublishedSQL},
+		{&s.selectNetworkPublishedStmt, selectNetworkPublishedSQL},
 	}.Prepare(db)
 }
 
 func (s *publishedStatements) UpsertRoomPublished(
-	ctx context.Context, txn *sql.Tx, roomID string, published bool,
+	ctx context.Context, txn *sql.Tx, roomID, appserviceID, networkID string, published bool,
 ) (err error) {
 	stmt := sqlutil.TxStmt(txn, s.upsertPublishedStmt)
-	_, err = stmt.ExecContext(ctx, roomID, published)
+	_, err = stmt.ExecContext(ctx, roomID, appserviceID, networkID, published)
 	return
 }
 
@@ -84,10 +103,18 @@ func (s *publishedStatements) SelectPublishedFromRoomID(
 }
 
 func (s *publishedStatements) SelectAllPublishedRooms(
-	ctx context.Context, txn *sql.Tx, published bool,
+	ctx context.Context, txn *sql.Tx, networkID string, published, includeAllNetworks bool,
 ) ([]string, error) {
-	stmt := sqlutil.TxStmt(txn, s.selectAllPublishedStmt)
+	var rows *sql.Rows
-	rows, err := stmt.QueryContext(ctx, published)
+	var err error
+	if networkID != "" {
+		stmt := sqlutil.TxStmt(txn, s.selectNetworkPublishedStmt)
+		rows, err = stmt.QueryContext(ctx, published, networkID)
+	} else {
+		stmt := sqlutil.TxStmt(txn, s.selectAllPublishedStmt)
+		rows, err = stmt.QueryContext(ctx, published, includeAllNetworks)
+
+	}
 	if err != nil {
 		return nil, err
 	}

roomserver/storage/shared/storage.go

@@ -722,9 +722,9 @@ func (d *Database) storeEvent(
 	}, redactionEvent, redactedEventID, err
 }
 
-func (d *Database) PublishRoom(ctx context.Context, roomID string, publish bool) error {
+func (d *Database) PublishRoom(ctx context.Context, roomID, appserviceID, networkID string, publish bool) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.PublishedTable.UpsertRoomPublished(ctx, txn, roomID, publish)
+		return d.PublishedTable.UpsertRoomPublished(ctx, txn, roomID, appserviceID, networkID, publish)
 	})
 }
 
@@ -732,8 +732,8 @@ func (d *Database) GetPublishedRoom(ctx context.Context, roomID string) (bool, e
 	return d.PublishedTable.SelectPublishedFromRoomID(ctx, nil, roomID)
 }
 
-func (d *Database) GetPublishedRooms(ctx context.Context) ([]string, error) {
+func (d *Database) GetPublishedRooms(ctx context.Context, networkID string, includeAllNetworks bool) ([]string, error) {
-	return d.PublishedTable.SelectAllPublishedRooms(ctx, nil, true)
+	return d.PublishedTable.SelectAllPublishedRooms(ctx, nil, networkID, true, includeAllNetworks)
 }
 
 func (d *Database) MissingAuthPrevEvents(

roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go

@@ -0,0 +1,64 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+)
+
+func UpPulishedAppservice(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `	ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp;
+CREATE TABLE IF NOT EXISTS roomserver_published (
+    room_id TEXT NOT NULL,
+    appservice_id TEXT NOT NULL,
+    network_id TEXT NOT NULL,
+    published BOOLEAN NOT NULL DEFAULT false,
+    CONSTRAINT unique_published_idx PRIMARY KEY (room_id, appservice_id, network_id)
+);
+INSERT
+    INTO roomserver_published (
+      room_id, published
+    ) SELECT
+        room_id, published
+    FROM roomserver_published_tmp
+;
+DROP TABLE roomserver_published_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownPublishedAppservice(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `	ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp;
+CREATE TABLE IF NOT EXISTS roomserver_published (
+    room_id TEXT NOT NULL PRIMARY KEY,
+    published BOOLEAN NOT NULL DEFAULT false
+);
+INSERT
+    INTO roomserver_published (
+      room_id, published
+    ) SELECT
+        room_id, published
+    FROM roomserver_published_tmp
+;
+DROP TABLE roomserver_published_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}

roomserver/storage/sqlite3/published_table.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 )
 
@@ -27,31 +28,49 @@ const publishedSchema = `
 -- Stores which rooms are published in the room directory
 CREATE TABLE IF NOT EXISTS roomserver_published (
     -- The room ID of the room
-    room_id TEXT NOT NULL PRIMARY KEY,
+    room_id TEXT NOT NULL,
+    -- The appservice ID of the room
+    appservice_id TEXT NOT NULL,
+    -- The network_id of the room
+    network_id TEXT NOT NULL,
     -- Whether it is published or not
-    published BOOLEAN NOT NULL DEFAULT false
+    published BOOLEAN NOT NULL DEFAULT false,
+    PRIMARY KEY (room_id, appservice_id, network_id)
 );
 `
 
 const upsertPublishedSQL = "" +
-	"INSERT OR REPLACE INTO roomserver_published (room_id, published) VALUES ($1, $2)"
+	"INSERT INTO roomserver_published (room_id, appservice_id, network_id, published) VALUES ($1, $2, $3, $4)" +
+	" ON CONFLICT (room_id, appservice_id, network_id) DO UPDATE SET published = $4"
 
 const selectAllPublishedSQL = "" +
-	"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
+	"SELECT room_id FROM roomserver_published WHERE published = $1 AND CASE WHEN $2 THEN 1=1 ELSE network_id = '' END ORDER BY room_id ASC"
+
+const selectNetworkPublishedSQL = "" +
+	"SELECT room_id FROM roomserver_published WHERE published = $1 AND network_id = $2 ORDER BY room_id ASC"
 
 const selectPublishedSQL = "" +
 	"SELECT published FROM roomserver_published WHERE room_id = $1"
 
 type publishedStatements struct {
-	db                     *sql.DB
+	db                         *sql.DB
-	upsertPublishedStmt    *sql.Stmt
+	upsertPublishedStmt        *sql.Stmt
-	selectAllPublishedStmt *sql.Stmt
+	selectAllPublishedStmt     *sql.Stmt
-	selectPublishedStmt    *sql.Stmt
+	selectPublishedStmt        *sql.Stmt
+	selectNetworkPublishedStmt *sql.Stmt
 }
 
 func CreatePublishedTable(db *sql.DB) error {
 	_, err := db.Exec(publishedSchema)
-	return err
+	if err != nil {
+		return err
+	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "roomserver: published appservice",
+		Up:      deltas.UpPulishedAppservice,
+	})
+	return m.Up(context.Background())
 }
 
 func PreparePublishedTable(db *sql.DB) (tables.Published, error) {
@@ -63,14 +82,15 @@ func PreparePublishedTable(db *sql.DB) (tables.Published, error) {
 		{&s.upsertPublishedStmt, upsertPublishedSQL},
 		{&s.selectAllPublishedStmt, selectAllPublishedSQL},
 		{&s.selectPublishedStmt, selectPublishedSQL},
+		{&s.selectNetworkPublishedStmt, selectNetworkPublishedSQL},
 	}.Prepare(db)
 }
 
 func (s *publishedStatements) UpsertRoomPublished(
-	ctx context.Context, txn *sql.Tx, roomID string, published bool,
+	ctx context.Context, txn *sql.Tx, roomID, appserviceID, networkID string, published bool,
 ) error {
 	stmt := sqlutil.TxStmt(txn, s.upsertPublishedStmt)
-	_, err := stmt.ExecContext(ctx, roomID, published)
+	_, err := stmt.ExecContext(ctx, roomID, appserviceID, networkID, published)
 	return err
 }
 
@@ -86,10 +106,17 @@ func (s *publishedStatements) SelectPublishedFromRoomID(
 }
 
 func (s *publishedStatements) SelectAllPublishedRooms(
-	ctx context.Context, txn *sql.Tx, published bool,
+	ctx context.Context, txn *sql.Tx, networkID string, published, includeAllNetworks bool,
 ) ([]string, error) {
-	stmt := sqlutil.TxStmt(txn, s.selectAllPublishedStmt)
+	var rows *sql.Rows
-	rows, err := stmt.QueryContext(ctx, published)
+	var err error
+	if networkID != "" {
+		stmt := sqlutil.TxStmt(txn, s.selectNetworkPublishedStmt)
+		rows, err = stmt.QueryContext(ctx, published, networkID)
+	} else {
+		stmt := sqlutil.TxStmt(txn, s.selectAllPublishedStmt)
+		rows, err = stmt.QueryContext(ctx, published, includeAllNetworks)
+	}
 	if err != nil {
 		return nil, err
 	}

roomserver/storage/tables/interface.go

@@ -146,9 +146,9 @@ type Membership interface {
 }
 
 type Published interface {
-	UpsertRoomPublished(ctx context.Context, txn *sql.Tx, roomID string, published bool) (err error)
+	UpsertRoomPublished(ctx context.Context, txn *sql.Tx, roomID, appserviceID, networkID string, published bool) (err error)
 	SelectPublishedFromRoomID(ctx context.Context, txn *sql.Tx, roomID string) (published bool, err error)
-	SelectAllPublishedRooms(ctx context.Context, txn *sql.Tx, published bool) ([]string, error)
+	SelectAllPublishedRooms(ctx context.Context, txn *sql.Tx, networkdID string, published, includeAllNetworks bool) ([]string, error)
 }
 
 type RedactionInfo struct {

roomserver/storage/tables/published_table_test.go

@@ -2,16 +2,18 @@ package tables_test
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/stretchr/testify/assert"
 )
 
 func mustCreatePublishedTable(t *testing.T, dbType test.DBType) (tab tables.Published, close func()) {
@@ -46,10 +48,12 @@ func TestPublishedTable(t *testing.T) {
 
 		// Publish some rooms
 		publishedRooms := []string{}
+		asID := ""
+		nwID := ""
 		for i := 0; i < 10; i++ {
 			room := test.NewRoom(t, alice)
 			published := i%2 == 0
-			err := tab.UpsertRoomPublished(ctx, nil, room.ID, published)
+			err := tab.UpsertRoomPublished(ctx, nil, room.ID, asID, nwID, published)
 			assert.NoError(t, err)
 			if published {
 				publishedRooms = append(publishedRooms, room.ID)
@@ -61,19 +65,36 @@ func TestPublishedTable(t *testing.T) {
 		sort.Strings(publishedRooms)
 
 		// check that we get the expected published rooms
-		roomIDs, err := tab.SelectAllPublishedRooms(ctx, nil, true)
+		roomIDs, err := tab.SelectAllPublishedRooms(ctx, nil, "", true, true)
 		assert.NoError(t, err)
 		assert.Equal(t, publishedRooms, roomIDs)
 
 		// test an actual upsert
 		room := test.NewRoom(t, alice)
-		err = tab.UpsertRoomPublished(ctx, nil, room.ID, true)
+		err = tab.UpsertRoomPublished(ctx, nil, room.ID, asID, nwID, true)
 		assert.NoError(t, err)
-		err = tab.UpsertRoomPublished(ctx, nil, room.ID, false)
+		err = tab.UpsertRoomPublished(ctx, nil, room.ID, asID, nwID, false)
 		assert.NoError(t, err)
 		// should now be false, due to the upsert
 		publishedRes, err := tab.SelectPublishedFromRoomID(ctx, nil, room.ID)
 		assert.NoError(t, err)
-		assert.False(t, publishedRes)
+		assert.False(t, publishedRes, fmt.Sprintf("expected room %s to be unpublished", room.ID))
+
+		// network specific test
+		nwID = "irc"
+		room = test.NewRoom(t, alice)
+		err = tab.UpsertRoomPublished(ctx, nil, room.ID, asID, nwID, true)
+		assert.NoError(t, err)
+		publishedRooms = append(publishedRooms, room.ID)
+		sort.Strings(publishedRooms)
+		// should only return the room for network "irc"
+		allNWPublished, err := tab.SelectAllPublishedRooms(ctx, nil, nwID, true, true)
+		assert.NoError(t, err)
+		assert.Equal(t, []string{room.ID}, allNWPublished)
+
+		// check that we still get all published rooms regardless networkID
+		roomIDs, err = tab.SelectAllPublishedRooms(ctx, nil, "", true, true)
+		assert.NoError(t, err)
+		assert.Equal(t, publishedRooms, roomIDs)
 	})
 }

setup/config/config_clientapi.go

@@ -32,6 +32,12 @@ type ClientAPI struct {
 	// Boolean stating whether catpcha registration is enabled
 	// and required
 	RecaptchaEnabled bool `yaml:"enable_registration_captcha"`
+	// Recaptcha api.js Url, for compatible with hcaptcha.com, etc.
+	RecaptchaApiJsUrl string `yaml:"recaptcha_api_js_url"`
+	// Recaptcha div class for sitekey, for compatible with hcaptcha.com, etc.
+	RecaptchaSitekeyClass string `yaml:"recaptcha_sitekey_class"`
+	// Recaptcha form field, for compatible with hcaptcha.com, etc.
+	RecaptchaFormField string `yaml:"recaptcha_form_field"`
 	// This Home Server's ReCAPTCHA public key.
 	RecaptchaPublicKey string `yaml:"recaptcha_public_key"`
 	// This Home Server's ReCAPTCHA private key.
@@ -75,6 +81,18 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 		checkNotEmpty(configErrs, "client_api.recaptcha_public_key", c.RecaptchaPublicKey)
 		checkNotEmpty(configErrs, "client_api.recaptcha_private_key", c.RecaptchaPrivateKey)
 		checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", c.RecaptchaSiteVerifyAPI)
+		if c.RecaptchaSiteVerifyAPI == "" {
+			c.RecaptchaSiteVerifyAPI = "https://www.google.com/recaptcha/api/siteverify"
+		}
+		if c.RecaptchaApiJsUrl == "" {
+			c.RecaptchaApiJsUrl = "https://www.google.com/recaptcha/api.js"
+		}
+		if c.RecaptchaFormField == "" {
+			c.RecaptchaFormField = "g-recaptcha"
+		}
+		if c.RecaptchaSitekeyClass == "" {
+			c.RecaptchaSitekeyClass = "g-recaptcha-response"
+		}
 	}
 	// Ensure there is any spam counter measure when enabling registration
 	if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled {

syncapi/routing/memberships.go

@@ -76,6 +76,13 @@ func GetMemberships(
 		}
 	}
 
+	if joinedOnly && !queryRes.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
 	db, err := syncDB.NewDatabaseSnapshot(req.Context())
 	if err != nil {
 		return jsonerror.InternalServerError()
@@ -102,19 +109,15 @@ func GetMemberships(
 		return jsonerror.InternalServerError()
 	}
 
-	result, err := db.Events(req.Context(), eventIDs)
+	qryRes := &api.QueryEventsByIDResponse{}
-	if err != nil {
+	if err := rsAPI.QueryEventsByID(req.Context(), &api.QueryEventsByIDRequest{EventIDs: eventIDs}, qryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("db.Events failed")
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryEventsByID failed")
 		return jsonerror.InternalServerError()
 	}
 
+	result := qryRes.Events
+
 	if joinedOnly {
-		if !queryRes.IsInRoom {
-			return util.JSONResponse{
-				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
-			}
-		}
 		var res getJoinedMembersResponse
 		res.Joined = make(map[string]joinedMember)
 		for _, ev := range result {

syncapi/streams/stream_pdu.go

@@ -101,7 +101,7 @@ func (p *PDUStreamProvider) CompleteSync(
 		)
 		if jerr != nil {
 			req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed")
-			if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone {
+			if ctxErr := req.Context.Err(); ctxErr != nil || jerr == sql.ErrTxDone {
 				return from
 			}
 			continue
@@ -216,6 +216,9 @@ func (p *PDUStreamProvider) IncrementalSync(
 	return newPos
 }
 
+// Limit the recent events to X when going backwards
+const recentEventBackwardsLimit = 100
+
 // nolint:gocyclo
 func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	ctx context.Context,
@@ -229,9 +232,15 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 ) (types.StreamPosition, error) {
 
 	originalLimit := eventFilter.Limit
-	if r.Backwards {
+	// If we're going backwards, grep at least X events, this is mostly to satisfy Sytest
-		eventFilter.Limit = int(r.From - r.To)
+	if r.Backwards && originalLimit < recentEventBackwardsLimit {
+		eventFilter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way
+		diff := r.From - r.To
+		if diff > 0 && diff < recentEventBackwardsLimit {
+			eventFilter.Limit = int(diff)
+		}
 	}
+
 	recentStreamEvents, limited, err := snapshot.RecentEvents(
 		ctx, delta.RoomID, r,
 		eventFilter, true, true,
@@ -242,8 +251,10 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		}
 		return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err)
 	}
-	recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents)
+	recentEvents := gomatrixserverlib.HeaderedReverseTopologicalOrdering(
-	delta.StateEvents = removeDuplicates(delta.StateEvents, recentEvents) // roll back
+		snapshot.StreamEventsToEvents(device, recentStreamEvents),
+		gomatrixserverlib.TopologicalOrderByPrevEvents,
+	)
 	prevBatch, err := snapshot.GetBackwardTopologyPos(ctx, recentStreamEvents)
 	if err != nil {
 		return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err)
@@ -254,10 +265,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		return r.To, nil
 	}
 
-	// Sort the events so that we can pick out the latest events from both sections.
-	recentEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(recentEvents, gomatrixserverlib.TopologicalOrderByPrevEvents)
-	delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(delta.StateEvents, gomatrixserverlib.TopologicalOrderByAuthEvents)
-
 	// Work out what the highest stream position is for all of the events in this
 	// room that were returned.
 	latestPosition := r.To
@@ -305,6 +312,14 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		limited = true
 	}
 
+	// Now that we've filtered the timeline, work out which state events are still
+	// left. Anything that appears in the filtered timeline will be removed from the
+	// "state" section and kept in "timeline".
+	delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(
+		removeDuplicates(delta.StateEvents, recentEvents),
+		gomatrixserverlib.TopologicalOrderByAuthEvents,
+	)
+
 	if len(delta.StateEvents) > 0 {
 		updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID())
 	}
@@ -498,7 +513,6 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	// transaction IDs for complete syncs, but we do it anyway because Sytest demands it for:
 	// "Can sync a room with a message with a transaction id" - which does a complete sync to check.
 	recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents)
-	stateEvents = removeDuplicates(stateEvents, recentEvents)
 
 	events := recentEvents
 	// Only apply history visibility checks if the response is for joined rooms
@@ -512,7 +526,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	// If we are limited by the filter AND the history visibility filter
 	// didn't "remove" events, return that the response is limited.
 	limited = limited && len(events) == len(recentEvents)
-
+	stateEvents = removeDuplicates(stateEvents, recentEvents)
 	if stateFilter.LazyLoadMembers {
 		if err != nil {
 			return nil, err

sytest-blacklist

@@ -40,4 +40,9 @@ Accesing an AS-hosted room alias asks the AS server
 Guest users can join guest_access rooms
 
 # This will fail in HTTP API mode, so blacklisted for now
-If a device list update goes missing, the server resyncs on the next one
+
+If a device list update goes missing, the server resyncs on the next one
+
+# Might be a bug in the test because leaves do appear :-(
+
+Leaves are present in non-gapped incremental syncs

sytest-whitelist

@@ -699,7 +699,7 @@ We do send redundant membership state across incremental syncs if asked
 Rejecting invite over federation doesn't break incremental /sync
 Gapped incremental syncs include all state changes
 Old leaves are present in gapped incremental syncs
-Leaves are present in non-gapped incremental syncs
+#Leaves are present in non-gapped incremental syncs
 Members from the gap are included in gappy incr LL sync
 Presence can be set from sync
 /state returns M_NOT_FOUND for a rejected message event
@@ -757,4 +757,6 @@ Can get rooms/{roomId}/messages for a departed room (SPEC-216)
 Local device key changes appear in /keys/changes
 Can get rooms/{roomId}/members at a given point
 Can filter rooms/{roomId}/members
-Current state appears in timeline in private history with many messages after
+Current state appears in timeline in private history with many messages after
+AS can publish rooms in their own list
+AS and main public room lists are separate