Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/migrate

.github/workflows/dendrite.yml

@@ -73,6 +73,26 @@ jobs:
     timeout-minutes: 5
     name: Unit tests (Go ${{ matrix.go }})
     runs-on: ubuntu-latest
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:13-alpine
+        # Provide the password for postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     strategy:
       fail-fast: false
       matrix:
@@ -92,6 +112,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go${{ matrix.go }}-test-
       - run: go test ./...
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
 
   # build Dendrite for linux with different architectures and go versions
   build:
@@ -233,7 +258,14 @@ jobs:
       - name: Summarise results.tap
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-
+      - name: Sytest List Maintenance
+        if: ${{ always() }}
+        run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
+        continue-on-error: true # not fatal
+      - name: Are We Synapse Yet?
+        if: ${{ always() }}
+        run: /src/are-we-synapse-yet.py /logs/results.tap -v
+        continue-on-error: true # not fatal
       - name: Upload Sytest logs
         uses: actions/upload-artifact@v2
         if: ${{ always() }}

CHANGES.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## Dendrite 0.8.1 (2022-04-07)
+
+### Fixes
+
+* A bug which could result in the sync API deadlocking due to lock contention in the notifier has been fixed
+
+## Dendrite 0.8.0 (2022-04-07)
+
+### Features
+
+* Support for presence has been added
+  * Presence is not enabled by default
+  * The `global.presence.enable_inbound` and `global.presence.enable_outbound` configuration options allow configuring inbound and outbound presence separately
+* Support for room upgrades via the `/room/{roomID}/upgrade` endpoint has been added (contributed by [DavidSpenler](https://github.com/DavidSpenler), [alexkursell](https://github.com/alexkursell))
+* Support for ignoring users has been added
+* Joined and invite user counts are now sent in the `/sync` room summaries
+* Queued federation and stale device list updates will now be staggered at startup over an up-to 2 minute warm-up period, rather than happening all at once
+* Memory pressure created by the sync notifier has been reduced
+* The EDU server component has now been removed, with the work being moved to more relevant components
+
+### Fixes
+
+* It is now possible to set the `power_level_content_override` when creating a room to include power levels over 100
+* `/send_join` and `/state` responses will now not unmarshal the JSON twice
+* The stream event consumer for push notifications will no longer request membership events that are irrelevant
+* Appservices will no longer incorrectly receive state events twice
+
 ## Dendrite 0.7.0 (2022-03-25)
 
 ### Features

README.md

@@ -82,20 +82,17 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of March 2022 we're at around 76% CS API coverage and 95% Federation coverage, though check
+updates with CI. As of April 2022 we're at around 83% CS API coverage and 95% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
-servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably:
-
-- Search
-- User Directory
-- Presence
+servers such as matrix.org reasonably well, although there are still some missing features (like Search).
 
 We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather
 than features that massive deployments may be interested in (User Directory, OpenID, Guests, Admin APIs, AS API).
 This means Dendrite supports amongst others:
 
 - Core room functionality (creating rooms, invites, auth rules)
-- Federation in rooms v1-v7
+- Full support for room versions 1 to 7
+- Experimental support for room versions 8 to 9
 - Backfilling locally and via federation
 - Accounts, Profiles and Devices
 - Published room lists
@@ -108,6 +105,8 @@ This means Dendrite supports amongst others:
 - Receipts
 - Push
 - Guests
+- User Directory
+- Presence
 
 ## Contributing
 

appservice/consumers/roomserver.go

@@ -83,20 +83,29 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 		return true
 	}
 
-	if output.Type != api.OutputTypeNewRoomEvent {
+	if output.Type != api.OutputTypeNewRoomEvent || output.NewRoomEvent == nil {
 		return true
 	}
 
-	events := []*gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
+	newEventID := output.NewRoomEvent.Event.EventID()
+	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(output.NewRoomEvent.AddsStateEventIDs))
+	events = append(events, output.NewRoomEvent.Event)
 	if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
 		eventsReq := &api.QueryEventsByIDRequest{
-			EventIDs: output.NewRoomEvent.AddsStateEventIDs,
+			EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
 		}
 		eventsRes := &api.QueryEventsByIDResponse{}
-		if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
-			return false
+		for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
+			if eventID != newEventID {
+				eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
+			}
+		}
+		if len(eventsReq.EventIDs) > 0 {
+			if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
+				return false
+			}
+			events = append(events, eventsRes.Events...)
 		}
-		events = append(events, eventsRes.Events...)
 	}
 
 	// Send event to any relevant application services

are-we-synapse-yet.list

@@ -106,10 +106,13 @@ rst Users cannot set notifications powerlevel higher than their own (2 subtests)
 rst Both GET and PUT work
 rct POST /rooms/:room_id/receipt can create receipts 
 red POST /rooms/:room_id/read_markers can create read marker
+med POST /media/v3/upload can create an upload
 med POST /media/r0/upload can create an upload 
+med GET /media/v3/download can fetch the value again
 med GET /media/r0/download can fetch the value again 
 cap GET /capabilities is present and well formed for registered user 
 cap GET /r0/capabilities is not public
+cap GET /v3/capabilities is not public
 reg Register with a recaptcha 
 reg registration is idempotent, without username specified 
 reg registration is idempotent, with username specified 
@@ -174,7 +177,9 @@ eph Ephemeral messages received from clients are correctly expired
 ali Room aliases can contain Unicode
 f,ali Remote room alias queries can handle Unicode
 ali Canonical alias can be set
+ali Canonical alias can be set (3 subtests)
 ali Canonical alias can include alt_aliases
+ali Canonical alias can include alt_aliases (4 subtests)
 ali Regular users can add and delete aliases in the default room configuration
 ali Regular users can add and delete aliases when m.room.aliases is restricted
 ali Deleting a non-existent alias should return a 404
@@ -207,11 +212,12 @@ plv Users cannot set kick powerlevel higher than their own (2 subtests)
 plv Users cannot set redact powerlevel higher than their own (2 subtests)
 v1s Check that event streams started after a client joined a room work (SYT-1)
 v1s Event stream catches up fully after many messages
-xxx POST /rooms/:room_id/redact/:event_id as power user redacts message
-xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message
-xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message
-xxx POST /redact disallows redaction of event in different room
+xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message
+xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message
+xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as random user does not redact message
+xxx PUT /redact disallows redaction of event in different room
 xxx Redaction of a redaction redacts the redaction reason
+xxx PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
 v1s A departed room is still included in /initialSync (SPEC-216)
 v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216)
 rst Can get rooms/{roomId}/state for a departed room (SPEC-216)
@@ -478,6 +484,30 @@ rmv Inbound federation rejects invites which include invalid JSON for room versi
 rmv Outbound federation rejects invite response which include invalid JSON for room version 6
 rmv Inbound federation rejects invite rejections which include invalid JSON for room version 6
 rmv Server rejects invalid JSON in a version 6 room
+rmv User can create and send/receive messages in a room with version 7 (2 subtests)
+rmv local user can join room with version 7
+rmv User can invite local user to room with version 7
+rmv remote user can join room with version 7
+rmv User can invite remote user to room with version 7
+rmv Remote user can backfill in a room with version 7
+rmv Can reject invites over federation for rooms with version 7
+rmv Can receive redactions from regular users over federation in room version 7
+rmv User can create and send/receive messages in a room with version 8 (2 subtests)
+rmv local user can join room with version 8
+rmv User can invite local user to room with version 8
+rmv remote user can join room with version 8
+rmv User can invite remote user to room with version 8
+rmv Remote user can backfill in a room with version 8
+rmv Can reject invites over federation for rooms with version 8
+rmv Can receive redactions from regular users over federation in room version 8
+rmv User can create and send/receive messages in a room with version 9 (2 subtests)
+rmv local user can join room with version 9
+rmv User can invite local user to room with version 9
+rmv remote user can join room with version 9
+rmv User can invite remote user to room with version 9
+rmv Remote user can backfill in a room with version 9
+rmv Can reject invites over federation for rooms with version 9
+rmv Can receive redactions from regular users over federation in room version 9
 pre Presence changes are reported to local room members 
 f,pre Presence changes are also reported to remote room members 
 pre Presence changes to UNAVAILABLE are reported to local room members 
@@ -772,12 +802,15 @@ app AS can make room aliases
 app Regular users cannot create room aliases within the AS namespace
 app AS-ghosted users can use rooms via AS 
 app AS-ghosted users can use rooms themselves 
+app AS-ghosted users can use rooms via AS (2 subtests)
+app AS-ghosted users can use rooms themselves (3 subtests)
 app Ghost user must register before joining room 
 app AS can set avatar for ghosted users 
 app AS can set displayname for ghosted users 
 app AS can't set displayname for random users
 app Inviting an AS-hosted user asks the AS server 
 app Accesing an AS-hosted room alias asks the AS server 
+app Accesing an AS-hosted room alias asks the AS server (2 subtests)
 app Events in rooms with AS-hosted room aliases are sent to AS server 
 app AS user (not ghost) can join room without registering 
 app AS user (not ghost) can join room without registering, with user_id query param
@@ -789,6 +822,8 @@ app AS can publish rooms in their own list
 app AS and main public room lists are separate 
 app AS can deactivate a user 
 psh Test that a message is pushed 
+psh Test that a message is pushed (6 subtests)
+psh Test that rejected pushers are removed. (4 subtests)
 psh Invites are pushed 
 psh Rooms with names are correctly named in pushed 
 psh Rooms with canonical alias are correctly named in pushed 
@@ -857,9 +892,12 @@ pre Left room members do not cause problems for presence
 crm Rooms can be created with an initial invite list (SYN-205) (1 subtests)
 typ Typing notifications don't leak 
 ban Non-present room members cannot ban others 
+ban Non-present room members cannot ban others (3 subtests)
 psh Getting push rules doesn't corrupt the cache SYN-390 
+psh Getting push rules doesn't corrupt the cache SYN-390 (3 subtests)
 inv Test that we can be reinvited to a room we created 
 syn Multiple calls to /sync should not cause 500 errors 
+syn Multiple calls to /sync should not cause 500 errors (6 subtests)
 gst Guest user can call /events on another world_readable room (SYN-606) 
 gst Real user can call /events on another world_readable room (SYN-606) 
 gst Events come down the correct room
@@ -884,3 +922,18 @@ msc We can't peek into rooms with invited history_visibility
 msc We can't peek into rooms with joined history_visibility
 msc Local users can peek by room alias
 msc Peeked rooms only turn up in the sync for the device who peeked them
+ban 'ban' event respects room powerlevel (2 subtests)
+inv Test that we can be reinvited to a room we created (11 subtests)
+fiv Rejecting invite over federation doesn't break incremental /sync
+pre Presence can be set from sync
+fst /state returns M_NOT_FOUND for an outlier
+fst /state_ids returns M_NOT_FOUND for an outlier
+fst /state returns M_NOT_FOUND for a rejected message event
+fst /state_ids returns M_NOT_FOUND for a rejected message event
+fst /state returns M_NOT_FOUND for a rejected state event
+fst /state_ids returns M_NOT_FOUND for a rejected state event
+fst Room state after a rejected message event is the same as before
+fst Room state after a rejected state event is the same as before
+fpb Federation publicRoom Name/topic keys are correct
+fed New federated private chats get full presence information (SYN-115) (10 subtests)
+dvk Rejects invalid device keys

are-we-synapse-yet.py

@@ -3,7 +3,7 @@
 from __future__ import division
 import argparse
 import re
-import sys
+import os
 
 # Usage: $ ./are-we-synapse-yet.py [-v] results.tap
 # This script scans a results.tap file from Dendrite's CI process and spits out
@@ -156,6 +156,7 @@ def parse_test_line(line):
 #    ✓ POST /register downcases capitals in usernames
 #    ...
 def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
+    ci = os.getenv("CI") # When running from GHA, this groups the subsections
     subsections = [] # Registration: 100% (13/13 tests)
     subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"]
     total_passing = 0
@@ -169,7 +170,7 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
         for name, passing in tests.items():
             if passing:
                 group_passing += 1
-            test_names_and_marks.append(f"{'✓' if passing else '×'} {name}")
+            test_names_and_marks.append(f"{'✅' if passing else '❌'} {name}")
             
         total_tests += group_total
         total_passing += group_passing
@@ -186,11 +187,11 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
     print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests))
     print("-" * (len(header_name)+1))
     for line in subsections:
-        print("  %s" % (line,))
+        print("%s%s" % ("::group::" if ci and verbose else "", line,))
         if verbose:
             for test_name_and_pass_mark in subsection_test_names[line]:
                 print("    %s" % (test_name_and_pass_mark,))
-            print("")
+            print("%s" % ("::endgroup::" if ci else ""))
     print("")
 
 def main(results_tap_path, verbose):

build.cmd

@@ -0,0 +1,51 @@
+@echo off
+
+:ENTRY_POINT
+    setlocal EnableDelayedExpansion
+
+    REM script base dir
+    set SCRIPTDIR=%~dp0
+    set PROJDIR=%SCRIPTDIR:~0,-1%
+
+    REM Put installed packages into ./bin
+    set GOBIN=%PROJDIR%\bin
+
+    set FLAGS=
+
+    REM Check if sources are under Git control
+    if not exist ".git" goto :CHECK_BIN
+
+    REM set BUILD=`git rev-parse --short HEAD \\ ""`
+    FOR /F "tokens=*" %%X IN ('git rev-parse --short HEAD') DO (
+        set BUILD=%%X
+    )
+
+    REM set BRANCH=`(git symbolic-ref --short HEAD \ tr -d \/ ) \\ ""`
+    FOR /F "tokens=*" %%X IN ('git symbolic-ref --short HEAD') DO (
+        set BRANCHRAW=%%X
+        set BRANCH=!BRANCHRAW:/=!
+    )
+    if "%BRANCH%" == "main" set BRANCH=
+
+    set FLAGS=-X github.com/matrix-org/dendrite/internal.branch=%BRANCH% -X github.com/matrix-org/dendrite/internal.build=%BUILD%
+
+:CHECK_BIN
+    if exist "bin" goto :ALL_SET
+    mkdir "bin"
+
+:ALL_SET
+    set CGO_ENABLED=1
+    for /D %%P in (cmd\*) do (
+        go build -trimpath -ldflags "%FLAGS%" -v -o ".\bin" ".\%%P"
+    )
+
+    set CGO_ENABLED=0
+    set GOOS=js
+    set GOARCH=wasm
+    go build -trimpath -ldflags "%FLAGS%" -o bin\main.wasm .\cmd\dendritejs-pinecone
+
+    goto :DONE
+
+:DONE
+    echo Done
+    endlocal

build/docker/config/dendrite.yaml

@@ -62,6 +62,17 @@ global:
     - matrix.org
     - vector.im
 
+  # Disables federation. Dendrite will not be able to make any outbound HTTP requests
+  # to other servers and the federation API will not be exposed.
+  disable_federation: false
+
+  # Configures the handling of presence events.
+  presence:
+    # Whether inbound presence events are allowed, e.g. receiving presence events from other servers
+    enable_inbound: false
+    # Whether outbound presence events are allowed, e.g. sending presence events to other servers
+    enable_outbound: false
+
   # Configuration for NATS JetStream
   jetstream:
     # A list of NATS Server addresses to connect to. If none are specified, an
@@ -173,12 +184,6 @@ federation_api:
     max_idle_conns: 2
     conn_max_lifetime: -1
 
-  # List of paths to X.509 certificates to be used by the external federation listeners.
-  # These certificates will be used to calculate the TLS fingerprints and other servers
-  # will expect the certificate to match these fingerprints. Certificates must be in PEM
-  # format.
-  federation_certificates: []
-
   # How many times we will try to resend a failed transaction to a specific server. The
   # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc.
   send_max_retries: 16

build/gobind-pinecone/monolith.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"os"
@@ -39,7 +38,6 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
@@ -54,6 +52,7 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 
+	pineconeConnections "github.com/matrix-org/pinecone/connections"
 	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
 	pineconeRouter "github.com/matrix-org/pinecone/router"
 	pineconeSessions "github.com/matrix-org/pinecone/sessions"
@@ -73,11 +72,9 @@ type DendriteMonolith struct {
 	PineconeRouter    *pineconeRouter.Router
 	PineconeMulticast *pineconeMulticast.Multicast
 	PineconeQUIC      *pineconeSessions.Sessions
+	PineconeManager   *pineconeConnections.ConnectionManager
 	StorageDirectory  string
 	CacheDirectory    string
-	staticPeerURI     string
-	staticPeerMutex   sync.RWMutex
-	staticPeerAttempt chan struct{}
 	listener          net.Listener
 	httpServer        *http.Server
 	processContext    *process.ProcessContext
@@ -93,7 +90,7 @@ func (m *DendriteMonolith) PeerCount(peertype int) int {
 }
 
 func (m *DendriteMonolith) SessionCount() int {
-	return len(m.PineconeQUIC.Sessions())
+	return len(m.PineconeQUIC.Protocol("matrix").Sessions())
 }
 
 func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
@@ -106,15 +103,8 @@ func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
 }
 
 func (m *DendriteMonolith) SetStaticPeer(uri string) {
-	m.staticPeerMutex.Lock()
-	m.staticPeerURI = strings.TrimSpace(uri)
-	m.staticPeerMutex.Unlock()
-	m.DisconnectType(int(pineconeRouter.PeerTypeRemote))
-	if uri != "" {
-		go func() {
-			m.staticPeerAttempt <- struct{}{}
-		}()
-	}
+	m.PineconeManager.RemovePeers()
+	m.PineconeManager.AddPeer(strings.TrimSpace(uri))
 }
 
 func (m *DendriteMonolith) DisconnectType(peertype int) {
@@ -212,43 +202,6 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
 	return loginRes.Device.AccessToken, nil
 }
 
-func (m *DendriteMonolith) staticPeerConnect() {
-	connected := map[string]bool{} // URI -> connected?
-	attempt := func() {
-		m.staticPeerMutex.RLock()
-		uri := m.staticPeerURI
-		m.staticPeerMutex.RUnlock()
-		if uri == "" {
-			return
-		}
-		for k := range connected {
-			delete(connected, k)
-		}
-		for _, uri := range strings.Split(uri, ",") {
-			connected[strings.TrimSpace(uri)] = false
-		}
-		for _, info := range m.PineconeRouter.Peers() {
-			connected[info.URI] = true
-		}
-		for k, online := range connected {
-			if !online {
-				if err := conn.ConnectToPeer(m.PineconeRouter, k); err != nil {
-					logrus.WithError(err).Error("Failed to connect to static peer")
-				}
-			}
-		}
-	}
-	for {
-		select {
-		case <-m.processContext.Context().Done():
-		case <-m.staticPeerAttempt:
-			attempt()
-		case <-time.After(time.Second * 5):
-			attempt()
-		}
-	}
-}
-
 // nolint:gocyclo
 func (m *DendriteMonolith) Start() {
 	var err error
@@ -272,7 +225,7 @@ func (m *DendriteMonolith) Start() {
 		pk = sk.Public().(ed25519.PublicKey)
 	}
 
-	m.listener, err = net.Listen("tcp", "localhost:65432")
+	m.listener, err = net.Listen("tcp", ":65432")
 	if err != nil {
 		panic(err)
 	}
@@ -283,10 +236,10 @@ func (m *DendriteMonolith) Start() {
 	m.logger.SetOutput(BindLogger{})
 	logrus.SetOutput(BindLogger{})
 
-	logger := log.New(os.Stdout, "PINECONE: ", 0)
-	m.PineconeRouter = pineconeRouter.NewRouter(logger, sk, false)
-	m.PineconeQUIC = pineconeSessions.NewSessions(logger, m.PineconeRouter)
-	m.PineconeMulticast = pineconeMulticast.NewMulticast(logger, m.PineconeRouter)
+	m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
+	m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
+	m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
+	m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter)
 
 	prefix := hex.EncodeToString(pk)
 	cfg := &config.Dendrite{}
@@ -374,7 +327,7 @@ func (m *DendriteMonolith) Start() {
 	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
 	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 
-	pHTTP := m.PineconeQUIC.HTTP()
+	pHTTP := m.PineconeQUIC.Protocol("matrix").HTTP()
 	pHTTP.Mux().Handle(users.PublicURL, pMux)
 	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
 	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
@@ -395,25 +348,14 @@ func (m *DendriteMonolith) Start() {
 
 	m.processContext = base.ProcessContext
 
-	m.staticPeerAttempt = make(chan struct{}, 1)
-	go m.staticPeerConnect()
-
 	go func() {
 		m.logger.Info("Listening on ", cfg.Global.ServerName)
-		m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC))
+		m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")))
 	}()
 	go func() {
 		logrus.Info("Listening on ", m.listener.Addr())
 		logrus.Fatal(http.Serve(m.listener, httpRouter))
 	}()
-	go func() {
-		logrus.Info("Sending wake-up message to known nodes")
-		req := &api.PerformBroadcastEDURequest{}
-		res := &api.PerformBroadcastEDUResponse{}
-		if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil {
-			logrus.WithError(err).Error("Failed to send wake-up message to known nodes")
-		}
-	}()
 }
 
 func (m *DendriteMonolith) Stop() {

clientapi/clientapi.go

@@ -48,7 +48,7 @@ func AddPublicRoutes(
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	mscCfg *config.MSCs,
 ) {
-	js, _ := jetstream.Prepare(process, &cfg.Matrix.JetStream)
+	js, natsClient := jetstream.Prepare(process, &cfg.Matrix.JetStream)
 
 	syncProducer := &producers.SyncAPIProducer{
 		JetStream:              js,
@@ -56,6 +56,7 @@ func AddPublicRoutes(
 		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
 		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
 		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
 		UserAPI:                userAPI,
 		ServerName:             cfg.Matrix.ServerName,
 	}
@@ -64,6 +65,6 @@ func AddPublicRoutes(
 		router, synapseAdminRouter, cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
 		syncProducer, transactionsCache, fsAPI, keyAPI,
-		extRoomsProvider, mscCfg,
+		extRoomsProvider, mscCfg, natsClient,
 	)
 }

clientapi/producers/syncapi.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"strconv"
+	"time"
 
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -34,13 +35,14 @@ type SyncAPIProducer struct {
 	TopicReceiptEvent      string
 	TopicSendToDeviceEvent string
 	TopicTypingEvent       string
+	TopicPresenceEvent     string
 	JetStream              nats.JetStreamContext
 	ServerName             gomatrixserverlib.ServerName
 	UserAPI                userapi.UserInternalAPI
 }
 
 // SendData sends account data to the sync API server
-func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON) error {
+func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON, ignoredUsers *types.IgnoredUsers) error {
 	m := &nats.Msg{
 		Subject: p.TopicClientData,
 		Header:  nats.Header{},
@@ -48,9 +50,10 @@ func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string
 	m.Header.Set(jetstream.UserID, userID)
 
 	data := eventutil.AccountData{
-		RoomID:     roomID,
-		Type:       dataType,
-		ReadMarker: readMarker,
+		RoomID:       roomID,
+		Type:         dataType,
+		ReadMarker:   readMarker,
+		IgnoredUsers: ignoredUsers,
 	}
 	var err error
 	m.Data, err = json.Marshal(data)
@@ -173,3 +176,19 @@ func (p *SyncAPIProducer) SendTyping(
 	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
 	return err
 }
+
+func (p *SyncAPIProducer) SendPresence(
+	ctx context.Context, userID string, presence types.Presence, statusMsg *string,
+) error {
+	m := nats.NewMsg(p.TopicPresenceEvent)
+	m.Header.Set(jetstream.UserID, userID)
+	m.Header.Set("presence", presence.String())
+	if statusMsg != nil {
+		m.Header.Set("status_msg", *statusMsg)
+	}
+
+	m.Header.Set("last_active_ts", strconv.Itoa(int(gomatrixserverlib.AsTimestamp(time.Now()))))
+
+	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}

clientapi/routing/account_data.go

@@ -25,6 +25,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/util"
@@ -94,10 +95,10 @@ func SaveAccountData(
 		}
 	}
 
-	if dataType == "m.fully_read" {
+	if dataType == "m.fully_read" || dataType == "m.push_rules" {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Unable to set read marker"),
+			JSON: jsonerror.Forbidden(fmt.Sprintf("Unable to modify %q using this API", dataType)),
 		}
 	}
 
@@ -126,8 +127,14 @@ func SaveAccountData(
 		return util.ErrorResponse(err)
 	}
 
+	var ignoredUsers *types.IgnoredUsers
+	if dataType == "m.ignored_user_list" {
+		ignoredUsers = &types.IgnoredUsers{}
+		_ = json.Unmarshal(body, ignoredUsers)
+	}
+
 	// TODO: user API should do this since it's account data
-	if err := syncProducer.SendData(userID, roomID, dataType, nil); err != nil {
+	if err := syncProducer.SendData(userID, roomID, dataType, nil, ignoredUsers); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
 		return jsonerror.InternalServerError()
 	}
@@ -184,7 +191,7 @@ func SaveReadMarker(
 		return util.ErrorResponse(err)
 	}
 
-	if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r); err != nil {
+	if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r, nil); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/presence.go

@@ -0,0 +1,137 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+)
+
+type presenceReq struct {
+	Presence  string  `json:"presence"`
+	StatusMsg *string `json:"status_msg,omitempty"`
+}
+
+func SetPresence(
+	req *http.Request,
+	cfg *config.ClientAPI,
+	device *api.Device,
+	producer *producers.SyncAPIProducer,
+	userID string,
+) util.JSONResponse {
+	if !cfg.Matrix.Presence.EnableOutbound {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: struct{}{},
+		}
+	}
+	if device.UserID != userID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Unable to set presence for other user."),
+		}
+	}
+	var presence presenceReq
+	parseErr := httputil.UnmarshalJSONRequest(req, &presence)
+	if parseErr != nil {
+		return *parseErr
+	}
+
+	presenceStatus, ok := types.PresenceFromString(presence.Presence)
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown(fmt.Sprintf("Unknown presence '%s'.", presence.Presence)),
+		}
+	}
+	err := producer.SendPresence(req.Context(), userID, presenceStatus, presence.StatusMsg)
+	if err != nil {
+		log.WithError(err).Errorf("failed to update presence")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func GetPresence(
+	req *http.Request,
+	device *api.Device,
+	natsClient *nats.Conn,
+	presenceTopic string,
+	userID string,
+) util.JSONResponse {
+	msg := nats.NewMsg(presenceTopic)
+	msg.Header.Set(jetstream.UserID, userID)
+
+	presence, err := natsClient.RequestMsg(msg, time.Second*10)
+	if err != nil {
+		log.WithError(err).Errorf("unable to get presence")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+
+	statusMsg := presence.Header.Get("status_msg")
+	e := presence.Header.Get("error")
+	if e != "" {
+		log.Errorf("received error msg from nats: %s", e)
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: types.PresenceClientResponse{
+				Presence: types.PresenceUnavailable.String(),
+			},
+		}
+	}
+	lastActive, err := strconv.Atoi(presence.Header.Get("last_active_ts"))
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+
+	p := types.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(lastActive)}
+	currentlyActive := p.CurrentlyActive()
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: types.PresenceClientResponse{
+			CurrentlyActive: &currentlyActive,
+			LastActiveAgo:   p.LastActiveAgo(),
+			Presence:        presence.Header.Get("presence"),
+			StatusMsg:       &statusMsg,
+		},
+	}
+}

clientapi/routing/profile.go

@@ -180,7 +180,7 @@ func SetAvatarURL(
 		return jsonerror.InternalServerError()
 	}
 
-	if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, false); err != nil {
+	if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, true); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/register.go

@@ -64,11 +64,6 @@ const (
 	sessionIDLength   = 24
 )
 
-func init() {
-	// Register prometheus metrics. They must be registered to be exposed.
-	prometheus.MustRegister(amtRegUsers)
-}
-
 // sessionsDict keeps track of completed auth stages for each session.
 // It shouldn't be passed by value because it contains a mutex.
 type sessionsDict struct {

clientapi/routing/room_tagging.go

@@ -98,7 +98,7 @@ func PutTag(
 		return jsonerror.InternalServerError()
 	}
 
-	if err = syncProducer.SendData(userID, roomID, "m.tag", nil); err != nil {
+	if err = syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
 		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
 	}
 
@@ -151,7 +151,7 @@ func DeleteTag(
 	}
 
 	// TODO: user API should do this since it's account data
-	if err := syncProducer.SendData(userID, roomID, "m.tag", nil); err != nil {
+	if err := syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
 		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
 	}
 

clientapi/routing/routing.go

@@ -32,9 +32,12 @@ import (
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -56,8 +59,10 @@ func Setup(
 	federationSender federationAPI.FederationInternalAPI,
 	keyAPI keyserverAPI.KeyInternalAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
-	mscCfg *config.MSCs,
+	mscCfg *config.MSCs, natsClient *nats.Conn,
 ) {
+	prometheus.MustRegister(amtRegUsers, sendEventDuration)
+
 	rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
 	userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg)
 
@@ -779,20 +784,6 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	// Element logs get flooded unless this is handled
-	v3mux.Handle("/presence/{userID}/status",
-		httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
-				return *r
-			}
-			// TODO: Set presence (probably the responsibility of a presence server not clientapi)
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: struct{}{},
-			}
-		}),
-	).Methods(http.MethodPut, http.MethodOptions)
-
 	v3mux.Handle("/voip/turnServer",
 		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
@@ -957,6 +948,16 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
+	v3mux.Handle("/rooms/{roomID}/upgrade",
+		httputil.MakeAuthAPI("rooms_upgrade", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return UpgradeRoom(req, device, cfg, vars["roomID"], userAPI, rsAPI, asAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	v3mux.Handle("/devices",
 		httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetDevicesByLocalpart(req, userAPI, device)
@@ -1298,4 +1299,22 @@ func Setup(
 			return SetReceipt(req, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"])
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
+	v3mux.Handle("/presence/{userId}/status",
+		httputil.MakeAuthAPI("set_presence", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SetPresence(req, cfg, device, syncProducer, vars["userId"])
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+	v3mux.Handle("/presence/{userId}/status",
+		httputil.MakeAuthAPI("get_presence", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
 }

clientapi/routing/sendevent.go

@@ -46,10 +46,6 @@ var (
 	userRoomSendMutexes sync.Map // (roomID+userID) -> mutex. mutexes to ensure correct ordering of sendEvents
 )
 
-func init() {
-	prometheus.MustRegister(sendEventDuration)
-}
-
 var sendEventDuration = prometheus.NewHistogramVec(
 	prometheus.HistogramOpts{
 		Namespace: "dendrite",
@@ -272,5 +268,24 @@ func generateSendEvent(
 			JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
 		}
 	}
+
+	// User should not be able to send a tombstone event to the same room.
+	if e.Type() == "m.room.tombstone" {
+		content := make(map[string]interface{})
+		if err = json.Unmarshal(e.Content(), &content); err != nil {
+			util.GetLogger(ctx).WithError(err).Error("Cannot unmarshal the event content.")
+			return nil, &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("Cannot unmarshal the event content."),
+			}
+		}
+		if content["replacement_room"] == e.RoomID() {
+			return nil, &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidParam("Cannot send tombstone event that points to the same room."),
+			}
+		}
+	}
+
 	return e.Event, nil
 }

clientapi/routing/upgrade_room.go

@@ -0,0 +1,92 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/dendrite/setup/config"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type upgradeRoomRequest struct {
+	NewVersion string `json:"new_version"`
+}
+
+type upgradeRoomResponse struct {
+	ReplacementRoom string `json:"replacement_room"`
+}
+
+// UpgradeRoom implements /upgrade
+func UpgradeRoom(
+	req *http.Request, device *userapi.Device,
+	cfg *config.ClientAPI,
+	roomID string, profileAPI userapi.UserProfileAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	var r upgradeRoomRequest
+	if rErr := httputil.UnmarshalJSONRequest(req, &r); rErr != nil {
+		return *rErr
+	}
+
+	// Validate that the room version is supported
+	if _, err := version.SupportedRoomVersion(gomatrixserverlib.RoomVersion(r.NewVersion)); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnsupportedRoomVersion("This server does not support that room version"),
+		}
+	}
+
+	upgradeReq := roomserverAPI.PerformRoomUpgradeRequest{
+		UserID:      device.UserID,
+		RoomID:      roomID,
+		RoomVersion: gomatrixserverlib.RoomVersion(r.NewVersion),
+	}
+	upgradeResp := roomserverAPI.PerformRoomUpgradeResponse{}
+
+	rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp)
+
+	if upgradeResp.Error != nil {
+		if upgradeResp.Error.Code == roomserverAPI.PerformErrorNoRoom {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: jsonerror.NotFound("Room does not exist"),
+			}
+		} else if upgradeResp.Error.Code == roomserverAPI.PerformErrorNotAllowed {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden(upgradeResp.Error.Msg),
+			}
+		} else {
+			return jsonerror.InternalServerError()
+		}
+
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: upgradeRoomResponse{
+			ReplacementRoom: upgradeResp.NewRoomID,
+		},
+	}
+}

clientapi/routing/voip.go

@@ -52,6 +52,7 @@ func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Client
 
 	if turnConfig.SharedSecret != "" {
 		expiry := time.Now().Add(duration).Unix()
+		resp.Username = fmt.Sprintf("%d:%s", expiry, device.UserID)
 		mac := hmac.New(sha1.New, []byte(turnConfig.SharedSecret))
 		_, err := mac.Write([]byte(resp.Username))
 
@@ -60,7 +61,6 @@ func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Client
 			return jsonerror.InternalServerError()
 		}
 
-		resp.Username = fmt.Sprintf("%d:%s", expiry, device.UserID)
 		resp.Password = base64.StdEncoding.EncodeToString(mac.Sum(nil))
 	} else if turnConfig.Username != "" && turnConfig.Password != "" {
 		resp.Username = turnConfig.Username

cmd/client-api-proxy/main.go

@@ -1,156 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"os"
-	"strings"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-)
-
-const usage = `Usage: %s
-
-Create a single endpoint URL which clients can be pointed at.
-
-The client-server API in Dendrite is split across multiple processes
-which listen on multiple ports. You cannot point a Matrix client at
-any of those ports, as there will be unimplemented functionality.
-In addition, all client-server API processes start with the additional
-path prefix '/api', which Matrix clients will be unaware of.
-
-This tool will proxy requests for all client-server URLs and forward
-them to their respective process. It will also add the '/api' path
-prefix to incoming requests.
-
-THIS TOOL IS FOR TESTING AND NOT INTENDED FOR PRODUCTION USE.
-
-Arguments:
-
-`
-
-var (
-	syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'")
-	clientAPIURL  = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'")
-	mediaAPIURL   = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'")
-	bindAddress   = flag.String("bind-address", ":8008", "The listening port for the proxy.")
-	certFile      = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
-	keyFile       = flag.String("tls-key", "", "The PEM private key to use for TLS")
-)
-
-func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
-	targetURL = strings.TrimSuffix(targetURL, "/")
-
-	// Check that we can parse the URL.
-	_, err := url.Parse(targetURL)
-	if err != nil {
-		return nil, err
-	}
-	return &httputil.ReverseProxy{
-		Director: func(req *http.Request) {
-			// URL.Path() removes the % escaping from the path.
-			// The % encoding will be added back when the url is encoded
-			// when the request is forwarded.
-			// This means that we will lose any unessecary escaping from the URL.
-			// Pratically this means that any distinction between '%2F' and '/'
-			// in the URL will be lost by the time it reaches the target.
-			path := req.URL.Path
-			log.WithFields(log.Fields{
-				"path":   path,
-				"url":    targetURL,
-				"method": req.Method,
-			}).Print("proxying request")
-			newURL, err := url.Parse(targetURL)
-			// Set the path separately as we need to preserve '#' characters
-			// that would otherwise be interpreted as being the start of a URL
-			// fragment.
-			newURL.Path += path
-			if err != nil {
-				// We already checked that we can parse the URL
-				// So this shouldn't ever get hit.
-				panic(err)
-			}
-			// Copy the query parameters from the request.
-			newURL.RawQuery = req.URL.RawQuery
-			req.URL = newURL
-		},
-	}, nil
-}
-
-func main() {
-	flag.Usage = func() {
-		fmt.Fprintf(os.Stderr, usage, os.Args[0])
-		flag.PrintDefaults()
-	}
-
-	flag.Parse()
-
-	if *syncServerURL == "" {
-		flag.Usage()
-		fmt.Fprintln(os.Stderr, "no --sync-api-server-url specified.")
-		os.Exit(1)
-	}
-
-	if *clientAPIURL == "" {
-		flag.Usage()
-		fmt.Fprintln(os.Stderr, "no --client-api-server-url specified.")
-		os.Exit(1)
-	}
-
-	if *mediaAPIURL == "" {
-		flag.Usage()
-		fmt.Fprintln(os.Stderr, "no --media-api-server-url specified.")
-		os.Exit(1)
-	}
-	syncProxy, err := makeProxy(*syncServerURL)
-	if err != nil {
-		panic(err)
-	}
-	clientProxy, err := makeProxy(*clientAPIURL)
-	if err != nil {
-		panic(err)
-	}
-	mediaProxy, err := makeProxy(*mediaAPIURL)
-	if err != nil {
-		panic(err)
-	}
-
-	http.Handle("/_matrix/client/r0/sync", syncProxy)
-	http.Handle("/_matrix/media/v1/", mediaProxy)
-	http.Handle("/", clientProxy)
-
-	srv := &http.Server{
-		Addr:         *bindAddress,
-		ReadTimeout:  1 * time.Minute, // how long we wait for the client to send the entire request (after connection accept)
-		WriteTimeout: 5 * time.Minute, // how long the proxy has to write the full response
-	}
-
-	fmt.Println("Proxying requests to:")
-	fmt.Println("  /_matrix/client/r0/sync            => ", *syncServerURL+"/api/_matrix/client/r0/sync")
-	fmt.Println("  /_matrix/media/v1                  => ", *mediaAPIURL+"/api/_matrix/media/v1")
-	fmt.Println("  /*                                 => ", *clientAPIURL+"/api/*")
-	fmt.Println("Listening on ", *bindAddress)
-	if *certFile != "" && *keyFile != "" {
-		panic(srv.ListenAndServeTLS(*certFile, *keyFile))
-	} else {
-		panic(srv.ListenAndServe())
-	}
-}

cmd/dendrite-demo-libp2p/main.go

@@ -1,230 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"crypto/ed25519"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/gorilla/mux"
-	gostream "github.com/libp2p/go-libp2p-gostream"
-	p2phttp "github.com/libp2p/go-libp2p-http"
-	p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery"
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/mscs"
-	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/sirupsen/logrus"
-
-	_ "github.com/mattn/go-sqlite3"
-)
-
-func createKeyDB(
-	base *P2PDendrite,
-	db *gomatrixserverlib.KeyRing,
-) {
-	mdns := mDNSListener{
-		host:  base.LibP2P,
-		keydb: db,
-	}
-	serv, err := p2pdisc.NewMdnsService(
-		base.LibP2PContext,
-		base.LibP2P,
-		time.Second*10,
-		"_matrix-dendrite-p2p._tcp",
-	)
-	if err != nil {
-		panic(err)
-	}
-	serv.RegisterNotifee(&mdns)
-}
-
-func createFederationClient(
-	base *P2PDendrite,
-) *gomatrixserverlib.FederationClient {
-	fmt.Println("Running in libp2p federation mode")
-	fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!")
-	tr := &http.Transport{}
-	tr.RegisterProtocol(
-		"matrix",
-		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
-	)
-	return gomatrixserverlib.NewFederationClient(
-		base.Base.Cfg.Global.ServerName, base.Base.Cfg.Global.KeyID,
-		base.Base.Cfg.Global.PrivateKey,
-		gomatrixserverlib.WithTransport(tr),
-	)
-}
-
-func createClient(
-	base *P2PDendrite,
-) *gomatrixserverlib.Client {
-	tr := &http.Transport{}
-	tr.RegisterProtocol(
-		"matrix",
-		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
-	)
-	return gomatrixserverlib.NewClient(
-		gomatrixserverlib.WithTransport(tr),
-	)
-}
-
-func main() {
-	instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance")
-	instancePort := flag.Int("port", 8080, "the port that the client API will listen on")
-	flag.Parse()
-
-	filename := fmt.Sprintf("%s-private.key", *instanceName)
-	_, err := os.Stat(filename)
-	var privKey ed25519.PrivateKey
-	if os.IsNotExist(err) {
-		_, privKey, _ = ed25519.GenerateKey(nil)
-		if err = ioutil.WriteFile(filename, privKey, 0600); err != nil {
-			fmt.Printf("Couldn't write private key to file '%s': %s\n", filename, err)
-		}
-	} else {
-		privKey, err = ioutil.ReadFile(filename)
-		if err != nil {
-			fmt.Printf("Couldn't read private key from file '%s': %s\n", filename, err)
-			_, privKey, _ = ed25519.GenerateKey(nil)
-		}
-	}
-
-	cfg := config.Dendrite{}
-	cfg.Defaults(true)
-	cfg.Global.ServerName = "p2p"
-	cfg.Global.PrivateKey = privKey
-	cfg.Global.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
-	cfg.FederationAPI.FederationMaxRetries = 6
-	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
-	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
-	cfg.MSCs.MSCs = []string{"msc2836"}
-	cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
-	if err = cfg.Derive(); err != nil {
-		panic(err)
-	}
-
-	base := NewP2PDendrite(&cfg, "Monolith")
-	defer base.Base.Close() // nolint: errcheck
-
-	accountDB := base.Base.CreateAccountsDB()
-	federation := createFederationClient(base)
-	keyAPI := keyserver.NewInternalAPI(&base.Base, &base.Base.Cfg.KeyServer, federation)
-
-	rsAPI := roomserver.NewInternalAPI(
-		&base.Base,
-	)
-
-	userAPI := userapi.NewInternalAPI(&base.Base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.Base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(userAPI)
-
-	asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
-	rsAPI.SetAppserviceAPI(asAPI)
-	fsAPI := federationapi.NewInternalAPI(
-		&base.Base, federation, rsAPI, base.Base.Caches, nil, true,
-	)
-	keyRing := fsAPI.KeyRing()
-	rsAPI.SetFederationAPI(fsAPI, keyRing)
-	provider := newPublicRoomsProvider(base.LibP2PPubsub, rsAPI)
-	err = provider.Start()
-	if err != nil {
-		panic("failed to create new public rooms provider: " + err.Error())
-	}
-
-	createKeyDB(
-		base, keyRing,
-	)
-
-	monolith := setup.Monolith{
-		Config:    base.Base.Cfg,
-		AccountDB: accountDB,
-		Client:    createClient(base),
-		FedClient: federation,
-		KeyRing:   keyRing,
-
-		AppserviceAPI:          asAPI,
-		FederationAPI:          fsAPI,
-		RoomserverAPI:          rsAPI,
-		UserAPI:                userAPI,
-		KeyAPI:                 keyAPI,
-		ExtPublicRoomsProvider: provider,
-	}
-	monolith.AddAllPublicRoutes(
-		base.Base.ProcessContext,
-		base.Base.PublicClientAPIMux,
-		base.Base.PublicFederationAPIMux,
-		base.Base.PublicKeyAPIMux,
-		base.Base.PublicWellKnownAPIMux,
-		base.Base.PublicMediaAPIMux,
-		base.Base.SynapseAdminMux,
-	)
-	if err := mscs.Enable(&base.Base, &monolith); err != nil {
-		logrus.WithError(err).Fatalf("Failed to enable MSCs")
-	}
-
-	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.Base.InternalAPIMux)
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Base.PublicMediaAPIMux)
-	embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
-
-	libp2pRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	libp2pRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.Base.PublicFederationAPIMux)
-	libp2pRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(base.Base.PublicKeyAPIMux)
-	libp2pRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Base.PublicMediaAPIMux)
-
-	// Expose the matrix APIs directly rather than putting them under a /api path.
-	go func() {
-		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
-		logrus.Info("Listening on ", httpBindAddr)
-		logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
-	}()
-	// Expose the matrix APIs also via libp2p
-	if base.LibP2P != nil {
-		go func() {
-			logrus.Info("Listening on libp2p host ID ", base.LibP2P.ID())
-			listener, err := gostream.Listen(base.LibP2P, "/matrix")
-			if err != nil {
-				panic(err)
-			}
-			defer func() {
-				logrus.Fatal(listener.Close())
-			}()
-			logrus.Fatal(http.Serve(listener, libp2pRouter))
-		}()
-	}
-
-	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
-	base.Base.WaitForShutdown()
-}

cmd/dendrite-demo-libp2p/mdnslistener.go

@@ -1,62 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"math"
-
-	"github.com/libp2p/go-libp2p-core/host"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-type mDNSListener struct {
-	keydb *gomatrixserverlib.KeyRing
-	host  host.Host
-}
-
-func (n *mDNSListener) HandlePeerFound(p peer.AddrInfo) {
-	if err := n.host.Connect(context.Background(), p); err != nil {
-		fmt.Println("Error adding peer", p.ID.String(), "via mDNS:", err)
-	}
-	if pubkey, err := p.ID.ExtractPublicKey(); err == nil {
-		raw, _ := pubkey.Raw()
-		if err := n.keydb.KeyDatabase.StoreKeys(
-			context.Background(),
-			map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{
-				{
-					ServerName: gomatrixserverlib.ServerName(p.ID.String()),
-					KeyID:      "ed25519:p2pdemo",
-				}: {
-					VerifyKey: gomatrixserverlib.VerifyKey{
-						Key: gomatrixserverlib.Base64Bytes(raw),
-					},
-					ValidUntilTS: math.MaxUint64 >> 1,
-					ExpiredTS:    gomatrixserverlib.PublicKeyNotExpired,
-				},
-			},
-		); err != nil {
-			fmt.Println("Failed to store keys:", err)
-		}
-	}
-	fmt.Println("Discovered", len(n.host.Peerstore().Peers())-1, "other libp2p peer(s):")
-	for _, peer := range n.host.Peerstore().Peers() {
-		if peer != n.host.ID() {
-			fmt.Println("-", peer)
-		}
-	}
-}

cmd/dendrite-demo-libp2p/p2pdendrite.go

@@ -1,126 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"fmt"
-
-	"errors"
-
-	pstore "github.com/libp2p/go-libp2p-core/peerstore"
-	record "github.com/libp2p/go-libp2p-record"
-
-	"github.com/libp2p/go-libp2p"
-	circuit "github.com/libp2p/go-libp2p-circuit"
-	crypto "github.com/libp2p/go-libp2p-core/crypto"
-	routing "github.com/libp2p/go-libp2p-core/routing"
-
-	host "github.com/libp2p/go-libp2p-core/host"
-	dht "github.com/libp2p/go-libp2p-kad-dht"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-// P2PDendrite is a Peer-to-Peer variant of BaseDendrite.
-type P2PDendrite struct {
-	Base base.BaseDendrite
-
-	// Store our libp2p object so that we can make outgoing connections from it
-	// later
-	LibP2P        host.Host
-	LibP2PContext context.Context
-	LibP2PCancel  context.CancelFunc
-	LibP2PDHT     *dht.IpfsDHT
-	LibP2PPubsub  *pubsub.PubSub
-}
-
-// NewP2PDendrite creates a new instance to be used by a component.
-// The componentName is used for logging purposes, and should be a friendly name
-// of the component running, e.g. SyncAPI.
-func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
-	baseDendrite := base.NewBaseDendrite(cfg, componentName)
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Global.PrivateKey[:])
-	if err != nil {
-		panic(err)
-	}
-
-	//defaultIP6ListenAddr, _ := multiaddr.NewMultiaddr("/ip6/::/tcp/0")
-	var libp2pdht *dht.IpfsDHT
-	libp2p, err := libp2p.New(ctx,
-		libp2p.Identity(privKey),
-		libp2p.DefaultListenAddrs,
-		//libp2p.ListenAddrs(defaultIP6ListenAddr),
-		libp2p.DefaultTransports,
-		libp2p.Routing(func(h host.Host) (r routing.PeerRouting, err error) {
-			libp2pdht, err = dht.New(ctx, h)
-			if err != nil {
-				return nil, err
-			}
-			libp2pdht.Validator = libP2PValidator{}
-			r = libp2pdht
-			return
-		}),
-		libp2p.EnableAutoRelay(),
-		libp2p.EnableRelay(circuit.OptHop),
-	)
-	if err != nil {
-		panic(err)
-	}
-
-	libp2ppubsub, err := pubsub.NewFloodSub(context.Background(), libp2p, []pubsub.Option{
-		pubsub.WithMessageSigning(true),
-	}...)
-	if err != nil {
-		panic(err)
-	}
-
-	fmt.Println("Our public key:", privKey.GetPublic())
-	fmt.Println("Our node ID:", libp2p.ID())
-	fmt.Println("Our addresses:", libp2p.Addrs())
-
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String())
-
-	return &P2PDendrite{
-		Base:          *baseDendrite,
-		LibP2P:        libp2p,
-		LibP2PContext: ctx,
-		LibP2PCancel:  cancel,
-		LibP2PDHT:     libp2pdht,
-		LibP2PPubsub:  libp2ppubsub,
-	}
-}
-
-type libP2PValidator struct {
-	KeyBook pstore.KeyBook
-}
-
-func (v libP2PValidator) Validate(key string, value []byte) error {
-	ns, _, err := record.SplitKey(key)
-	if err != nil || ns != "matrix" {
-		return errors.New("not Matrix path")
-	}
-	return nil
-}
-
-func (v libP2PValidator) Select(k string, vals [][]byte) (int, error) {
-	return 0, nil
-}

cmd/dendrite-demo-libp2p/publicrooms.go

@@ -1,153 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-const MaintenanceInterval = time.Second * 10
-
-type discoveredRoom struct {
-	time time.Time
-	room gomatrixserverlib.PublicRoom
-}
-
-type publicRoomsProvider struct {
-	pubsub           *pubsub.PubSub
-	topic            *pubsub.Topic
-	subscription     *pubsub.Subscription
-	foundRooms       map[string]discoveredRoom // additional rooms we have learned about from the DHT
-	foundRoomsMutex  sync.RWMutex              // protects foundRooms
-	maintenanceTimer *time.Timer               //
-	roomsAdvertised  atomic.Value              // stores int
-	rsAPI            roomserverAPI.RoomserverInternalAPI
-}
-
-func newPublicRoomsProvider(ps *pubsub.PubSub, rsAPI roomserverAPI.RoomserverInternalAPI) *publicRoomsProvider {
-	return &publicRoomsProvider{
-		foundRooms: make(map[string]discoveredRoom),
-		pubsub:     ps,
-		rsAPI:      rsAPI,
-	}
-}
-
-func (p *publicRoomsProvider) Start() error {
-	if topic, err := p.pubsub.Join("/matrix/publicRooms"); err != nil {
-		return err
-	} else if sub, err := topic.Subscribe(); err == nil {
-		p.topic = topic
-		p.subscription = sub
-		go p.MaintenanceTimer()
-		go p.FindRooms()
-		p.roomsAdvertised.Store(0)
-	} else {
-		return err
-	}
-	return nil
-}
-
-func (p *publicRoomsProvider) MaintenanceTimer() {
-	if p.maintenanceTimer != nil && !p.maintenanceTimer.Stop() {
-		<-p.maintenanceTimer.C
-	}
-	p.Interval()
-}
-
-func (p *publicRoomsProvider) Interval() {
-	p.foundRoomsMutex.Lock()
-	for k, v := range p.foundRooms {
-		if time.Since(v.time) > time.Minute {
-			delete(p.foundRooms, k)
-		}
-	}
-	p.foundRoomsMutex.Unlock()
-	if err := p.AdvertiseRooms(); err != nil {
-		fmt.Println("Failed to advertise room in DHT:", err)
-	}
-	p.foundRoomsMutex.RLock()
-	defer p.foundRoomsMutex.RUnlock()
-	fmt.Println("Found", len(p.foundRooms), "room(s), advertised", p.roomsAdvertised.Load(), "room(s)")
-	p.maintenanceTimer = time.AfterFunc(MaintenanceInterval, p.Interval)
-}
-
-func (p *publicRoomsProvider) AdvertiseRooms() error {
-	ctx := context.Background()
-	var queryRes roomserverAPI.QueryPublishedRoomsResponse
-	// Query published rooms on our server. This will not invoke clientapi.ExtraPublicRoomsProvider
-	err := p.rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
-		return err
-	}
-	ourRooms, err := roomserverAPI.PopulatePublicRooms(ctx, queryRes.RoomIDs, p.rsAPI)
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("PopulatePublicRooms failed")
-		return err
-	}
-	advertised := 0
-	for _, room := range ourRooms {
-		if j, err := json.Marshal(room); err == nil {
-			if err := p.topic.Publish(context.TODO(), j); err != nil {
-				fmt.Println("Failed to publish public room:", err)
-			} else {
-				advertised++
-			}
-		}
-	}
-
-	p.roomsAdvertised.Store(advertised)
-	return nil
-}
-
-func (p *publicRoomsProvider) FindRooms() {
-	for {
-		msg, err := p.subscription.Next(context.Background())
-		if err != nil {
-			continue
-		}
-		received := discoveredRoom{
-			time: time.Now(),
-		}
-		if err := json.Unmarshal(msg.Data, &received.room); err != nil {
-			fmt.Println("Unmarshal error:", err)
-			continue
-		}
-		fmt.Printf("received %+v \n", received)
-		p.foundRoomsMutex.Lock()
-		p.foundRooms[received.room.RoomID] = received
-		p.foundRoomsMutex.Unlock()
-	}
-}
-
-func (p *publicRoomsProvider) Rooms() (rooms []gomatrixserverlib.PublicRoom) {
-	p.foundRoomsMutex.RLock()
-	defer p.foundRoomsMutex.RUnlock()
-	for _, dr := range p.foundRooms {
-		rooms = append(rooms, dr.room)
-	}
-	return
-}

cmd/dendrite-demo-pinecone/conn/client.go

@@ -67,21 +67,22 @@ func (y *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 }
 
 func createTransport(s *pineconeSessions.Sessions) *http.Transport {
+	proto := s.Protocol("matrix")
 	tr := &http.Transport{
 		DisableKeepAlives: false,
-		Dial:              s.Dial,
-		DialContext:       s.DialContext,
-		DialTLS:           s.DialTLS,
-		DialTLSContext:    s.DialTLSContext,
+		Dial:              proto.Dial,
+		DialContext:       proto.DialContext,
+		DialTLS:           proto.DialTLS,
+		DialTLSContext:    proto.DialTLSContext,
 	}
 	tr.RegisterProtocol(
 		"matrix", &RoundTripper{
 			inner: &http.Transport{
 				DisableKeepAlives: false,
-				Dial:              s.Dial,
-				DialContext:       s.DialContext,
-				DialTLS:           s.DialTLS,
-				DialTLSContext:    s.DialTLSContext,
+				Dial:              proto.Dial,
+				DialContext:       proto.DialContext,
+				DialTLS:           proto.DialTLS,
+				DialTLSContext:    proto.DialTLSContext,
 			},
 		},
 	)

cmd/dendrite-demo-pinecone/main.go

@@ -22,11 +22,9 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -38,7 +36,6 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver"
@@ -49,6 +46,7 @@ import (
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
+	pineconeConnections "github.com/matrix-org/pinecone/connections"
 	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
 	pineconeRouter "github.com/matrix-org/pinecone/router"
 	pineconeSessions "github.com/matrix-org/pinecone/sessions"
@@ -91,8 +89,14 @@ func main() {
 		pk = sk.Public().(ed25519.PublicKey)
 	}
 
-	logger := log.New(os.Stdout, "", 0)
-	pRouter := pineconeRouter.NewRouter(logger, sk, false)
+	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
+	pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
+	pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
+	pManager := pineconeConnections.NewConnectionManager(pRouter)
+	pMulticast.Start()
+	if instancePeer != nil && *instancePeer != "" {
+		pManager.AddPeer(*instancePeer)
+	}
 
 	go func() {
 		listener, err := net.Listen("tcp", *instanceListen)
@@ -122,36 +126,6 @@ func main() {
 		}
 	}()
 
-	pQUIC := pineconeSessions.NewSessions(logger, pRouter)
-	pMulticast := pineconeMulticast.NewMulticast(logger, pRouter)
-	pMulticast.Start()
-
-	connectToStaticPeer := func() {
-		connected := map[string]bool{} // URI -> connected?
-		for _, uri := range strings.Split(*instancePeer, ",") {
-			connected[strings.TrimSpace(uri)] = false
-		}
-		attempt := func() {
-			for k := range connected {
-				connected[k] = false
-			}
-			for _, info := range pRouter.Peers() {
-				connected[info.URI] = true
-			}
-			for k, online := range connected {
-				if !online {
-					if err := conn.ConnectToPeer(pRouter, k); err != nil {
-						logrus.WithError(err).Error("Failed to connect to static peer")
-					}
-				}
-			}
-		}
-		for {
-			attempt()
-			time.Sleep(time.Second * 5)
-		}
-	}
-
 	cfg := &config.Dendrite{}
 	cfg.Defaults(true)
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
@@ -253,7 +227,7 @@ func main() {
 	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
 	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 
-	pHTTP := pQUIC.HTTP()
+	pHTTP := pQUIC.Protocol("matrix").HTTP()
 	pHTTP.Mux().Handle(users.PublicURL, pMux)
 	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
 	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
@@ -271,25 +245,16 @@ func main() {
 		Handler: pMux,
 	}
 
-	go connectToStaticPeer()
 	go func() {
 		pubkey := pRouter.PublicKey()
 		logrus.Info("Listening on ", hex.EncodeToString(pubkey[:]))
-		logrus.Fatal(httpServer.Serve(pQUIC))
+		logrus.Fatal(httpServer.Serve(pQUIC.Protocol("matrix")))
 	}()
 	go func() {
 		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
 		logrus.Info("Listening on ", httpBindAddr)
 		logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
 	}()
-	go func() {
-		logrus.Info("Sending wake-up message to known nodes")
-		req := &api.PerformBroadcastEDURequest{}
-		res := &api.PerformBroadcastEDUResponse{}
-		if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil {
-			logrus.WithError(err).Error("Failed to send wake-up message to known nodes")
-		}
-	}()
 
 	base.WaitForShutdown()
 }

cmd/dendritejs-pinecone/main.go

@@ -21,10 +21,7 @@ import (
 	"crypto/ed25519"
 	"encoding/hex"
 	"fmt"
-	"log"
-	"os"
 	"syscall/js"
-	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
@@ -46,6 +43,7 @@ import (
 
 	_ "github.com/matrix-org/go-sqlite3-js"
 
+	pineconeConnections "github.com/matrix-org/pinecone/connections"
 	pineconeRouter "github.com/matrix-org/pinecone/router"
 	pineconeSessions "github.com/matrix-org/pinecone/sessions"
 )
@@ -154,9 +152,10 @@ func startup() {
 	sk := generateKey()
 	pk := sk.Public().(ed25519.PublicKey)
 
-	logger := log.New(os.Stdout, "", 0)
-	pRouter := pineconeRouter.NewRouter(logger, sk, false)
-	pSessions := pineconeSessions.NewSessions(logger, pRouter)
+	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
+	pSessions := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
+	pManager := pineconeConnections.NewConnectionManager(pRouter)
+	pManager.AddPeer("wss://pinecone.matrix.org/public")
 
 	cfg := &config.Dendrite{}
 	cfg.Defaults(true)
@@ -228,7 +227,7 @@ func startup() {
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 
-	p2pRouter := pSessions.HTTP().Mux()
+	p2pRouter := pSessions.Protocol("matrix").HTTP().Mux()
 	p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux)
 	p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux)
 
@@ -240,20 +239,4 @@ func startup() {
 		}
 		s.ListenAndServe("fetch")
 	}()
-
-	// Connect to the static peer
-	go func() {
-		for {
-			if pRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 {
-				if err := conn.ConnectToPeer(pRouter, publicPeer); err != nil {
-					logrus.WithError(err).Error("Failed to connect to static peer")
-				}
-			}
-			select {
-			case <-base.ProcessContext.Context().Done():
-				return
-			case <-time.After(time.Second * 5):
-			}
-		}
-	}()
 }

cmd/dendritejs/jsServer.go

@@ -1,101 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"bufio"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"syscall/js"
-)
-
-// JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it.
-type JSServer struct {
-	// The router which will service requests
-	Mux http.Handler
-}
-
-// OnRequestFromJS is the function that JS will invoke when there is a new request.
-// The JS function signature is:
-//   function(reqString: string): Promise<{result: string, error: string}>
-// Usage is like:
-//   const res = await global._go_js_server.fetch(reqString);
-//   if (res.error) {
-//     // handle error: this is a 'network' error, not a non-2xx error.
-//   }
-//   const rawHttpResponse = res.result;
-func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
-	// we HAVE to spawn a new goroutine and return immediately or else Go will deadlock
-	// if this request blocks at all e.g for /sync calls
-	httpStr := args[0].String()
-	promise := js.Global().Get("Promise").New(js.FuncOf(func(pthis js.Value, pargs []js.Value) interface{} {
-		// The initial callback code for new Promise() is also called on the critical path, which is why
-		// we need to put this in an immediately invoked goroutine.
-		go func() {
-			resolve := pargs[0]
-			resStr, err := h.handle(httpStr)
-			errStr := ""
-			if err != nil {
-				errStr = err.Error()
-			}
-			resolve.Invoke(map[string]interface{}{
-				"result": resStr,
-				"error":  errStr,
-			})
-		}()
-		return nil
-	}))
-	return promise
-}
-
-// handle invokes the http.ServeMux for this request and returns the raw HTTP response.
-func (h *JSServer) handle(httpStr string) (resStr string, err error) {
-	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(httpStr)))
-	if err != nil {
-		return
-	}
-	w := httptest.NewRecorder()
-
-	h.Mux.ServeHTTP(w, req)
-
-	res := w.Result()
-	var resBuffer strings.Builder
-	err = res.Write(&resBuffer)
-	return resBuffer.String(), err
-}
-
-// ListenAndServe registers a variable in JS-land with the given namespace. This variable is
-// a function which JS-land can call to 'send' HTTP requests. The function is attached to
-// a global object called "_go_js_server". See OnRequestFromJS for more info.
-func (h *JSServer) ListenAndServe(namespace string) {
-	globalName := "_go_js_server"
-	// register a hook in JS-land for it to invoke stuff
-	server := js.Global().Get(globalName)
-	if !server.Truthy() {
-		server = js.Global().Get("Object").New()
-		js.Global().Set(globalName, server)
-	}
-
-	server.Set(namespace, js.FuncOf(h.OnRequestFromJS))
-
-	fmt.Printf("Listening for requests from JS on function %s.%s\n", globalName, namespace)
-	// Block forever to mimic http.ListenAndServe
-	select {}
-}

cmd/dendritejs/keyfetcher.go

@@ -1,87 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-const libp2pMatrixKeyID = "ed25519:libp2p-dendrite"
-
-type libp2pKeyFetcher struct {
-}
-
-// FetchKeys looks up a batch of public keys.
-// Takes a map from (server name, key ID) pairs to timestamp.
-// The timestamp is when the keys need to be vaild up to.
-// Returns a map from (server name, key ID) pairs to server key objects for
-// that server name containing that key ID
-// The result may have fewer (server name, key ID) pairs than were in the request.
-// The result may have more (server name, key ID) pairs than were in the request.
-// Returns an error if there was a problem fetching the keys.
-func (f *libp2pKeyFetcher) FetchKeys(
-	ctx context.Context,
-	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
-) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) {
-	res := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult)
-	for req := range requests {
-		if req.KeyID != libp2pMatrixKeyID {
-			return nil, fmt.Errorf("FetchKeys: cannot fetch key with ID %s, should be %s", req.KeyID, libp2pMatrixKeyID)
-		}
-
-		// The server name is a libp2p peer ID
-		peerIDStr := string(req.ServerName)
-		peerID, err := peer.Decode(peerIDStr)
-		if err != nil {
-			return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %w", peerIDStr, err)
-		}
-		pubKey, err := peerID.ExtractPublicKey()
-		if err != nil {
-			return nil, fmt.Errorf("Failed to extract public key from peer ID: %w", err)
-		}
-		pubKeyBytes, err := pubKey.Raw()
-		if err != nil {
-			return nil, fmt.Errorf("Failed to extract raw bytes from public key: %w", err)
-		}
-		b64Key := gomatrixserverlib.Base64Bytes(pubKeyBytes)
-		res[req] = gomatrixserverlib.PublicKeyLookupResult{
-			VerifyKey: gomatrixserverlib.VerifyKey{
-				Key: b64Key,
-			},
-			ExpiredTS:    gomatrixserverlib.PublicKeyNotExpired,
-			ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(24 * time.Hour * 365)),
-		}
-	}
-	return res, nil
-}
-
-// FetcherName returns the name of this fetcher, which can then be used for
-// logging errors etc.
-func (f *libp2pKeyFetcher) FetcherName() string {
-	return "libp2pKeyFetcher"
-}
-
-// no-op function for storing keys - we don't do any work to fetch them so don't bother storing.
-func (f *libp2pKeyFetcher) StoreKeys(ctx context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error {
-	return nil
-}

cmd/dendritejs/main.go

@@ -1,270 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"crypto/ed25519"
-	"fmt"
-	"syscall/js"
-
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi"
-	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
-
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/sirupsen/logrus"
-
-	_ "github.com/matrix-org/go-sqlite3-js"
-)
-
-var GitCommit string
-
-func init() {
-	fmt.Printf("[%s] dendrite.js starting...\n", GitCommit)
-}
-
-const keyNameEd25519 = "_go_ed25519_key"
-
-func readKeyFromLocalStorage() (key ed25519.PrivateKey, err error) {
-	localforage := js.Global().Get("localforage")
-	if !localforage.Truthy() {
-		err = fmt.Errorf("readKeyFromLocalStorage: no localforage")
-		return
-	}
-	// https://localforage.github.io/localForage/
-	item, ok := await(localforage.Call("getItem", keyNameEd25519))
-	if !ok || !item.Truthy() {
-		err = fmt.Errorf("readKeyFromLocalStorage: no key in localforage")
-		return
-	}
-	fmt.Println("Found key in localforage")
-	// extract []byte and make an ed25519 key
-	seed := make([]byte, 32, 32)
-	js.CopyBytesToGo(seed, item)
-
-	return ed25519.NewKeyFromSeed(seed), nil
-}
-
-func writeKeyToLocalStorage(key ed25519.PrivateKey) error {
-	localforage := js.Global().Get("localforage")
-	if !localforage.Truthy() {
-		return fmt.Errorf("writeKeyToLocalStorage: no localforage")
-	}
-
-	// make a Uint8Array from the key's seed
-	seed := key.Seed()
-	jsSeed := js.Global().Get("Uint8Array").New(len(seed))
-	js.CopyBytesToJS(jsSeed, seed)
-	// write it
-	localforage.Call("setItem", keyNameEd25519, jsSeed)
-	return nil
-}
-
-// taken from https://go-review.googlesource.com/c/go/+/150917
-
-// await waits until the promise v has been resolved or rejected and returns the promise's result value.
-// The boolean value ok is true if the promise has been resolved, false if it has been rejected.
-// If v is not a promise, v itself is returned as the value and ok is true.
-func await(v js.Value) (result js.Value, ok bool) {
-	if v.Type() != js.TypeObject || v.Get("then").Type() != js.TypeFunction {
-		return v, true
-	}
-	done := make(chan struct{})
-	onResolve := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
-		result = args[0]
-		ok = true
-		close(done)
-		return nil
-	})
-	defer onResolve.Release()
-	onReject := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
-		result = args[0]
-		ok = false
-		close(done)
-		return nil
-	})
-	defer onReject.Release()
-	v.Call("then", onResolve, onReject)
-	<-done
-	return
-}
-
-func generateKey() ed25519.PrivateKey {
-	// attempt to look for a seed in JS-land and if it exists use it.
-	priv, err := readKeyFromLocalStorage()
-	if err == nil {
-		fmt.Println("Read key from localStorage")
-		return priv
-	}
-	// generate a new key
-	fmt.Println(err, " : Generating new ed25519 key")
-	_, priv, err = ed25519.GenerateKey(nil)
-	if err != nil {
-		logrus.Fatalf("Failed to generate ed25519 key: %s", err)
-	}
-	if err := writeKeyToLocalStorage(priv); err != nil {
-		fmt.Println("failed to write key to localStorage: ", err)
-		// non-fatal, we'll just have amnesia for a while
-	}
-	return priv
-}
-
-func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.FederationClient {
-	fmt.Println("Running in js-libp2p federation mode")
-	fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!")
-	tr := go_http_js_libp2p.NewP2pTransport(node)
-
-	fed := gomatrixserverlib.NewFederationClient(
-		cfg.Global.ServerName, cfg.Global.KeyID, cfg.Global.PrivateKey,
-		gomatrixserverlib.WithTransport(tr),
-	)
-
-	return fed
-}
-
-func createClient(node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.Client {
-	tr := go_http_js_libp2p.NewP2pTransport(node)
-	return gomatrixserverlib.NewClient(
-		gomatrixserverlib.WithTransport(tr),
-	)
-}
-
-func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) {
-	hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/"
-	node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}, "p2p")
-	serverName = node.Id
-	fmt.Println("p2p assigned ServerName: ", serverName)
-	return
-}
-
-func main() {
-	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
-	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
-	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
-	cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
-	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
-	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
-	cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db"
-	cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db"
-	cfg.Global.JetStream.StoragePath = "file:/idb/dendritejs/"
-	cfg.Global.TrustedIDServers = []string{
-		"matrix.org", "vector.im",
-	}
-	cfg.Global.KeyID = libp2pMatrixKeyID
-	cfg.Global.PrivateKey = generateKey()
-
-	serverName, node := createP2PNode(cfg.Global.PrivateKey)
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(serverName)
-
-	if err := cfg.Derive(); err != nil {
-		logrus.Fatalf("Failed to derive values from config: %s", err)
-	}
-	base := setup.NewBaseDendrite(cfg, "Monolith")
-	defer base.Close() // nolint: errcheck
-
-	accountDB := base.CreateAccountsDB()
-	federation := createFederationClient(cfg, node)
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
-	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
-	keyAPI.SetUserAPI(userAPI)
-
-	fetcher := &libp2pKeyFetcher{}
-	keyRing := gomatrixserverlib.KeyRing{
-		KeyFetchers: []gomatrixserverlib.KeyFetcher{
-			fetcher,
-		},
-		KeyDatabase: fetcher,
-	}
-
-	rsAPI := roomserver.NewInternalAPI(base)
-	asQuery := appservice.NewInternalAPI(
-		base, userAPI, rsAPI,
-	)
-	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, keyRing, true)
-	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
-	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation)
-
-	psAPI := pushserver.NewInternalAPI(base)
-
-	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		AccountDB: accountDB,
-		Client:    createClient(node),
-		FedClient: federation,
-		KeyRing:   &keyRing,
-
-		AppserviceAPI:       asQuery,
-		FederationSenderAPI: fedSenderAPI,
-		RoomserverAPI:       rsAPI,
-		UserAPI:             userAPI,
-		KeyAPI:              keyAPI,
-		PushserverAPI:       psAPI,
-		//ServerKeyAPI:        serverKeyAPI,
-		ExtPublicRoomsProvider: p2pPublicRoomProvider,
-	}
-	monolith.AddAllPublicRoutes(
-		base.ProcessContext,
-		base.PublicClientAPIMux,
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicMediaAPIMux,
-		base.SynapseAdminMux,
-	)
-
-	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-
-	libp2pRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	libp2pRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
-	libp2pRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-
-	// Expose the matrix APIs via libp2p-js - for federation traffic
-	if node != nil {
-		go func() {
-			logrus.Info("Listening on libp2p-js host ID ", node.Id)
-			s := JSServer{
-				Mux: libp2pRouter,
-			}
-			s.ListenAndServe("p2p")
-		}()
-	}
-
-	// Expose the matrix APIs via fetch - for local traffic
-	go func() {
-		logrus.Info("Listening for service-worker fetch traffic")
-		s := JSServer{
-			Mux: httpRouter,
-		}
-		s.ListenAndServe("fetch")
-	}()
-
-	// We want to block forever to let the fetch and libp2p handler serve the APIs
-	select {}
-}

cmd/dendritejs/publicrooms.go

@@ -1,155 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"context"
-	"sync"
-	"time"
-
-	"github.com/matrix-org/dendrite/federationapi/api"
-	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-type libp2pPublicRoomsProvider struct {
-	node      *go_http_js_libp2p.P2pLocalNode
-	providers []go_http_js_libp2p.PeerInfo
-	fedSender api.FederationInternalAPI
-	fedClient *gomatrixserverlib.FederationClient
-}
-
-func NewLibP2PPublicRoomsProvider(
-	node *go_http_js_libp2p.P2pLocalNode, fedSender api.FederationInternalAPI, fedClient *gomatrixserverlib.FederationClient,
-) *libp2pPublicRoomsProvider {
-	p := &libp2pPublicRoomsProvider{
-		node:      node,
-		fedSender: fedSender,
-		fedClient: fedClient,
-	}
-	node.RegisterFoundProviders(p.foundProviders)
-	return p
-}
-
-func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p.PeerInfo) {
-	// work out the diff then poke for new ones
-	seen := make(map[string]bool, len(p.providers))
-	for _, pr := range p.providers {
-		seen[pr.Id] = true
-	}
-	var newPeers []gomatrixserverlib.ServerName
-	for _, pi := range peerInfos {
-		if !seen[pi.Id] {
-			newPeers = append(newPeers, gomatrixserverlib.ServerName(pi.Id))
-		}
-	}
-	if len(newPeers) > 0 {
-		var res api.PerformServersAliveResponse
-		// ignore errors, we don't care.
-		p.fedSender.PerformServersAlive(context.Background(), &api.PerformServersAliveRequest{
-			Servers: newPeers,
-		}, &res)
-	}
-
-	p.providers = peerInfos
-}
-
-func (p *libp2pPublicRoomsProvider) Rooms() []gomatrixserverlib.PublicRoom {
-	return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, p.homeservers())
-}
-
-func (p *libp2pPublicRoomsProvider) homeservers() []string {
-	result := make([]string, len(p.providers))
-	for i := range p.providers {
-		result[i] = p.providers[i].Id
-	}
-	return result
-}
-
-// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
-// Returns a list of public rooms.
-func bulkFetchPublicRoomsFromServers(
-	ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string,
-) (publicRooms []gomatrixserverlib.PublicRoom) {
-	limit := 200
-	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
-	// goroutines send rooms to this channel
-	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
-	// signalling channel to tell goroutines to stop sending rooms and quit
-	done := make(chan bool)
-	// signalling to say when we can close the room channel
-	var wg sync.WaitGroup
-	wg.Add(len(homeservers))
-	// concurrently query for public rooms
-	for _, hs := range homeservers {
-		go func(homeserverDomain string) {
-			defer wg.Done()
-			util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
-			fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "")
-			if err != nil {
-				util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn(
-					"bulkFetchPublicRoomsFromServers: failed to query hs",
-				)
-				return
-			}
-			for _, room := range fres.Chunk {
-				// atomically send a room or stop
-				select {
-				case roomCh <- room:
-				case <-done:
-					util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms")
-					return
-				}
-			}
-		}(hs)
-	}
-
-	// Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request.
-	// This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be
-	// closed.
-	go func() {
-		wg.Wait()
-		util.GetLogger(ctx).Info("Cleaning up resources")
-		close(roomCh)
-	}()
-
-	// fan-in results with timeout. We stop when we reach the limit.
-FanIn:
-	for len(publicRooms) < int(limit) || limit == 0 {
-		// add a room or timeout
-		select {
-		case room, ok := <-roomCh:
-			if !ok {
-				util.GetLogger(ctx).Info("All homeservers have been queried, returning results.")
-				break FanIn
-			}
-			publicRooms = append(publicRooms, room)
-		case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got.
-			util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early")
-			break FanIn
-		case <-ctx.Done(): // the client hung up on us, let's stop.
-			util.GetLogger(ctx).Info("Client hung up, returning early")
-			break FanIn
-		}
-	}
-	// tell goroutines to stop
-	close(done)
-
-	return publicRooms
-}

cmd/federation-api-proxy/main.go

@@ -1,138 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"os"
-	"strings"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-)
-
-const usage = `Usage: %s
-
-Create a single endpoint URL which remote matrix servers can be pointed at.
-
-The server-server API in Dendrite is split across multiple processes
-which listen on multiple ports. You cannot point a Matrix server at
-any of those ports, as there will be unimplemented functionality.
-In addition, all server-server API processes start with the additional
-path prefix '/api', which Matrix servers will be unaware of.
-
-This tool will proxy requests for all server-server URLs and forward
-them to their respective process. It will also add the '/api' path
-prefix to incoming requests.
-
-THIS TOOL IS FOR TESTING AND NOT INTENDED FOR PRODUCTION USE.
-
-Arguments:
-
-`
-
-var (
-	federationAPIURL = flag.String("federation-api-url", "", "The base URL of the listening 'dendrite-federation-api-server' process. E.g. 'http://localhost:4200'")
-	mediaAPIURL      = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'")
-	bindAddress      = flag.String("bind-address", ":8448", "The listening port for the proxy.")
-	certFile         = flag.String("tls-cert", "server.crt", "The PEM formatted X509 certificate to use for TLS")
-	keyFile          = flag.String("tls-key", "server.key", "The PEM private key to use for TLS")
-)
-
-func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
-	if !strings.HasSuffix(targetURL, "/") {
-		targetURL += "/"
-	}
-	// Check that we can parse the URL.
-	_, err := url.Parse(targetURL)
-	if err != nil {
-		return nil, err
-	}
-	return &httputil.ReverseProxy{
-		Director: func(req *http.Request) {
-			// URL.Path() removes the % escaping from the path.
-			// The % encoding will be added back when the url is encoded
-			// when the request is forwarded.
-			// This means that we will lose any unessecary escaping from the URL.
-			// Pratically this means that any distinction between '%2F' and '/'
-			// in the URL will be lost by the time it reaches the target.
-			path := req.URL.Path
-			log.WithFields(log.Fields{
-				"path":   path,
-				"url":    targetURL,
-				"method": req.Method,
-			}).Print("proxying request")
-			newURL, err := url.Parse(targetURL + path)
-			if err != nil {
-				// We already checked that we can parse the URL
-				// So this shouldn't ever get hit.
-				panic(err)
-			}
-			// Copy the query parameters from the request.
-			newURL.RawQuery = req.URL.RawQuery
-			req.URL = newURL
-		},
-	}, nil
-}
-
-func main() {
-	flag.Usage = func() {
-		fmt.Fprintf(os.Stderr, usage, os.Args[0])
-		flag.PrintDefaults()
-	}
-
-	flag.Parse()
-
-	if *federationAPIURL == "" {
-		flag.Usage()
-		fmt.Fprintln(os.Stderr, "no --federation-api-url specified.")
-		os.Exit(1)
-	}
-
-	if *mediaAPIURL == "" {
-		flag.Usage()
-		fmt.Fprintln(os.Stderr, "no --media-api-server-url specified.")
-		os.Exit(1)
-	}
-
-	federationProxy, err := makeProxy(*federationAPIURL)
-	if err != nil {
-		panic(err)
-	}
-
-	mediaProxy, err := makeProxy(*mediaAPIURL)
-	if err != nil {
-		panic(err)
-	}
-
-	http.Handle("/_matrix/media/v1/", mediaProxy)
-	http.Handle("/", federationProxy)
-
-	srv := &http.Server{
-		Addr:         *bindAddress,
-		ReadTimeout:  1 * time.Minute, // how long we wait for the client to send the entire request (after connection accept)
-		WriteTimeout: 5 * time.Minute, // how long the proxy has to write the full response
-	}
-
-	fmt.Println("Proxying requests to:")
-	fmt.Println("  /_matrix/media/v1        => ", *mediaAPIURL+"/api/_matrix/media/v1")
-	fmt.Println("  /*                       => ", *federationAPIURL+"/api/*")
-	fmt.Println("Listening on ", *bindAddress)
-	panic(srv.ListenAndServeTLS(*certFile, *keyFile))
-}

cmd/generate-config/main.go

@@ -91,6 +91,10 @@ func main() {
 		cfg.UserAPI.BCryptCost = bcrypt.MinCost
 		cfg.Global.JetStream.InMemory = true
 		cfg.ClientAPI.RegistrationSharedSecret = "complement"
+		cfg.Global.Presence = config.PresenceOptions{
+			EnableInbound:  true,
+			EnableOutbound: true,
+		}
 	}
 
 	j, err := yaml.Marshal(cfg)

dendrite-config.yaml

@@ -68,6 +68,13 @@ global:
   # to other servers and the federation API will not be exposed.
   disable_federation: false
 
+  # Configures the handling of presence events.
+  presence:
+    # Whether inbound presence events are allowed, e.g. receiving presence events from other servers
+    enable_inbound: false
+    # Whether outbound presence events are allowed, e.g. sending presence events to other servers
+    enable_outbound: false
+
   # Server notices allows server admins to send messages to all users.
   server_notices:
     enabled: false
@@ -200,12 +207,6 @@ federation_api:
     max_idle_conns: 2
     conn_max_lifetime: -1
 
-  # List of paths to X.509 certificates to be used by the external federation listeners.
-  # These certificates will be used to calculate the TLS fingerprints and other servers
-  # will expect the certificate to match these fingerprints. Certificates must be in PEM
-  # format.
-  federation_certificates: []
-
   # How many times we will try to resend a failed transaction to a specific server. The
   # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc.
   send_max_retries: 16

docs/INSTALL.md

@@ -34,6 +34,10 @@ If you want to run a polylith deployment, you also need:
 
 * A standalone [NATS Server](https://github.com/nats-io/nats-server) deployment with JetStream enabled
 
+If you want to build it on Windows, you need `gcc` in the path:
+
+* [MinGW-w64](https://www.mingw-w64.org/)
+
 ## Building Dendrite
 
 Start by cloning the code:
@@ -45,9 +49,15 @@ cd dendrite
 
 Then build it:
 
-```bash
-./build.sh
-```
+* Linux or UNIX-like systems:
+  ```bash
+  ./build.sh
+  ```
+
+* Windows:
+  ```dos
+  build.cmd
+  ```
 
 ## Install NATS Server
 

federationapi/consumers/presence.go

@@ -0,0 +1,143 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/storage"
+	fedTypes "github.com/matrix-org/dendrite/federationapi/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+)
+
+// OutputReceiptConsumer consumes events that originate in the clientapi.
+type OutputPresenceConsumer struct {
+	ctx                     context.Context
+	jetstream               nats.JetStreamContext
+	durable                 string
+	db                      storage.Database
+	queues                  *queue.OutgoingQueues
+	ServerName              gomatrixserverlib.ServerName
+	topic                   string
+	outboundPresenceEnabled bool
+}
+
+// NewOutputPresenceConsumer creates a new OutputPresenceConsumer. Call Start() to begin consuming events.
+func NewOutputPresenceConsumer(
+	process *process.ProcessContext,
+	cfg *config.FederationAPI,
+	js nats.JetStreamContext,
+	queues *queue.OutgoingQueues,
+	store storage.Database,
+) *OutputPresenceConsumer {
+	return &OutputPresenceConsumer{
+		ctx:                     process.Context(),
+		jetstream:               js,
+		queues:                  queues,
+		db:                      store,
+		ServerName:              cfg.Matrix.ServerName,
+		durable:                 cfg.Matrix.JetStream.Durable("FederationAPIPresenceConsumer"),
+		topic:                   cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		outboundPresenceEnabled: cfg.Matrix.Presence.EnableOutbound,
+	}
+}
+
+// Start consuming from the clientapi
+func (t *OutputPresenceConsumer) Start() error {
+	if !t.outboundPresenceEnabled {
+		return nil
+	}
+	return jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(),
+	)
+}
+
+// onMessage is called in response to a message received on the presence
+// events topic from the client api.
+func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+	// only send presence events which originated from us
+	userID := msg.Header.Get(jetstream.UserID)
+	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		log.WithError(err).WithField("user_id", userID).Error("failed to extract domain from receipt sender")
+		return true
+	}
+	if serverName != t.ServerName {
+		return true
+	}
+
+	presence := msg.Header.Get("presence")
+
+	ts, err := strconv.Atoi(msg.Header.Get("last_active_ts"))
+	if err != nil {
+		return true
+	}
+
+	joined, err := t.db.GetAllJoinedHosts(ctx)
+	if err != nil {
+		log.WithError(err).Error("failed to get joined hosts")
+		return true
+	}
+	if len(joined) == 0 {
+		return true
+	}
+
+	var statusMsg *string = nil
+	if data, ok := msg.Header["status_msg"]; ok && len(data) > 0 {
+		status := msg.Header.Get("status_msg")
+		statusMsg = &status
+	}
+
+	p := types.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(ts)}
+
+	content := fedTypes.Presence{
+		Push: []fedTypes.PresenceContent{
+			{
+				CurrentlyActive: p.CurrentlyActive(),
+				LastActiveAgo:   p.LastActiveAgo(),
+				Presence:        presence,
+				StatusMsg:       statusMsg,
+				UserID:          userID,
+			},
+		},
+	}
+
+	edu := &gomatrixserverlib.EDU{
+		Type:   gomatrixserverlib.MPresence,
+		Origin: string(t.ServerName),
+	}
+	if edu.Content, err = json.Marshal(content); err != nil {
+		log.WithError(err).Error("failed to marshal EDU JSON")
+		return true
+	}
+
+	log.Debugf("sending presence EDU to %d servers", len(joined))
+	if err = t.queues.SendEDU(edu, t.ServerName, joined); err != nil {
+		log.WithError(err).Error("failed to send EDU")
+		return false
+	}
+
+	return true
+}

federationapi/federationapi.go

@@ -66,6 +66,7 @@ func AddPublicRoutes(
 		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
 		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
 		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
 		ServerName:             cfg.Matrix.ServerName,
 		UserAPI:                userAPI,
 	}
@@ -149,5 +150,11 @@ func NewInternalAPI(
 		logrus.WithError(err).Panic("failed to start key server consumer")
 	}
 
+	presenceConsumer := consumers.NewOutputPresenceConsumer(
+		base.ProcessContext, cfg, js, queues, federationDB,
+	)
+	if err = presenceConsumer.Start(); err != nil {
+		logrus.WithError(err).Panic("failed to start presence consumer")
+	}
 	return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, federation, stats, caches, queues, keyRing)
 }

federationapi/internal/perform.go

@@ -392,17 +392,17 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 
 	// we have the peek state now so let's process regardless of whether upstream gives up
 	ctx = context.Background()
-
 	respState := respPeek.ToRespState()
-	authEvents := respState.AuthEvents.UntrustedEvents(respPeek.RoomVersion)
+
 	// authenticate the state returned (check its auth events etc)
 	// the equivalent of CheckSendJoinResponse()
+	authEvents, _, err := respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName))
+	if err != nil {
+		return fmt.Errorf("error checking state returned from peeking: %w", err)
+	}
 	if err = sanityCheckAuthChain(authEvents); err != nil {
 		return fmt.Errorf("sanityCheckAuthChain: %w", err)
 	}
-	if err = respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName)); err != nil {
-		return fmt.Errorf("error checking state returned from peeking: %w", err)
-	}
 
 	// If we've got this far, the remote server is peeking.
 	if renewing {

federationapi/producers/syncapi.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"strconv"
+	"time"
 
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi/types"
@@ -32,6 +33,7 @@ type SyncAPIProducer struct {
 	TopicReceiptEvent      string
 	TopicSendToDeviceEvent string
 	TopicTypingEvent       string
+	TopicPresenceEvent     string
 	JetStream              nats.JetStreamContext
 	ServerName             gomatrixserverlib.ServerName
 	UserAPI                userapi.UserInternalAPI
@@ -142,3 +144,20 @@ func (p *SyncAPIProducer) SendTyping(
 	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
 	return err
 }
+
+func (p *SyncAPIProducer) SendPresence(
+	ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveAgo int64,
+) error {
+	m := nats.NewMsg(p.TopicPresenceEvent)
+	m.Header.Set(jetstream.UserID, userID)
+	m.Header.Set("presence", presence.String())
+	if statusMsg != nil {
+		m.Header.Set("status_msg", *statusMsg)
+	}
+	lastActiveTS := gomatrixserverlib.AsTimestamp(time.Now().Add(-(time.Duration(lastActiveAgo) * time.Millisecond)))
+
+	m.Header.Set("last_active_ts", strconv.Itoa(int(lastActiveTS)))
+	log.Debugf("Sending presence to syncAPI: %+v", m.Header)
+	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}

federationapi/queue/queue.go

@@ -104,28 +104,31 @@ func NewOutgoingQueues(
 	}
 	// Look up which servers we have pending items for and then rehydrate those queues.
 	if !disabled {
-		time.AfterFunc(time.Second*5, func() {
-			serverNames := map[gomatrixserverlib.ServerName]struct{}{}
-			if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil {
-				for _, serverName := range names {
-					serverNames[serverName] = struct{}{}
-				}
-			} else {
-				log.WithError(err).Error("Failed to get PDU server names for destination queue hydration")
+		serverNames := map[gomatrixserverlib.ServerName]struct{}{}
+		if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil {
+			for _, serverName := range names {
+				serverNames[serverName] = struct{}{}
 			}
-			if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil {
-				for _, serverName := range names {
-					serverNames[serverName] = struct{}{}
-				}
-			} else {
-				log.WithError(err).Error("Failed to get EDU server names for destination queue hydration")
+		} else {
+			log.WithError(err).Error("Failed to get PDU server names for destination queue hydration")
+		}
+		if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil {
+			for _, serverName := range names {
+				serverNames[serverName] = struct{}{}
 			}
-			for serverName := range serverNames {
-				if queue := queues.getQueue(serverName); queue != nil {
-					queue.wakeQueueIfNeeded()
-				}
+		} else {
+			log.WithError(err).Error("Failed to get EDU server names for destination queue hydration")
+		}
+		offset, step := time.Second*5, time.Second
+		if max := len(serverNames); max > 120 {
+			step = (time.Second * 120) / time.Duration(max)
+		}
+		for serverName := range serverNames {
+			if queue := queues.getQueue(serverName); queue != nil {
+				time.AfterFunc(offset, queue.wakeQueueIfNeeded)
+				offset += step
 			}
-		})
+		}
 	}
 	return queues
 }

federationapi/routing/routing.go

@@ -29,6 +29,7 @@ import (
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -53,6 +54,10 @@ func Setup(
 	servers federationAPI.ServersInRoomProvider,
 	producer *producers.SyncAPIProducer,
 ) {
+	prometheus.MustRegister(
+		pduCountTotal, eduCountTotal,
+	)
+
 	v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
 	v1fedmux := fedMux.PathPrefix("/v1").Subrouter()
 	v2fedmux := fedMux.PathPrefix("/v2").Subrouter()

federationapi/routing/send.go

@@ -30,6 +30,7 @@ import (
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
@@ -73,12 +74,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(
-		pduCountTotal, eduCountTotal,
-	)
-}
-
 var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse
 
 // Send implements /_matrix/federation/v1/send/{txnID}
@@ -127,13 +122,14 @@ func Send(
 	defer inFlightTxnsPerOrigin.Delete(index)
 
 	t := txnReq{
-		rsAPI:      rsAPI,
-		keys:       keys,
-		federation: federation,
-		servers:    servers,
-		keyAPI:     keyAPI,
-		roomsMu:    mu,
-		producer:   producer,
+		rsAPI:                  rsAPI,
+		keys:                   keys,
+		federation:             federation,
+		servers:                servers,
+		keyAPI:                 keyAPI,
+		roomsMu:                mu,
+		producer:               producer,
+		inboundPresenceEnabled: cfg.Matrix.Presence.EnableInbound,
 	}
 
 	var txnEvents struct {
@@ -185,13 +181,14 @@ func Send(
 
 type txnReq struct {
 	gomatrixserverlib.Transaction
-	rsAPI      api.RoomserverInternalAPI
-	keyAPI     keyapi.KeyInternalAPI
-	keys       gomatrixserverlib.JSONVerifier
-	federation txnFederationClient
-	roomsMu    *internal.MutexByRoom
-	servers    federationAPI.ServersInRoomProvider
-	producer   *producers.SyncAPIProducer
+	rsAPI                  api.RoomserverInternalAPI
+	keyAPI                 keyapi.KeyInternalAPI
+	keys                   gomatrixserverlib.JSONVerifier
+	federation             txnFederationClient
+	roomsMu                *internal.MutexByRoom
+	servers                federationAPI.ServersInRoomProvider
+	producer               *producers.SyncAPIProducer
+	inboundPresenceEnabled bool
 }
 
 // A subset of FederationClient functionality that txn requires. Useful for testing.
@@ -389,12 +386,36 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			if err := t.processSigningKeyUpdate(ctx, e); err != nil {
 				logrus.WithError(err).Errorf("Failed to process signing key update")
 			}
+		case gomatrixserverlib.MPresence:
+			if t.inboundPresenceEnabled {
+				if err := t.processPresence(ctx, e); err != nil {
+					logrus.WithError(err).Errorf("Failed to process presence update")
+				}
+			}
 		default:
 			util.GetLogger(ctx).WithField("type", e.Type).Debug("Unhandled EDU")
 		}
 	}
 }
 
+// processPresence handles m.receipt events
+func (t *txnReq) processPresence(ctx context.Context, e gomatrixserverlib.EDU) error {
+	payload := types.Presence{}
+	if err := json.Unmarshal(e.Content, &payload); err != nil {
+		return err
+	}
+	for _, content := range payload.Push {
+		presence, ok := syncTypes.PresenceFromString(content.Presence)
+		if !ok {
+			continue
+		}
+		if err := t.producer.SendPresence(ctx, content.UserID, presence, content.StatusMsg, content.LastActiveAgo); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverlib.EDU) error {
 	var updatePayload keyapi.CrossSigningKeyUpdate
 	if err := json.Unmarshal(e.Content, &updatePayload); err != nil {

federationapi/types/types.go

@@ -66,3 +66,15 @@ type FederationReceiptData struct {
 type ReceiptTS struct {
 	TS gomatrixserverlib.Timestamp `json:"ts"`
 }
+
+type Presence struct {
+	Push []PresenceContent `json:"push"`
+}
+
+type PresenceContent struct {
+	CurrentlyActive bool    `json:"currently_active,omitempty"`
+	LastActiveAgo   int64   `json:"last_active_ago"`
+	Presence        string  `json:"presence"`
+	StatusMsg       *string `json:"status_msg,omitempty"`
+	UserID          string  `json:"user_id"`
+}

go.mod

@@ -1,8 +1,8 @@
 module github.com/matrix-org/dendrite
 
-replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e
+replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9
 
-replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c
+replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e
 
 require (
 	github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0
@@ -12,44 +12,34 @@ require (
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/codeclysm/extract v2.2.0+incompatible
-	github.com/containerd/containerd v1.5.9 // indirect
-	github.com/docker/docker v20.10.12+incompatible
+	github.com/containerd/containerd v1.6.2 // indirect
+	github.com/docker/docker v20.10.14+incompatible
 	github.com/docker/go-connections v0.4.0
-	github.com/frankban/quicktest v1.14.0 // indirect
-	github.com/getsentry/sentry-go v0.12.0
+	github.com/frankban/quicktest v1.14.3 // indirect
+	github.com/getsentry/sentry-go v0.13.0
 	github.com/gologme/log v1.3.0
-	github.com/google/go-cmp v0.5.6
-	github.com/google/uuid v1.2.0
+	github.com/google/go-cmp v0.5.7
+	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
-	github.com/gorilla/websocket v1.4.2
+	github.com/gorilla/websocket v1.5.0
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/hashicorp/golang-lru v0.5.4
-	github.com/juju/testing v0.0.0-20211215003918-77eb13d6cad2 // indirect
-	github.com/lib/pq v1.10.4
-	github.com/libp2p/go-libp2p v0.13.0
-	github.com/libp2p/go-libp2p-circuit v0.4.0
-	github.com/libp2p/go-libp2p-core v0.8.3
-	github.com/libp2p/go-libp2p-gostream v0.3.1
-	github.com/libp2p/go-libp2p-http v0.2.0
-	github.com/libp2p/go-libp2p-kad-dht v0.11.1
-	github.com/libp2p/go-libp2p-pubsub v0.4.1
-	github.com/libp2p/go-libp2p-record v0.1.3
+	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
+	github.com/lib/pq v1.10.5
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
-	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
-	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
+	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0
-	github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f
+	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
-	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/miekg/dns v1.1.31 // indirect
 	github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb
-	github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d
+	github.com/nats-io/nats.go v1.14.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31
-	github.com/onsi/ginkgo v1.16.4 // indirect
-	github.com/onsi/gomega v1.13.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
@@ -61,15 +51,14 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.3
 	go.uber.org/atomic v1.9.0
-	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
-	golang.org/x/image v0.0.0-20211028202545-6944b10bf410
-	golang.org/x/mobile v0.0.0-20220112015953-858099ff7816
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
-	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
+	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29
+	golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
+	golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2
+	golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3
+	golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
-	gopkg.in/h2non/bimg.v1 v1.1.5
+	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	nhooyr.io/websocket v1.8.7
 )
 

internal/caching/cache_lazy_load_members.go

@@ -0,0 +1,86 @@
+package caching
+
+import (
+	"fmt"
+	"time"
+
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+const (
+	LazyLoadCacheName           = "lazy_load_members"
+	LazyLoadCacheMaxEntries     = 128
+	LazyLoadCacheMaxUserEntries = 128
+	LazyLoadCacheMutable        = true
+	LazyLoadCacheMaxAge         = time.Minute * 30
+)
+
+type LazyLoadCache struct {
+	// InMemoryLRUCachePartition containing other InMemoryLRUCachePartitions
+	// with the actual cached members
+	userCaches *InMemoryLRUCachePartition
+}
+
+// NewLazyLoadCache creates a new LazyLoadCache.
+func NewLazyLoadCache() (*LazyLoadCache, error) {
+	cache, err := NewInMemoryLRUCachePartition(
+		LazyLoadCacheName,
+		LazyLoadCacheMutable,
+		LazyLoadCacheMaxEntries,
+		LazyLoadCacheMaxAge,
+		true,
+	)
+	if err != nil {
+		return nil, err
+	}
+	go cacheCleaner(cache)
+	return &LazyLoadCache{
+		userCaches: cache,
+	}, nil
+}
+
+func (c *LazyLoadCache) lazyLoadCacheForUser(device *userapi.Device) (*InMemoryLRUCachePartition, error) {
+	cacheName := fmt.Sprintf("%s/%s", device.UserID, device.ID)
+	userCache, ok := c.userCaches.Get(cacheName)
+	if ok && userCache != nil {
+		if cache, ok := userCache.(*InMemoryLRUCachePartition); ok {
+			return cache, nil
+		}
+	}
+	cache, err := NewInMemoryLRUCachePartition(
+		LazyLoadCacheName,
+		LazyLoadCacheMutable,
+		LazyLoadCacheMaxUserEntries,
+		LazyLoadCacheMaxAge,
+		false,
+	)
+	if err != nil {
+		return nil, err
+	}
+	c.userCaches.Set(cacheName, cache)
+	go cacheCleaner(cache)
+	return cache, nil
+}
+
+func (c *LazyLoadCache) StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string) {
+	cache, err := c.lazyLoadCacheForUser(device)
+	if err != nil {
+		return
+	}
+	cacheKey := fmt.Sprintf("%s/%s/%s/%s", device.UserID, device.ID, roomID, userID)
+	cache.Set(cacheKey, eventID)
+}
+
+func (c *LazyLoadCache) IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool) {
+	cache, err := c.lazyLoadCacheForUser(device)
+	if err != nil {
+		return "", false
+	}
+
+	cacheKey := fmt.Sprintf("%s/%s/%s/%s", device.UserID, device.ID, roomID, userID)
+	val, ok := cache.Get(cacheKey)
+	if !ok {
+		return "", ok
+	}
+	return val.(string), ok
+}

internal/eventutil/types.go

@@ -17,6 +17,8 @@ package eventutil
 import (
 	"errors"
 	"strconv"
+
+	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 // ErrProfileNoExists is returned when trying to lookup a user's profile that
@@ -26,9 +28,10 @@ var ErrProfileNoExists = errors.New("no known profile for given user ID")
 // AccountData represents account data sent from the client API server to the
 // sync API server
 type AccountData struct {
-	RoomID     string          `json:"room_id"`
-	Type       string          `json:"type"`
-	ReadMarker *ReadMarkerJSON `json:"read_marker,omitempty"` // optional
+	RoomID       string              `json:"room_id"`
+	Type         string              `json:"type"`
+	ReadMarker   *ReadMarkerJSON     `json:"read_marker,omitempty"`   // optional
+	IgnoredUsers *types.IgnoredUsers `json:"ignored_users,omitempty"` // optional
 }
 
 type ReadMarkerJSON struct {

internal/httputil/httpapi.go

@@ -169,8 +169,9 @@ func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request)
 	return promhttp.InstrumentHandlerCounter(
 		promauto.NewCounterVec(
 			prometheus.CounterOpts{
-				Name: metricsName,
-				Help: "Total number of http requests for HTML resources",
+				Name:      metricsName,
+				Help:      "Total number of http requests for HTML resources",
+				Namespace: "dendrite",
 			},
 			[]string{"code"},
 		),
@@ -201,7 +202,28 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse
 		h.ServeHTTP(w, req)
 	}
 
-	return http.HandlerFunc(withSpan)
+	return promhttp.InstrumentHandlerCounter(
+		promauto.NewCounterVec(
+			prometheus.CounterOpts{
+				Name:      metricsName + "_requests_total",
+				Help:      "Total number of internal API calls",
+				Namespace: "dendrite",
+			},
+			[]string{"code"},
+		),
+		promhttp.InstrumentHandlerResponseSize(
+			promauto.NewHistogramVec(
+				prometheus.HistogramOpts{
+					Namespace: "dendrite",
+					Name:      metricsName + "_response_size_bytes",
+					Help:      "A histogram of response sizes for requests.",
+					Buckets:   []float64{200, 500, 900, 1500, 5000, 15000, 50000, 100000},
+				},
+				[]string{},
+			),
+			http.HandlerFunc(withSpan),
+		),
+	)
 }
 
 // MakeFedAPI makes an http.Handler that checks matrix federation authentication.

internal/pushgateway/pushgateway.go

@@ -3,8 +3,6 @@ package pushgateway
 import (
 	"context"
 	"encoding/json"
-
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // A Client is how interactions with a Push Gateway is done.
@@ -47,11 +45,11 @@ type Counts struct {
 }
 
 type Device struct {
-	AppID     string                      `json:"app_id"`  // Required
-	Data      map[string]interface{}      `json:"data"`    // Required. UNSPEC: Sytests require this to allow unknown keys.
-	PushKey   string                      `json:"pushkey"` // Required
-	PushKeyTS gomatrixserverlib.Timestamp `json:"pushkey_ts,omitempty"`
-	Tweaks    map[string]interface{}      `json:"tweaks,omitempty"`
+	AppID     string                 `json:"app_id"`  // Required
+	Data      map[string]interface{} `json:"data"`    // Required. UNSPEC: Sytests require this to allow unknown keys.
+	PushKey   string                 `json:"pushkey"` // Required
+	PushKeyTS int64                  `json:"pushkey_ts,omitempty"`
+	Tweaks    map[string]interface{} `json:"tweaks,omitempty"`
 }
 
 type Prio string

internal/test/config.go

@@ -78,8 +78,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress())
 	cfg.Global.PrivateKeyPath = config.Path(serverKeyPath)
 
-	cfg.FederationAPI.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)}
-
 	cfg.MediaAPI.BasePath = config.Path(mediaBasePath)
 
 	cfg.Global.JetStream.Addresses = []string{kafkaURI}

internal/version.go

@@ -16,8 +16,8 @@ var build string
 
 const (
 	VersionMajor = 0
-	VersionMinor = 7
-	VersionPatch = 0
+	VersionMinor = 8
+	VersionPatch = 1
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/internal/device_list_update.go

@@ -157,8 +157,15 @@ func (u *DeviceListUpdater) Start() error {
 	if err != nil {
 		return err
 	}
+	offset, step := time.Second*10, time.Second
+	if max := len(staleLists); max > 120 {
+		step = (time.Second * 120) / time.Duration(max)
+	}
 	for _, userID := range staleLists {
-		u.notifyWorkers(userID)
+		time.AfterFunc(offset, func() {
+			u.notifyWorkers(userID)
+		})
+		offset += step
 	}
 	return nil
 }

mediaapi/mediaapi.go

@@ -32,7 +32,7 @@ func AddPublicRoutes(
 	userAPI userapi.UserInternalAPI,
 	client *gomatrixserverlib.Client,
 ) {
-	mediaDB, err := storage.Open(&cfg.Database)
+	mediaDB, err := storage.NewMediaAPIDatasource(&cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to media db")
 	}

mediaapi/routing/upload.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strings"
 
@@ -311,6 +312,26 @@ func (r *uploadRequest) storeFileAndMetadata(
 	}
 
 	go func() {
+		file, err := os.Open(string(finalPath))
+		if err != nil {
+			r.Logger.WithError(err).Error("unable to open file")
+			return
+		}
+		defer file.Close() // nolint: errcheck
+		// http.DetectContentType only needs 512 bytes
+		buf := make([]byte, 512)
+		_, err = file.Read(buf)
+		if err != nil {
+			r.Logger.WithError(err).Error("unable to read file")
+			return
+		}
+		// Check if we need to generate thumbnails
+		fileType := http.DetectContentType(buf)
+		if !strings.HasPrefix(fileType, "image") {
+			r.Logger.WithField("contentType", fileType).Debugf("uploaded file is not an image or can not be thumbnailed, not generating thumbnails")
+			return
+		}
+
 		busy, err := thumbnailer.GenerateThumbnails(
 			context.Background(), finalPath, thumbnailSizes, r.MediaMetadata,
 			activeThumbnailGeneration, maxThumbnailGenerators, db, r.Logger,

mediaapi/routing/upload_test.go

@@ -51,7 +51,7 @@ func Test_uploadRequest_doUpload(t *testing.T) {
 	_ = os.Mkdir(testdataPath, os.ModePerm)
 	defer fileutils.RemoveDir(types.Path(testdataPath), nil)
 
-	db, err := storage.Open(&config.DatabaseOptions{
+	db, err := storage.NewMediaAPIDatasource(&config.DatabaseOptions{
 		ConnectionString:       "file::memory:?cache=shared",
 		MaxOpenConnections:     100,
 		MaxIdleConnections:     2,

mediaapi/storage/interface.go

@@ -22,9 +22,17 @@ import (
 )
 
 type Database interface {
+	MediaRepository
+	Thumbnails
+}
+
+type MediaRepository interface {
 	StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error
 	GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error)
 	GetMediaMetadataByHash(ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error)
+}
+
+type Thumbnails interface {
 	StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error
 	GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error)
 	GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error)

mediaapi/storage/postgres/media_repository_table.go

@@ -20,6 +20,8 @@ import (
 	"database/sql"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -69,24 +71,25 @@ type mediaStatements struct {
 	selectMediaByHashStmt *sql.Stmt
 }
 
-func (s *mediaStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(mediaSchema)
+func NewPostgresMediaRepositoryTable(db *sql.DB) (tables.MediaRepository, error) {
+	s := &mediaStatements{}
+	_, err := db.Exec(mediaSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
 
-	return statementList{
+	return s, sqlutil.StatementList{
 		{&s.insertMediaStmt, insertMediaSQL},
 		{&s.selectMediaStmt, selectMediaSQL},
 		{&s.selectMediaByHashStmt, selectMediaByHashSQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *mediaStatements) insertMedia(
-	ctx context.Context, mediaMetadata *types.MediaMetadata,
+func (s *mediaStatements) InsertMedia(
+	ctx context.Context, txn *sql.Tx, mediaMetadata *types.MediaMetadata,
 ) error {
-	mediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000)
-	_, err := s.insertMediaStmt.ExecContext(
+	mediaMetadata.CreationTimestamp = gomatrixserverlib.AsTimestamp(time.Now())
+	_, err := sqlutil.TxStmtContext(ctx, txn, s.insertMediaStmt).ExecContext(
 		ctx,
 		mediaMetadata.MediaID,
 		mediaMetadata.Origin,
@@ -100,14 +103,14 @@ func (s *mediaStatements) insertMedia(
 	return err
 }
 
-func (s *mediaStatements) selectMedia(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+func (s *mediaStatements) SelectMedia(
+	ctx context.Context, txn *sql.Tx, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
 ) (*types.MediaMetadata, error) {
 	mediaMetadata := types.MediaMetadata{
 		MediaID: mediaID,
 		Origin:  mediaOrigin,
 	}
-	err := s.selectMediaStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectMediaStmt).QueryRowContext(
 		ctx, mediaMetadata.MediaID, mediaMetadata.Origin,
 	).Scan(
 		&mediaMetadata.ContentType,
@@ -120,14 +123,14 @@ func (s *mediaStatements) selectMedia(
 	return &mediaMetadata, err
 }
 
-func (s *mediaStatements) selectMediaByHash(
-	ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
+func (s *mediaStatements) SelectMediaByHash(
+	ctx context.Context, txn *sql.Tx, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
 ) (*types.MediaMetadata, error) {
 	mediaMetadata := types.MediaMetadata{
 		Base64Hash: mediaHash,
 		Origin:     mediaOrigin,
 	}
-	err := s.selectMediaStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectMediaByHashStmt).QueryRowContext(
 		ctx, mediaMetadata.Base64Hash, mediaMetadata.Origin,
 	).Scan(
 		&mediaMetadata.ContentType,

mediaapi/storage/postgres/mediaapi.go

@@ -0,0 +1,46 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	// Import the postgres database driver.
+	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/shared"
+	"github.com/matrix-org/dendrite/setup/config"
+)
+
+// NewDatabase opens a postgres database.
+func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
+	db, err := sqlutil.Open(dbProperties)
+	if err != nil {
+		return nil, err
+	}
+	mediaRepo, err := NewPostgresMediaRepositoryTable(db)
+	if err != nil {
+		return nil, err
+	}
+	thumbnails, err := NewPostgresThumbnailsTable(db)
+	if err != nil {
+		return nil, err
+	}
+	return &shared.Database{
+		MediaRepository: mediaRepo,
+		Thumbnails:      thumbnails,
+		DB:              db,
+		Writer:          sqlutil.NewExclusiveWriter(),
+	}, nil
+}

mediaapi/storage/postgres/prepare.go

@@ -1,38 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// FIXME: This should be made internal!
-
-package postgres
-
-import (
-	"database/sql"
-)
-
-// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
-type statementList []struct {
-	statement **sql.Stmt
-	sql       string
-}
-
-// prepare the SQL for each statement in the list and assign the result to the prepared statement.
-func (s statementList) prepare(db *sql.DB) (err error) {
-	for _, statement := range s {
-		if *statement.statement, err = db.Prepare(statement.sql); err != nil {
-			return
-		}
-	}
-	return
-}

mediaapi/storage/postgres/sql.go

@@ -1,36 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"database/sql"
-)
-
-type statements struct {
-	media     mediaStatements
-	thumbnail thumbnailStatements
-}
-
-func (s *statements) prepare(db *sql.DB) (err error) {
-	if err = s.media.prepare(db); err != nil {
-		return
-	}
-	if err = s.thumbnail.prepare(db); err != nil {
-		return
-	}
-
-	return
-}

mediaapi/storage/postgres/thumbnail_table.go

@@ -21,6 +21,8 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -63,7 +65,7 @@ SELECT content_type, file_size_bytes, creation_ts FROM mediaapi_thumbnail WHERE
 
 // Note: this selects all thumbnails for a media_origin and media_id
 const selectThumbnailsSQL = `
-SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2
+SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2 ORDER BY creation_ts ASC
 `
 
 type thumbnailStatements struct {
@@ -72,24 +74,25 @@ type thumbnailStatements struct {
 	selectThumbnailsStmt *sql.Stmt
 }
 
-func (s *thumbnailStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(thumbnailSchema)
+func NewPostgresThumbnailsTable(db *sql.DB) (tables.Thumbnails, error) {
+	s := &thumbnailStatements{}
+	_, err := db.Exec(thumbnailSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
 
-	return statementList{
+	return s, sqlutil.StatementList{
 		{&s.insertThumbnailStmt, insertThumbnailSQL},
 		{&s.selectThumbnailStmt, selectThumbnailSQL},
 		{&s.selectThumbnailsStmt, selectThumbnailsSQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *thumbnailStatements) insertThumbnail(
-	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
+func (s *thumbnailStatements) InsertThumbnail(
+	ctx context.Context, txn *sql.Tx, thumbnailMetadata *types.ThumbnailMetadata,
 ) error {
-	thumbnailMetadata.MediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000)
-	_, err := s.insertThumbnailStmt.ExecContext(
+	thumbnailMetadata.MediaMetadata.CreationTimestamp = gomatrixserverlib.AsTimestamp(time.Now())
+	_, err := sqlutil.TxStmtContext(ctx, txn, s.insertThumbnailStmt).ExecContext(
 		ctx,
 		thumbnailMetadata.MediaMetadata.MediaID,
 		thumbnailMetadata.MediaMetadata.Origin,
@@ -103,8 +106,9 @@ func (s *thumbnailStatements) insertThumbnail(
 	return err
 }
 
-func (s *thumbnailStatements) selectThumbnail(
+func (s *thumbnailStatements) SelectThumbnail(
 	ctx context.Context,
+	txn *sql.Tx,
 	mediaID types.MediaID,
 	mediaOrigin gomatrixserverlib.ServerName,
 	width, height int,
@@ -121,7 +125,7 @@ func (s *thumbnailStatements) selectThumbnail(
 			ResizeMethod: resizeMethod,
 		},
 	}
-	err := s.selectThumbnailStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectThumbnailStmt).QueryRowContext(
 		ctx,
 		thumbnailMetadata.MediaMetadata.MediaID,
 		thumbnailMetadata.MediaMetadata.Origin,
@@ -136,10 +140,10 @@ func (s *thumbnailStatements) selectThumbnail(
 	return &thumbnailMetadata, err
 }
 
-func (s *thumbnailStatements) selectThumbnails(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+func (s *thumbnailStatements) SelectThumbnails(
+	ctx context.Context, txn *sql.Tx, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
 ) ([]*types.ThumbnailMetadata, error) {
-	rows, err := s.selectThumbnailsStmt.QueryContext(
+	rows, err := sqlutil.TxStmtContext(ctx, txn, s.selectThumbnailsStmt).QueryContext(
 		ctx, mediaID, mediaOrigin,
 	)
 	if err != nil {

mediaapi/storage/shared/mediaapi.go

@@ -1,5 +1,4 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+// Copyright 2022 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,54 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package postgres
+package shared
 
 import (
 	"context"
 	"database/sql"
 
-	// Import the postgres database driver.
-	_ "github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// Database is used to store metadata about a repository of media files.
 type Database struct {
-	statements statements
-	db         *sql.DB
-}
-
-// Open opens a postgres database.
-func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
-	var d Database
-	var err error
-	if d.db, err = sqlutil.Open(dbProperties); err != nil {
-		return nil, err
-	}
-	if err = d.statements.prepare(d.db); err != nil {
-		return nil, err
-	}
-	return &d, nil
+	DB              *sql.DB
+	Writer          sqlutil.Writer
+	MediaRepository tables.MediaRepository
+	Thumbnails      tables.Thumbnails
 }
 
 // StoreMediaMetadata inserts the metadata about the uploaded media into the database.
 // Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreMediaMetadata(
-	ctx context.Context, mediaMetadata *types.MediaMetadata,
-) error {
-	return d.statements.media.insertMedia(ctx, mediaMetadata)
+func (d Database) StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.MediaRepository.InsertMedia(ctx, txn, mediaMetadata)
+	})
 }
 
 // GetMediaMetadata returns metadata about media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this media.
-func (d *Database) GetMediaMetadata(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.media.selectMedia(ctx, mediaID, mediaOrigin)
+func (d Database) GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) {
+	mediaMetadata, err := d.MediaRepository.SelectMedia(ctx, nil, mediaID, mediaOrigin)
 	if err != nil && err == sql.ErrNoRows {
 		return nil, nil
 	}
@@ -70,10 +53,8 @@ func (d *Database) GetMediaMetadata(
 // GetMediaMetadataByHash returns metadata about media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this media.
-func (d *Database) GetMediaMetadataByHash(
-	ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
-) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.media.selectMediaByHash(ctx, mediaHash, mediaOrigin)
+func (d Database) GetMediaMetadataByHash(ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) {
+	mediaMetadata, err := d.MediaRepository.SelectMediaByHash(ctx, nil, mediaHash, mediaOrigin)
 	if err != nil && err == sql.ErrNoRows {
 		return nil, nil
 	}
@@ -82,40 +63,36 @@ func (d *Database) GetMediaMetadataByHash(
 
 // StoreThumbnail inserts the metadata about the thumbnail into the database.
 // Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreThumbnail(
-	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
-) error {
-	return d.statements.thumbnail.insertThumbnail(ctx, thumbnailMetadata)
+func (d Database) StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.Thumbnails.InsertThumbnail(ctx, txn, thumbnailMetadata)
+	})
 }
 
 // GetThumbnail returns metadata about a specific thumbnail.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this thumbnail.
-func (d *Database) GetThumbnail(
-	ctx context.Context,
-	mediaID types.MediaID,
-	mediaOrigin gomatrixserverlib.ServerName,
-	width, height int,
-	resizeMethod string,
-) (*types.ThumbnailMetadata, error) {
-	thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail(
-		ctx, mediaID, mediaOrigin, width, height, resizeMethod,
-	)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
+func (d Database) GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) {
+	metadata, err := d.Thumbnails.SelectThumbnail(ctx, nil, mediaID, mediaOrigin, width, height, resizeMethod)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
 	}
-	return thumbnailMetadata, err
+	return metadata, err
 }
 
 // GetThumbnails returns metadata about all thumbnails for a specific media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there are no thumbnails associated with this media.
-func (d *Database) GetThumbnails(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) ([]*types.ThumbnailMetadata, error) {
-	thumbnails, err := d.statements.thumbnail.selectThumbnails(ctx, mediaID, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
+func (d Database) GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) {
+	metadatas, err := d.Thumbnails.SelectThumbnails(ctx, nil, mediaID, mediaOrigin)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
 	}
-	return thumbnails, err
+	return metadatas, err
 }

mediaapi/storage/sqlite3/media_repository_table.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -66,57 +67,53 @@ SELECT content_type, file_size_bytes, creation_ts, upload_name, media_id, user_i
 
 type mediaStatements struct {
 	db                    *sql.DB
-	writer                sqlutil.Writer
 	insertMediaStmt       *sql.Stmt
 	selectMediaStmt       *sql.Stmt
 	selectMediaByHashStmt *sql.Stmt
 }
 
-func (s *mediaStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
-	s.db = db
-	s.writer = writer
-
-	_, err = db.Exec(mediaSchema)
+func NewSQLiteMediaRepositoryTable(db *sql.DB) (tables.MediaRepository, error) {
+	s := &mediaStatements{
+		db: db,
+	}
+	_, err := db.Exec(mediaSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
 
-	return statementList{
+	return s, sqlutil.StatementList{
 		{&s.insertMediaStmt, insertMediaSQL},
 		{&s.selectMediaStmt, selectMediaSQL},
 		{&s.selectMediaByHashStmt, selectMediaByHashSQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *mediaStatements) insertMedia(
-	ctx context.Context, mediaMetadata *types.MediaMetadata,
+func (s *mediaStatements) InsertMedia(
+	ctx context.Context, txn *sql.Tx, mediaMetadata *types.MediaMetadata,
 ) error {
-	mediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000)
-	return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
-		stmt := sqlutil.TxStmt(txn, s.insertMediaStmt)
-		_, err := stmt.ExecContext(
-			ctx,
-			mediaMetadata.MediaID,
-			mediaMetadata.Origin,
-			mediaMetadata.ContentType,
-			mediaMetadata.FileSizeBytes,
-			mediaMetadata.CreationTimestamp,
-			mediaMetadata.UploadName,
-			mediaMetadata.Base64Hash,
-			mediaMetadata.UserID,
-		)
-		return err
-	})
+	mediaMetadata.CreationTimestamp = gomatrixserverlib.AsTimestamp(time.Now())
+	_, err := sqlutil.TxStmtContext(ctx, txn, s.insertMediaStmt).ExecContext(
+		ctx,
+		mediaMetadata.MediaID,
+		mediaMetadata.Origin,
+		mediaMetadata.ContentType,
+		mediaMetadata.FileSizeBytes,
+		mediaMetadata.CreationTimestamp,
+		mediaMetadata.UploadName,
+		mediaMetadata.Base64Hash,
+		mediaMetadata.UserID,
+	)
+	return err
 }
 
-func (s *mediaStatements) selectMedia(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+func (s *mediaStatements) SelectMedia(
+	ctx context.Context, txn *sql.Tx, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
 ) (*types.MediaMetadata, error) {
 	mediaMetadata := types.MediaMetadata{
 		MediaID: mediaID,
 		Origin:  mediaOrigin,
 	}
-	err := s.selectMediaStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectMediaStmt).QueryRowContext(
 		ctx, mediaMetadata.MediaID, mediaMetadata.Origin,
 	).Scan(
 		&mediaMetadata.ContentType,
@@ -129,14 +126,14 @@ func (s *mediaStatements) selectMedia(
 	return &mediaMetadata, err
 }
 
-func (s *mediaStatements) selectMediaByHash(
-	ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
+func (s *mediaStatements) SelectMediaByHash(
+	ctx context.Context, txn *sql.Tx, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
 ) (*types.MediaMetadata, error) {
 	mediaMetadata := types.MediaMetadata{
 		Base64Hash: mediaHash,
 		Origin:     mediaOrigin,
 	}
-	err := s.selectMediaStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectMediaByHashStmt).QueryRowContext(
 		ctx, mediaMetadata.Base64Hash, mediaMetadata.Origin,
 	).Scan(
 		&mediaMetadata.ContentType,

mediaapi/storage/sqlite3/mediaapi.go

@@ -16,23 +16,30 @@
 package sqlite3
 
 import (
-	"database/sql"
-
+	// Import the postgres database driver.
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/shared"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
-type statements struct {
-	media     mediaStatements
-	thumbnail thumbnailStatements
-}
-
-func (s *statements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
-	if err = s.media.prepare(db, writer); err != nil {
-		return
+// NewDatabase opens a SQLIte database.
+func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
+	db, err := sqlutil.Open(dbProperties)
+	if err != nil {
+		return nil, err
 	}
-	if err = s.thumbnail.prepare(db, writer); err != nil {
-		return
+	mediaRepo, err := NewSQLiteMediaRepositoryTable(db)
+	if err != nil {
+		return nil, err
 	}
-
-	return
+	thumbnails, err := NewSQLiteThumbnailsTable(db)
+	if err != nil {
+		return nil, err
+	}
+	return &shared.Database{
+		MediaRepository: mediaRepo,
+		Thumbnails:      thumbnails,
+		DB:              db,
+		Writer:          sqlutil.NewExclusiveWriter(),
+	}, nil
 }

mediaapi/storage/sqlite3/prepare.go

@@ -1,38 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// FIXME: This should be made internal!
-
-package sqlite3
-
-import (
-	"database/sql"
-)
-
-// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
-type statementList []struct {
-	statement **sql.Stmt
-	sql       string
-}
-
-// prepare the SQL for each statement in the list and assign the result to the prepared statement.
-func (s statementList) prepare(db *sql.DB) (err error) {
-	for _, statement := range s {
-		if *statement.statement, err = db.Prepare(statement.sql); err != nil {
-			return
-		}
-	}
-	return
-}

mediaapi/storage/sqlite3/storage.go

@@ -1,123 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-
-	// Import the postgres database driver.
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/mediaapi/types"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// Database is used to store metadata about a repository of media files.
-type Database struct {
-	statements statements
-	db         *sql.DB
-	writer     sqlutil.Writer
-}
-
-// Open opens a postgres database.
-func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
-	d := Database{
-		writer: sqlutil.NewExclusiveWriter(),
-	}
-	var err error
-	if d.db, err = sqlutil.Open(dbProperties); err != nil {
-		return nil, err
-	}
-	if err = d.statements.prepare(d.db, d.writer); err != nil {
-		return nil, err
-	}
-	return &d, nil
-}
-
-// StoreMediaMetadata inserts the metadata about the uploaded media into the database.
-// Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreMediaMetadata(
-	ctx context.Context, mediaMetadata *types.MediaMetadata,
-) error {
-	return d.statements.media.insertMedia(ctx, mediaMetadata)
-}
-
-// GetMediaMetadata returns metadata about media stored on this server.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there is no metadata associated with this media.
-func (d *Database) GetMediaMetadata(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.media.selectMedia(ctx, mediaID, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return mediaMetadata, err
-}
-
-// GetMediaMetadataByHash returns metadata about media stored on this server.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there is no metadata associated with this media.
-func (d *Database) GetMediaMetadataByHash(
-	ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
-) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.media.selectMediaByHash(ctx, mediaHash, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return mediaMetadata, err
-}
-
-// StoreThumbnail inserts the metadata about the thumbnail into the database.
-// Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreThumbnail(
-	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
-) error {
-	return d.statements.thumbnail.insertThumbnail(ctx, thumbnailMetadata)
-}
-
-// GetThumbnail returns metadata about a specific thumbnail.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there is no metadata associated with this thumbnail.
-func (d *Database) GetThumbnail(
-	ctx context.Context,
-	mediaID types.MediaID,
-	mediaOrigin gomatrixserverlib.ServerName,
-	width, height int,
-	resizeMethod string,
-) (*types.ThumbnailMetadata, error) {
-	thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail(
-		ctx, mediaID, mediaOrigin, width, height, resizeMethod,
-	)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return thumbnailMetadata, err
-}
-
-// GetThumbnails returns metadata about all thumbnails for a specific media stored on this server.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there are no thumbnails associated with this media.
-func (d *Database) GetThumbnails(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) ([]*types.ThumbnailMetadata, error) {
-	thumbnails, err := d.statements.thumbnail.selectThumbnails(ctx, mediaID, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return thumbnails, err
-}

mediaapi/storage/sqlite3/thumbnail_table.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -54,55 +55,48 @@ SELECT content_type, file_size_bytes, creation_ts FROM mediaapi_thumbnail WHERE
 
 // Note: this selects all thumbnails for a media_origin and media_id
 const selectThumbnailsSQL = `
-SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2
+SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2 ORDER BY creation_ts ASC
 `
 
 type thumbnailStatements struct {
-	db                   *sql.DB
-	writer               sqlutil.Writer
 	insertThumbnailStmt  *sql.Stmt
 	selectThumbnailStmt  *sql.Stmt
 	selectThumbnailsStmt *sql.Stmt
 }
 
-func (s *thumbnailStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
-	_, err = db.Exec(thumbnailSchema)
+func NewSQLiteThumbnailsTable(db *sql.DB) (tables.Thumbnails, error) {
+	s := &thumbnailStatements{}
+	_, err := db.Exec(thumbnailSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
-	s.db = db
-	s.writer = writer
 
-	return statementList{
+	return s, sqlutil.StatementList{
 		{&s.insertThumbnailStmt, insertThumbnailSQL},
 		{&s.selectThumbnailStmt, selectThumbnailSQL},
 		{&s.selectThumbnailsStmt, selectThumbnailsSQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *thumbnailStatements) insertThumbnail(
-	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
-) error {
-	thumbnailMetadata.MediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000)
-	return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
-		stmt := sqlutil.TxStmt(txn, s.insertThumbnailStmt)
-		_, err := stmt.ExecContext(
-			ctx,
-			thumbnailMetadata.MediaMetadata.MediaID,
-			thumbnailMetadata.MediaMetadata.Origin,
-			thumbnailMetadata.MediaMetadata.ContentType,
-			thumbnailMetadata.MediaMetadata.FileSizeBytes,
-			thumbnailMetadata.MediaMetadata.CreationTimestamp,
-			thumbnailMetadata.ThumbnailSize.Width,
-			thumbnailMetadata.ThumbnailSize.Height,
-			thumbnailMetadata.ThumbnailSize.ResizeMethod,
-		)
-		return err
-	})
+func (s *thumbnailStatements) InsertThumbnail(ctx context.Context, txn *sql.Tx, thumbnailMetadata *types.ThumbnailMetadata) error {
+	thumbnailMetadata.MediaMetadata.CreationTimestamp = gomatrixserverlib.AsTimestamp(time.Now())
+	_, err := sqlutil.TxStmtContext(ctx, txn, s.insertThumbnailStmt).ExecContext(
+		ctx,
+		thumbnailMetadata.MediaMetadata.MediaID,
+		thumbnailMetadata.MediaMetadata.Origin,
+		thumbnailMetadata.MediaMetadata.ContentType,
+		thumbnailMetadata.MediaMetadata.FileSizeBytes,
+		thumbnailMetadata.MediaMetadata.CreationTimestamp,
+		thumbnailMetadata.ThumbnailSize.Width,
+		thumbnailMetadata.ThumbnailSize.Height,
+		thumbnailMetadata.ThumbnailSize.ResizeMethod,
+	)
+	return err
 }
 
-func (s *thumbnailStatements) selectThumbnail(
+func (s *thumbnailStatements) SelectThumbnail(
 	ctx context.Context,
+	txn *sql.Tx,
 	mediaID types.MediaID,
 	mediaOrigin gomatrixserverlib.ServerName,
 	width, height int,
@@ -119,7 +113,7 @@ func (s *thumbnailStatements) selectThumbnail(
 			ResizeMethod: resizeMethod,
 		},
 	}
-	err := s.selectThumbnailStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectThumbnailStmt).QueryRowContext(
 		ctx,
 		thumbnailMetadata.MediaMetadata.MediaID,
 		thumbnailMetadata.MediaMetadata.Origin,
@@ -134,10 +128,11 @@ func (s *thumbnailStatements) selectThumbnail(
 	return &thumbnailMetadata, err
 }
 
-func (s *thumbnailStatements) selectThumbnails(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+func (s *thumbnailStatements) SelectThumbnails(
+	ctx context.Context, txn *sql.Tx, mediaID types.MediaID,
+	mediaOrigin gomatrixserverlib.ServerName,
 ) ([]*types.ThumbnailMetadata, error) {
-	rows, err := s.selectThumbnailsStmt.QueryContext(
+	rows, err := sqlutil.TxStmtContext(ctx, txn, s.selectThumbnailsStmt).QueryContext(
 		ctx, mediaID, mediaOrigin,
 	)
 	if err != nil {

mediaapi/storage/storage.go

@@ -25,13 +25,13 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-// Open opens a postgres database.
-func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+// NewMediaAPIDatasource opens a database connection.
+func NewMediaAPIDatasource(dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.Open(dbProperties)
+		return sqlite3.NewDatabase(dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.Open(dbProperties)
+		return postgres.NewDatabase(dbProperties)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}

mediaapi/storage/storage_test.go

@@ -0,0 +1,135 @@
+package storage_test
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/matrix-org/dendrite/mediaapi/storage"
+	"github.com/matrix-org/dendrite/mediaapi/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+)
+
+func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := storage.NewMediaAPIDatasource(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	})
+	if err != nil {
+		t.Fatalf("NewSyncServerDatasource returned %s", err)
+	}
+	return db, close
+}
+func TestMediaRepository(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+		ctx := context.Background()
+		t.Run("can insert media & query media", func(t *testing.T) {
+			metadata := &types.MediaMetadata{
+				MediaID:       "testing",
+				Origin:        "localhost",
+				ContentType:   "image/png",
+				FileSizeBytes: 10,
+				UploadName:    "upload test",
+				Base64Hash:    "dGVzdGluZw==",
+				UserID:        "@alice:localhost",
+			}
+			if err := db.StoreMediaMetadata(ctx, metadata); err != nil {
+				t.Fatalf("unable to store media metadata: %v", err)
+			}
+			// query by media id
+			gotMetadata, err := db.GetMediaMetadata(ctx, metadata.MediaID, metadata.Origin)
+			if err != nil {
+				t.Fatalf("unable to query media metadata: %v", err)
+			}
+			if !reflect.DeepEqual(metadata, gotMetadata) {
+				t.Fatalf("expected metadata %+v, got %v", metadata, gotMetadata)
+			}
+			// query by media hash
+			gotMetadata, err = db.GetMediaMetadataByHash(ctx, metadata.Base64Hash, metadata.Origin)
+			if err != nil {
+				t.Fatalf("unable to query media metadata by hash: %v", err)
+			}
+			if !reflect.DeepEqual(metadata, gotMetadata) {
+				t.Fatalf("expected metadata %+v, got %v", metadata, gotMetadata)
+			}
+		})
+	})
+}
+
+func TestThumbnailsStorage(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+		ctx := context.Background()
+		t.Run("can insert thumbnails & query media", func(t *testing.T) {
+			thumbnails := []*types.ThumbnailMetadata{
+				{
+					MediaMetadata: &types.MediaMetadata{
+						MediaID:       "testing",
+						Origin:        "localhost",
+						ContentType:   "image/png",
+						FileSizeBytes: 6,
+					},
+					ThumbnailSize: types.ThumbnailSize{
+						Width:        5,
+						Height:       5,
+						ResizeMethod: types.Crop,
+					},
+				},
+				{
+					MediaMetadata: &types.MediaMetadata{
+						MediaID:       "testing",
+						Origin:        "localhost",
+						ContentType:   "image/png",
+						FileSizeBytes: 7,
+					},
+					ThumbnailSize: types.ThumbnailSize{
+						Width:        1,
+						Height:       1,
+						ResizeMethod: types.Scale,
+					},
+				},
+			}
+			for i := range thumbnails {
+				if err := db.StoreThumbnail(ctx, thumbnails[i]); err != nil {
+					t.Fatalf("unable to store thumbnail metadata: %v", err)
+				}
+			}
+			// query by single thumbnail
+			gotMetadata, err := db.GetThumbnail(ctx,
+				thumbnails[0].MediaMetadata.MediaID,
+				thumbnails[0].MediaMetadata.Origin,
+				thumbnails[0].ThumbnailSize.Width, thumbnails[0].ThumbnailSize.Height,
+				thumbnails[0].ThumbnailSize.ResizeMethod,
+			)
+			if err != nil {
+				t.Fatalf("unable to query thumbnail metadata: %v", err)
+			}
+			if !reflect.DeepEqual(thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata) {
+				t.Fatalf("expected metadata %+v, got %+v", thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata)
+			}
+			if !reflect.DeepEqual(thumbnails[0].ThumbnailSize, gotMetadata.ThumbnailSize) {
+				t.Fatalf("expected metadata %+v, got %+v", thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata)
+			}
+			// query by all thumbnails
+			gotMediadatas, err := db.GetThumbnails(ctx, thumbnails[0].MediaMetadata.MediaID, thumbnails[0].MediaMetadata.Origin)
+			if err != nil {
+				t.Fatalf("unable to query media metadata by hash: %v", err)
+			}
+			if len(gotMediadatas) != len(thumbnails) {
+				t.Fatalf("expected %d stored thumbnail metadata, got %d", len(thumbnails), len(gotMediadatas))
+			}
+			for i := range gotMediadatas {
+				if !reflect.DeepEqual(thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata) {
+					t.Fatalf("expected metadata %+v, got %v", thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata)
+				}
+				if !reflect.DeepEqual(thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize) {
+					t.Fatalf("expected metadata %+v, got %v", thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize)
+				}
+			}
+		})
+	})
+}

mediaapi/storage/storage_wasm.go

@@ -22,10 +22,10 @@ import (
 )
 
 // Open opens a postgres database.
-func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewMediaAPIDatasource(dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.Open(dbProperties)
+		return sqlite3.NewDatabase(dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:

mediaapi/storage/tables/interface.go

@@ -0,0 +1,46 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/mediaapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type Thumbnails interface {
+	InsertThumbnail(ctx context.Context, txn *sql.Tx, thumbnailMetadata *types.ThumbnailMetadata) error
+	SelectThumbnail(
+		ctx context.Context, txn *sql.Tx,
+		mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+		width, height int,
+		resizeMethod string,
+	) (*types.ThumbnailMetadata, error)
+	SelectThumbnails(
+		ctx context.Context, txn *sql.Tx, mediaID types.MediaID,
+		mediaOrigin gomatrixserverlib.ServerName,
+	) ([]*types.ThumbnailMetadata, error)
+}
+
+type MediaRepository interface {
+	InsertMedia(ctx context.Context, txn *sql.Tx, mediaMetadata *types.MediaMetadata) error
+	SelectMedia(ctx context.Context, txn *sql.Tx, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error)
+	SelectMediaByHash(
+		ctx context.Context, txn *sql.Tx,
+		mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
+	) (*types.MediaMetadata, error)
+}

mediaapi/types/types.go

@@ -45,16 +45,13 @@ type RequestMethod string
 // MatrixUserID is a Matrix user ID string in the form @user:domain e.g. @alice:matrix.org
 type MatrixUserID string
 
-// UnixMs is the milliseconds since the Unix epoch
-type UnixMs int64
-
 // MediaMetadata is metadata associated with a media file
 type MediaMetadata struct {
 	MediaID           MediaID
 	Origin            gomatrixserverlib.ServerName
 	ContentType       ContentType
 	FileSizeBytes     FileSizeBytes
-	CreationTimestamp UnixMs
+	CreationTimestamp gomatrixserverlib.Timestamp
 	UploadName        Filename
 	Base64Hash        Base64Hash
 	UserID            MatrixUserID

roomserver/api/api.go

@@ -170,6 +170,9 @@ type RoomserverInternalAPI interface {
 	// PerformForget forgets a rooms history for a specific user
 	PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error
 
+	// PerformRoomUpgrade upgrades a room to a newer version
+	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse)
+
 	// Asks for the default room version as preferred by the server.
 	QueryRoomVersionCapabilities(
 		ctx context.Context,

roomserver/api/api_trace.go

@@ -67,6 +67,15 @@ func (t *RoomserverInternalAPITrace) PerformUnpeek(
 	util.GetLogger(ctx).Infof("PerformUnpeek req=%+v res=%+v", js(req), js(res))
 }
 
+func (t *RoomserverInternalAPITrace) PerformRoomUpgrade(
+	ctx context.Context,
+	req *PerformRoomUpgradeRequest,
+	res *PerformRoomUpgradeResponse,
+) {
+	t.Impl.PerformRoomUpgrade(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformRoomUpgrade req=%+v res=%+v", js(req), js(res))
+}
+
 func (t *RoomserverInternalAPITrace) PerformJoin(
 	ctx context.Context,
 	req *PerformJoinRequest,

roomserver/api/perform.go

@@ -203,3 +203,14 @@ type PerformForgetRequest struct {
 }
 
 type PerformForgetResponse struct{}
+
+type PerformRoomUpgradeRequest struct {
+	RoomID      string                        `json:"room_id"`
+	UserID      string                        `json:"user_id"`
+	RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
+}
+
+type PerformRoomUpgradeResponse struct {
+	NewRoomID string
+	Error     *PerformError
+}

roomserver/api/query.go

@@ -128,6 +128,8 @@ type QueryMembershipForUserResponse struct {
 type QueryMembershipsForRoomRequest struct {
 	// If true, only returns the membership events of "join" membership
 	JoinedOnly bool `json:"joined_only"`
+	// If true, only returns the membership events of local users
+	LocalOnly bool `json:"local_only"`
 	// ID of the room to fetch memberships from
 	RoomID string `json:"room_id"`
 	// Optional - ID of the user sending the request, for checking if the

roomserver/internal/api.go

@@ -34,6 +34,7 @@ type RoomserverInternalAPI struct {
 	*perform.Publisher
 	*perform.Backfiller
 	*perform.Forgetter
+	*perform.Upgrader
 	ProcessContext         *process.ProcessContext
 	DB                     storage.Database
 	Cfg                    *config.RoomServer
@@ -159,6 +160,10 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalA
 	r.Forgetter = &perform.Forgetter{
 		DB: r.DB,
 	}
+	r.Upgrader = &perform.Upgrader{
+		Cfg:    r.Cfg,
+		URSAPI: r,
+	}
 
 	if err := r.Inputer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start roomserver input API")

roomserver/internal/input/input.go

@@ -167,6 +167,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) {
 // will look to see if we have a worker for that room which has its
 // own consumer. If we don't, we'll start one.
 func (r *Inputer) Start() error {
+	prometheus.MustRegister(roomserverInputBackpressure, processRoomEventDuration)
 	_, err := r.JetStream.Subscribe(
 		"", // This is blank because we specified it in BindStream.
 		func(m *nats.Msg) {
@@ -421,10 +422,6 @@ func (r *Inputer) WriteOutputEvents(roomID string, updates []api.OutputEvent) er
 	return nil
 }
 
-func init() {
-	prometheus.MustRegister(roomserverInputBackpressure)
-}
-
 var roomserverInputBackpressure = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Namespace: "dendrite",

roomserver/internal/input/input_events.go

@@ -37,10 +37,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func init() {
-	prometheus.MustRegister(processRoomEventDuration)
-}
-
 // TODO: Does this value make sense?
 const MaximumMissingProcessingTime = time.Minute * 2
 

roomserver/internal/input/input_missing.go

@@ -613,12 +613,13 @@ func (t *missingStateReq) lookupMissingStateViaState(
 		return nil, err
 	}
 	// Check that the returned state is valid.
-	if err := state.Check(ctx, roomVersion, t.keys, nil); err != nil {
+	authEvents, stateEvents, err := state.Check(ctx, roomVersion, t.keys, nil)
+	if err != nil {
 		return nil, err
 	}
 	parsedState := &parsedRespState{
-		AuthEvents:  make([]*gomatrixserverlib.Event, len(state.AuthEvents)),
-		StateEvents: make([]*gomatrixserverlib.Event, len(state.StateEvents)),
+		AuthEvents:  authEvents,
+		StateEvents: stateEvents,
 	}
 	// Cache the results of this state lookup and deduplicate anything we already
 	// have in the cache, freeing up memory.

roomserver/internal/input/input_test.go

@@ -2,7 +2,6 @@ package input_test
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"testing"
 	"time"
@@ -12,30 +11,22 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
 )
 
-func psqlConnectionString() config.DataSource {
-	user := os.Getenv("POSTGRES_USER")
-	if user == "" {
-		user = "dendrite"
-	}
-	dbName := os.Getenv("POSTGRES_DB")
-	if dbName == "" {
-		dbName = "dendrite"
-	}
-	connStr := fmt.Sprintf(
-		"user=%s dbname=%s sslmode=disable", user, dbName,
-	)
-	password := os.Getenv("POSTGRES_PASSWORD")
-	if password != "" {
-		connStr += fmt.Sprintf(" password=%s", password)
-	}
-	host := os.Getenv("POSTGRES_HOST")
-	if host != "" {
-		connStr += fmt.Sprintf(" host=%s", host)
-	}
-	return config.DataSource(connStr)
+var js nats.JetStreamContext
+var jc *nats.Conn
+
+func TestMain(m *testing.M) {
+	var pc *process.ProcessContext
+	pc, js, jc = jetstream.PrepareForTests()
+	code := m.Run()
+	pc.ShutdownDendrite()
+	pc.WaitForComponentsToFinish()
+	os.Exit(code)
 }
 
 func TestSingleTransactionOnInput(t *testing.T) {
@@ -63,7 +54,7 @@ func TestSingleTransactionOnInput(t *testing.T) {
 	}
 	db, err := storage.Open(
 		&config.DatabaseOptions{
-			ConnectionString:   psqlConnectionString(),
+			ConnectionString:   "",
 			MaxOpenConnections: 1,
 			MaxIdleConnections: 1,
 		},
@@ -74,7 +65,9 @@ func TestSingleTransactionOnInput(t *testing.T) {
 		t.SkipNow()
 	}
 	inputter := &input.Inputer{
-		DB: db,
+		DB:         db,
+		JetStream:  js,
+		NATSClient: jc,
 	}
 	res := &api.InputRoomEventsResponse{}
 	inputter.InputRoomEvents(

roomserver/internal/perform/perform_upgrade.go

@@ -0,0 +1,709 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package perform
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+)
+
+type Upgrader struct {
+	Cfg    *config.RoomServer
+	URSAPI api.RoomserverInternalAPI
+}
+
+// fledglingEvent is a helper representation of an event used when creating many events in succession.
+type fledglingEvent struct {
+	Type     string      `json:"type"`
+	StateKey string      `json:"state_key"`
+	Content  interface{} `json:"content"`
+}
+
+// PerformRoomUpgrade upgrades a room from one version to another
+func (r *Upgrader) PerformRoomUpgrade(
+	ctx context.Context,
+	req *api.PerformRoomUpgradeRequest,
+	res *api.PerformRoomUpgradeResponse,
+) {
+	res.NewRoomID, res.Error = r.performRoomUpgrade(ctx, req)
+	if res.Error != nil {
+		res.NewRoomID = ""
+		logrus.WithContext(ctx).WithError(res.Error).Error("Room upgrade failed")
+	}
+}
+
+func (r *Upgrader) performRoomUpgrade(
+	ctx context.Context,
+	req *api.PerformRoomUpgradeRequest,
+) (string, *api.PerformError) {
+	roomID := req.RoomID
+	userID := req.UserID
+	evTime := time.Now()
+
+	// Return an immediate error if the room does not exist
+	if err := r.validateRoomExists(ctx, roomID); err != nil {
+		return "", &api.PerformError{
+			Code: api.PerformErrorNoRoom,
+			Msg:  "Error validating that the room exists",
+		}
+	}
+
+	// 1. Check if the user is authorized to actually perform the upgrade (can send m.room.tombstone)
+	if !r.userIsAuthorized(ctx, userID, roomID) {
+		return "", &api.PerformError{
+			Code: api.PerformErrorNotAllowed,
+			Msg:  "You don't have permission to upgrade the room, power level too low.",
+		}
+	}
+
+	// TODO (#267): Check room ID doesn't clash with an existing one, and we
+	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
+	newRoomID := fmt.Sprintf("!%s:%s", util.RandomString(16), r.Cfg.Matrix.ServerName)
+
+	// Get the existing room state for the old room.
+	oldRoomReq := &api.QueryLatestEventsAndStateRequest{
+		RoomID: roomID,
+	}
+	oldRoomRes := &api.QueryLatestEventsAndStateResponse{}
+	if err := r.URSAPI.QueryLatestEventsAndState(ctx, oldRoomReq, oldRoomRes); err != nil {
+		return "", &api.PerformError{
+			Msg: fmt.Sprintf("Failed to get latest state: %s", err),
+		}
+	}
+
+	// Make the tombstone event
+	tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, userID, roomID, newRoomID)
+	if pErr != nil {
+		return "", pErr
+	}
+
+	// Generate the initial events we need to send into the new room. This includes copied state events and bans
+	// as well as the power level events needed to set up the room
+	eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, userID, roomID, string(req.RoomVersion), tombstoneEvent)
+	if pErr != nil {
+		return "", pErr
+	}
+
+	// 5. Send the tombstone event to the old room (must do this before we set the new canonical_alias)
+	if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil {
+		return "", pErr
+	}
+
+	// Send the setup events to the new room
+	if pErr = r.sendInitialEvents(ctx, evTime, userID, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil {
+		return "", pErr
+	}
+
+	// If the old room was public, make sure the new one is too
+	if pErr = r.publishIfOldRoomWasPublic(ctx, roomID, newRoomID); pErr != nil {
+		return "", pErr
+	}
+
+	// If the old room had a canonical alias event, it should be deleted in the old room
+	if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, userID, roomID); pErr != nil {
+		return "", pErr
+	}
+
+	// 4. Move local aliases to the new room
+	if pErr = moveLocalAliases(ctx, roomID, newRoomID, userID, r.URSAPI); pErr != nil {
+		return "", pErr
+	}
+
+	// 6. Restrict power levels in the old room
+	if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, userID, roomID); pErr != nil {
+		return "", pErr
+	}
+
+	return newRoomID, nil
+}
+
+func (r *Upgrader) getRoomPowerLevels(ctx context.Context, roomID string) (*gomatrixserverlib.PowerLevelContent, *api.PerformError) {
+	oldPowerLevelsEvent := api.GetStateEvent(ctx, r.URSAPI, roomID, gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomPowerLevels,
+		StateKey:  "",
+	})
+	powerLevelContent, err := oldPowerLevelsEvent.PowerLevels()
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error()
+		return nil, &api.PerformError{
+			Msg: "powerLevel event was not actually a power level event",
+		}
+	}
+	return powerLevelContent, nil
+}
+
+func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.Time, userID, roomID string) *api.PerformError {
+	restrictedPowerLevelContent, pErr := r.getRoomPowerLevels(ctx, roomID)
+	if pErr != nil {
+		return pErr
+	}
+
+	// From: https://spec.matrix.org/v1.2/client-server-api/#server-behaviour-16
+	// If possible, the power levels in the old room should also be modified to
+	// prevent sending of events and inviting new users. For example, setting
+	// events_default and invite to the greater of 50 and users_default + 1.
+	restrictedDefaultPowerLevel := int64(50)
+	if restrictedPowerLevelContent.UsersDefault+1 > restrictedDefaultPowerLevel {
+		restrictedDefaultPowerLevel = restrictedPowerLevelContent.UsersDefault + 1
+	}
+	restrictedPowerLevelContent.EventsDefault = restrictedDefaultPowerLevel
+	restrictedPowerLevelContent.Invite = restrictedDefaultPowerLevel
+
+	restrictedPowerLevelsHeadered, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{
+		Type:     gomatrixserverlib.MRoomPowerLevels,
+		StateKey: "",
+		Content:  restrictedPowerLevelContent,
+	})
+	if resErr != nil {
+		if resErr.Code == api.PerformErrorNotAllowed {
+			util.GetLogger(ctx).WithField(logrus.ErrorKey, resErr).Warn("UpgradeRoom: Could not restrict power levels in old room")
+		} else {
+			return resErr
+		}
+	} else {
+		if resErr = r.sendHeaderedEvent(ctx, restrictedPowerLevelsHeadered); resErr != nil {
+			return resErr
+		}
+	}
+	return nil
+}
+
+func moveLocalAliases(ctx context.Context,
+	roomID, newRoomID, userID string,
+	URSAPI api.RoomserverInternalAPI) *api.PerformError {
+	var err error
+
+	aliasReq := api.GetAliasesForRoomIDRequest{RoomID: roomID}
+	aliasRes := api.GetAliasesForRoomIDResponse{}
+	if err = URSAPI.GetAliasesForRoomID(ctx, &aliasReq, &aliasRes); err != nil {
+		return &api.PerformError{
+			Msg: "Could not get aliases for old room",
+		}
+	}
+
+	for _, alias := range aliasRes.Aliases {
+		removeAliasReq := api.RemoveRoomAliasRequest{UserID: userID, Alias: alias}
+		removeAliasRes := api.RemoveRoomAliasResponse{}
+		if err = URSAPI.RemoveRoomAlias(ctx, &removeAliasReq, &removeAliasRes); err != nil {
+			return &api.PerformError{
+				Msg: "api.RemoveRoomAlias failed",
+			}
+		}
+
+		setAliasReq := api.SetRoomAliasRequest{UserID: userID, Alias: alias, RoomID: newRoomID}
+		setAliasRes := api.SetRoomAliasResponse{}
+		if err = URSAPI.SetRoomAlias(ctx, &setAliasReq, &setAliasRes); err != nil {
+			return &api.PerformError{
+				Msg: "api.SetRoomAlias failed",
+			}
+		}
+	}
+	return nil
+}
+
+func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, evTime time.Time, userID, roomID string) *api.PerformError {
+	for _, event := range oldRoom.StateEvents {
+		if event.Type() != gomatrixserverlib.MRoomCanonicalAlias || !event.StateKeyEquals("") {
+			continue
+		}
+		var aliasContent struct {
+			Alias      string   `json:"alias"`
+			AltAliases []string `json:"alt_aliases"`
+		}
+		if err := json.Unmarshal(event.Content(), &aliasContent); err != nil {
+			return &api.PerformError{
+				Msg: fmt.Sprintf("Failed to unmarshal canonical aliases: %s", err),
+			}
+		}
+		if aliasContent.Alias == "" && len(aliasContent.AltAliases) == 0 {
+			// There are no canonical aliases to clear, therefore do nothing.
+			return nil
+		}
+	}
+
+	emptyCanonicalAliasEvent, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{
+		Type:    gomatrixserverlib.MRoomCanonicalAlias,
+		Content: map[string]interface{}{},
+	})
+	if resErr != nil {
+		if resErr.Code == api.PerformErrorNotAllowed {
+			util.GetLogger(ctx).WithField(logrus.ErrorKey, resErr).Warn("UpgradeRoom: Could not set empty canonical alias event in old room")
+		} else {
+			return resErr
+		}
+	} else {
+		if resErr = r.sendHeaderedEvent(ctx, emptyCanonicalAliasEvent); resErr != nil {
+			return resErr
+		}
+	}
+	return nil
+}
+
+func (r *Upgrader) publishIfOldRoomWasPublic(ctx context.Context, roomID, newRoomID string) *api.PerformError {
+	// check if the old room was published
+	var pubQueryRes api.QueryPublishedRoomsResponse
+	err := r.URSAPI.QueryPublishedRooms(ctx, &api.QueryPublishedRoomsRequest{
+		RoomID: roomID,
+	}, &pubQueryRes)
+	if err != nil {
+		return &api.PerformError{
+			Msg: "QueryPublishedRooms failed",
+		}
+	}
+
+	// if the old room is published (was public), publish the new room
+	if len(pubQueryRes.RoomIDs) == 1 {
+		publishNewRoomAndUnpublishOldRoom(ctx, r.URSAPI, roomID, newRoomID)
+	}
+	return nil
+}
+
+func publishNewRoomAndUnpublishOldRoom(
+	ctx context.Context,
+	URSAPI api.RoomserverInternalAPI,
+	oldRoomID, newRoomID string,
+) {
+	// expose this room in the published room list
+	var pubNewRoomRes api.PerformPublishResponse
+	URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{
+		RoomID:     newRoomID,
+		Visibility: "public",
+	}, &pubNewRoomRes)
+	if pubNewRoomRes.Error != nil {
+		// treat as non-fatal since the room is already made by this point
+		util.GetLogger(ctx).WithError(pubNewRoomRes.Error).Error("failed to visibility:public")
+	}
+
+	var unpubOldRoomRes api.PerformPublishResponse
+	// remove the old room from the published room list
+	URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{
+		RoomID:     oldRoomID,
+		Visibility: "private",
+	}, &unpubOldRoomRes)
+	if unpubOldRoomRes.Error != nil {
+		// treat as non-fatal since the room is already made by this point
+		util.GetLogger(ctx).WithError(unpubOldRoomRes.Error).Error("failed to visibility:private")
+	}
+}
+
+func (r *Upgrader) validateRoomExists(ctx context.Context, roomID string) error {
+	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
+	verRes := api.QueryRoomVersionForRoomResponse{}
+	if err := r.URSAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
+		return &api.PerformError{
+			Code: api.PerformErrorNoRoom,
+			Msg:  "Room does not exist",
+		}
+	}
+	return nil
+}
+
+func (r *Upgrader) userIsAuthorized(ctx context.Context, userID, roomID string,
+) bool {
+	plEvent := api.GetStateEvent(ctx, r.URSAPI, roomID, gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomPowerLevels,
+		StateKey:  "",
+	})
+	if plEvent == nil {
+		return false
+	}
+	pl, err := plEvent.PowerLevels()
+	if err != nil {
+		return false
+	}
+	// Check for power level required to send tombstone event (marks the current room as obsolete),
+	// if not found, use the StateDefault power level
+	return pl.UserLevel(userID) >= pl.EventLevel("m.room.tombstone", true)
+}
+
+// nolint:gocyclo
+func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, userID, roomID, newVersion string, tombstoneEvent *gomatrixserverlib.HeaderedEvent) ([]fledglingEvent, *api.PerformError) {
+	state := make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent, len(oldRoom.StateEvents))
+	for _, event := range oldRoom.StateEvents {
+		if event.StateKey() == nil {
+			// This shouldn't ever happen, but better to be safe than sorry.
+			continue
+		}
+		if event.Type() == gomatrixserverlib.MRoomMember && !event.StateKeyEquals(userID) {
+			// With the exception of bans and invites which we do want to copy, we
+			// should ignore membership events that aren't our own, as event auth will
+			// prevent us from being able to create membership events on behalf of other
+			// users anyway unless they are invites or bans.
+			membership, err := event.Membership()
+			if err != nil {
+				continue
+			}
+			switch membership {
+			case gomatrixserverlib.Ban:
+			case gomatrixserverlib.Invite:
+			default:
+				continue
+			}
+		}
+		state[gomatrixserverlib.StateKeyTuple{EventType: event.Type(), StateKey: *event.StateKey()}] = event
+	}
+
+	// The following events are ones that we are going to override manually
+	// in the following section.
+	override := map[gomatrixserverlib.StateKeyTuple]struct{}{
+		{EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}:      {},
+		{EventType: gomatrixserverlib.MRoomMember, StateKey: userID}:  {},
+		{EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}: {},
+		{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}:   {},
+	}
+
+	// The overridden events are essential events that must be present in the
+	// old room state. Check that they are there.
+	for tuple := range override {
+		if _, ok := state[tuple]; !ok {
+			return nil, &api.PerformError{
+				Msg: fmt.Sprintf("Essential event of type %q state key %q is missing", tuple.EventType, tuple.StateKey),
+			}
+		}
+	}
+
+	oldCreateEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}]
+	oldMembershipEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomMember, StateKey: userID}]
+	oldPowerLevelsEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}]
+	oldJoinRulesEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}]
+
+	// Create the new room create event. Using a map here instead of CreateContent
+	// means that we preserve any other interesting fields that might be present
+	// in the create event (such as for the room types MSC).
+	newCreateContent := map[string]interface{}{}
+	_ = json.Unmarshal(oldCreateEvent.Content(), &newCreateContent)
+	newCreateContent["creator"] = userID
+	newCreateContent["room_version"] = newVersion
+	newCreateContent["predecessor"] = gomatrixserverlib.PreviousRoom{
+		EventID: tombstoneEvent.EventID(),
+		RoomID:  roomID,
+	}
+	newCreateEvent := fledglingEvent{
+		Type:     gomatrixserverlib.MRoomCreate,
+		StateKey: "",
+		Content:  newCreateContent,
+	}
+
+	// Now create the new membership event. Same rules apply as above, so
+	// that we preserve fields we don't otherwise know about. We'll always
+	// set the membership to join though, because that is necessary to auth
+	// the events after it.
+	newMembershipContent := map[string]interface{}{}
+	_ = json.Unmarshal(oldMembershipEvent.Content(), &newMembershipContent)
+	newMembershipContent["membership"] = gomatrixserverlib.Join
+	newMembershipEvent := fledglingEvent{
+		Type:     gomatrixserverlib.MRoomMember,
+		StateKey: userID,
+		Content:  newMembershipContent,
+	}
+
+	// We might need to temporarily give ourselves a higher power level
+	// than we had in the old room in order to be able to send all of
+	// the relevant state events. This function will return whether we
+	// had to override the power level events or not — if we did, we
+	// need to send the original power levels again later on.
+	powerLevelContent, err := oldPowerLevelsEvent.PowerLevels()
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error()
+		return nil, &api.PerformError{
+			Msg: "Power level event content was invalid",
+		}
+	}
+	tempPowerLevelsEvent, powerLevelsOverridden := createTemporaryPowerLevels(powerLevelContent, userID)
+
+	// Now do the join rules event, same as the create and membership
+	// events. We'll set a sane default of "invite" so that if the
+	// existing join rules contains garbage, the room can still be
+	// upgraded.
+	newJoinRulesContent := map[string]interface{}{
+		"join_rule": gomatrixserverlib.Invite, // sane default
+	}
+	_ = json.Unmarshal(oldJoinRulesEvent.Content(), &newJoinRulesContent)
+	newJoinRulesEvent := fledglingEvent{
+		Type:     gomatrixserverlib.MRoomJoinRules,
+		StateKey: "",
+		Content:  newJoinRulesContent,
+	}
+
+	eventsToMake := make([]fledglingEvent, 0, len(state))
+	eventsToMake = append(
+		eventsToMake, newCreateEvent, newMembershipEvent,
+		tempPowerLevelsEvent, newJoinRulesEvent,
+	)
+
+	// For some reason Sytest expects there to be a guest access event.
+	// Create one if it doesn't exist.
+	if _, ok := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomGuestAccess, StateKey: ""}]; !ok {
+		eventsToMake = append(eventsToMake, fledglingEvent{
+			Type: gomatrixserverlib.MRoomGuestAccess,
+			Content: map[string]string{
+				"guest_access": "forbidden",
+			},
+		})
+	}
+
+	// Duplicate all of the old state events into the new room.
+	for tuple, event := range state {
+		if _, ok := override[tuple]; ok {
+			// Don't duplicate events we have overridden already. They
+			// are already in `eventsToMake`.
+			continue
+		}
+		newEvent := fledglingEvent{
+			Type:     tuple.EventType,
+			StateKey: tuple.StateKey,
+		}
+		if err = json.Unmarshal(event.Content(), &newEvent.Content); err != nil {
+			logrus.WithError(err).Error("Failed to unmarshal old event")
+			continue
+		}
+		eventsToMake = append(eventsToMake, newEvent)
+	}
+
+	// If we sent a temporary power level event into the room before,
+	// override that now by restoring the original power levels.
+	if powerLevelsOverridden {
+		eventsToMake = append(eventsToMake, fledglingEvent{
+			Type:    gomatrixserverlib.MRoomPowerLevels,
+			Content: powerLevelContent,
+		})
+	}
+	return eventsToMake, nil
+}
+
+func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, userID, newRoomID, newVersion string, eventsToMake []fledglingEvent) *api.PerformError {
+	var err error
+	var builtEvents []*gomatrixserverlib.HeaderedEvent
+	authEvents := gomatrixserverlib.NewAuthEvents(nil)
+	for i, e := range eventsToMake {
+		depth := i + 1 // depth starts at 1
+
+		builder := gomatrixserverlib.EventBuilder{
+			Sender:   userID,
+			RoomID:   newRoomID,
+			Type:     e.Type,
+			StateKey: &e.StateKey,
+			Depth:    int64(depth),
+		}
+		err = builder.SetContent(e.Content)
+		if err != nil {
+			return &api.PerformError{
+				Msg: "builder.SetContent failed",
+			}
+		}
+		if i > 0 {
+			builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
+		}
+		var event *gomatrixserverlib.Event
+		event, err = r.buildEvent(&builder, &authEvents, evTime, gomatrixserverlib.RoomVersion(newVersion))
+		if err != nil {
+			return &api.PerformError{
+				Msg: "buildEvent failed",
+			}
+		}
+
+		if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil {
+			return &api.PerformError{
+				Msg: "gomatrixserverlib.Allowed failed",
+			}
+		}
+
+		// Add the event to the list of auth events
+		builtEvents = append(builtEvents, event.Headered(gomatrixserverlib.RoomVersion(newVersion)))
+		err = authEvents.AddEvent(event)
+		if err != nil {
+			return &api.PerformError{
+				Msg: "authEvents.AddEvent failed",
+			}
+		}
+	}
+
+	inputs := make([]api.InputRoomEvent, 0, len(builtEvents))
+	for _, event := range builtEvents {
+		inputs = append(inputs, api.InputRoomEvent{
+			Kind:         api.KindNew,
+			Event:        event,
+			Origin:       r.Cfg.Matrix.ServerName,
+			SendAsServer: api.DoNotSendToOtherServers,
+		})
+	}
+	if err = api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil {
+		return &api.PerformError{
+			Msg: "api.SendInputRoomEvents failed",
+		}
+	}
+	return nil
+}
+
+func (r *Upgrader) makeTombstoneEvent(
+	ctx context.Context,
+	evTime time.Time,
+	userID, roomID, newRoomID string,
+) (*gomatrixserverlib.HeaderedEvent, *api.PerformError) {
+	content := map[string]interface{}{
+		"body":             "This room has been replaced",
+		"replacement_room": newRoomID,
+	}
+	event := fledglingEvent{
+		Type:    "m.room.tombstone",
+		Content: content,
+	}
+	return r.makeHeaderedEvent(ctx, evTime, userID, roomID, event)
+}
+
+func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, userID, roomID string, event fledglingEvent) (*gomatrixserverlib.HeaderedEvent, *api.PerformError) {
+	builder := gomatrixserverlib.EventBuilder{
+		Sender:   userID,
+		RoomID:   roomID,
+		Type:     event.Type,
+		StateKey: &event.StateKey,
+	}
+	err := builder.SetContent(event.Content)
+	if err != nil {
+		return nil, &api.PerformError{
+			Msg: "builder.SetContent failed",
+		}
+	}
+	var queryRes api.QueryLatestEventsAndStateResponse
+	headeredEvent, err := eventutil.QueryAndBuildEvent(ctx, &builder, r.Cfg.Matrix, evTime, r.URSAPI, &queryRes)
+	if err == eventutil.ErrRoomNoExists {
+		return nil, &api.PerformError{
+			Code: api.PerformErrorNoRoom,
+			Msg:  "Room does not exist",
+		}
+	} else if e, ok := err.(gomatrixserverlib.BadJSONError); ok {
+		return nil, &api.PerformError{
+			Msg: e.Error(),
+		}
+	} else if e, ok := err.(gomatrixserverlib.EventValidationError); ok {
+		if e.Code == gomatrixserverlib.EventValidationTooLarge {
+			return nil, &api.PerformError{
+				Msg: e.Error(),
+			}
+		}
+		return nil, &api.PerformError{
+			Msg: e.Error(),
+		}
+	} else if err != nil {
+		return nil, &api.PerformError{
+			Msg: "eventutil.BuildEvent failed",
+		}
+	}
+	// check to see if this user can perform this operation
+	stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents))
+	for i := range queryRes.StateEvents {
+		stateEvents[i] = queryRes.StateEvents[i].Event
+	}
+	provider := gomatrixserverlib.NewAuthEvents(stateEvents)
+	if err = gomatrixserverlib.Allowed(headeredEvent.Event, &provider); err != nil {
+		return nil, &api.PerformError{
+			Code: api.PerformErrorNotAllowed,
+			Msg:  err.Error(), // TODO: Is this error string comprehensible to the client?
+		}
+	}
+
+	return headeredEvent, nil
+}
+
+func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelContent, userID string) (fledglingEvent, bool) {
+	// Work out what power level we need in order to be able to send events
+	// of all types into the room.
+	neededPowerLevel := powerLevelContent.StateDefault
+	for _, powerLevel := range powerLevelContent.Events {
+		if powerLevel > neededPowerLevel {
+			neededPowerLevel = powerLevel
+		}
+	}
+
+	// Make a copy of the existing power level content.
+	tempPowerLevelContent := *powerLevelContent
+	powerLevelsOverridden := false
+
+	// At this point, the "Users", "Events" and "Notifications" keys are all
+	// pointing to the map of the original PL content, so we will specifically
+	// override the users map with a new one and duplicate the values deeply,
+	// so that we can modify them without modifying the original.
+	tempPowerLevelContent.Users = make(map[string]int64, len(powerLevelContent.Users))
+	for key, value := range powerLevelContent.Users {
+		tempPowerLevelContent.Users[key] = value
+	}
+
+	// If the user who is upgrading the room doesn't already have sufficient
+	// power, then elevate their power levels.
+	if tempPowerLevelContent.UserLevel(userID) < neededPowerLevel {
+		tempPowerLevelContent.Users[userID] = neededPowerLevel
+		powerLevelsOverridden = true
+	}
+
+	// Then return the temporary power levels event.
+	return fledglingEvent{
+		Type:    gomatrixserverlib.MRoomPowerLevels,
+		Content: tempPowerLevelContent,
+	}, powerLevelsOverridden
+}
+
+func (r *Upgrader) sendHeaderedEvent(
+	ctx context.Context,
+	headeredEvent *gomatrixserverlib.HeaderedEvent,
+) *api.PerformError {
+	var inputs []api.InputRoomEvent
+	inputs = append(inputs, api.InputRoomEvent{
+		Kind:         api.KindNew,
+		Event:        headeredEvent,
+		Origin:       r.Cfg.Matrix.ServerName,
+		SendAsServer: api.DoNotSendToOtherServers,
+	})
+	if err := api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil {
+		return &api.PerformError{
+			Msg: "api.SendInputRoomEvents failed",
+		}
+	}
+
+	return nil
+}
+
+func (r *Upgrader) buildEvent(
+	builder *gomatrixserverlib.EventBuilder,
+	provider gomatrixserverlib.AuthEventProvider,
+	evTime time.Time,
+	roomVersion gomatrixserverlib.RoomVersion,
+) (*gomatrixserverlib.Event, error) {
+	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
+	if err != nil {
+		return nil, err
+	}
+	refs, err := eventsNeeded.AuthEventReferences(provider)
+	if err != nil {
+		return nil, err
+	}
+	builder.AuthEvents = refs
+	event, err := builder.Build(
+		evTime, r.Cfg.Matrix.ServerName, r.Cfg.Matrix.KeyID,
+		r.Cfg.Matrix.PrivateKey, roomVersion,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err)
+	}
+	return event, nil
+}

roomserver/internal/query/query.go

@@ -220,7 +220,7 @@ func (r *Queryer) QueryMembershipsForRoom(
 	if request.Sender == "" {
 		var events []types.Event
 		var eventNIDs []types.EventNID
-		eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, request.JoinedOnly, false)
+		eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, request.JoinedOnly, request.LocalOnly)
 		if err != nil {
 			return fmt.Errorf("r.DB.GetMembershipEventNIDsForRoom: %w", err)
 		}

roomserver/inthttp/client.go

@@ -32,6 +32,7 @@ const (
 	RoomserverPerformInvitePath      = "/roomserver/performInvite"
 	RoomserverPerformPeekPath        = "/roomserver/performPeek"
 	RoomserverPerformUnpeekPath      = "/roomserver/performUnpeek"
+	RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade"
 	RoomserverPerformJoinPath        = "/roomserver/performJoin"
 	RoomserverPerformLeavePath       = "/roomserver/performLeave"
 	RoomserverPerformBackfillPath    = "/roomserver/performBackfill"
@@ -252,6 +253,23 @@ func (h *httpRoomserverInternalAPI) PerformUnpeek(
 	}
 }
 
+func (h *httpRoomserverInternalAPI) PerformRoomUpgrade(
+	ctx context.Context,
+	request *api.PerformRoomUpgradeRequest,
+	response *api.PerformRoomUpgradeResponse,
+) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformRoomUpgrade")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverPerformRoomUpgradePath
+	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	if err != nil {
+		response.Error = &api.PerformError{
+			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
+		}
+	}
+}
+
 func (h *httpRoomserverInternalAPI) PerformLeave(
 	ctx context.Context,
 	request *api.PerformLeaveRequest,

roomserver/inthttp/server.go

@@ -96,6 +96,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverPerformRoomUpgradePath,
+		httputil.MakeInternalAPI("performRoomUpgrade", func(req *http.Request) util.JSONResponse {
+			var request api.PerformRoomUpgradeRequest
+			var response api.PerformRoomUpgradeResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			r.PerformRoomUpgrade(req.Context(), &request, &response)
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	internalAPIMux.Handle(RoomserverPerformPublishPath,
 		httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse {
 			var request api.PerformPublishRequest

setup/config/config_federationapi.go

@@ -12,13 +12,6 @@ type FederationAPI struct {
 	// send transactions to remote servers.
 	Database DatabaseOptions `yaml:"database"`
 
-	// List of paths to X509 certificates used by the external federation listeners.
-	// These are used to calculate the TLS fingerprints to publish for this server.
-	// Other matrix servers talking to this server will expect the x509 certificate
-	// to match one of these certificates.
-	// The certificates should be in PEM format.
-	FederationCertificatePaths []Path `yaml:"federation_certificates"`
-
 	// Federation failure threshold. How many consecutive failures that we should
 	// tolerate when sending federation requests to a specific server. The backoff
 	// is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc.
@@ -57,8 +50,6 @@ func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 		checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
 	}
 	checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
-	// TODO: not applicable always, e.g. in demos
-	//checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths)))
 }
 
 // The config for setting a proxy to use for server->server requests

setup/config/config_global.go

@@ -41,6 +41,9 @@ type Global struct {
 	// to other servers and the federation API will not be exposed.
 	DisableFederation bool `yaml:"disable_federation"`
 
+	// Configures the handling of presence events.
+	Presence PresenceOptions `yaml:"presence"`
+
 	// List of domains that the server will trust as identity servers to
 	// verify third-party identifiers.
 	// Defaults to an empty array.
@@ -225,3 +228,11 @@ func (c *DNSCacheOptions) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkPositive(configErrs, "cache_size", int64(c.CacheSize))
 	checkPositive(configErrs, "cache_lifetime", int64(c.CacheLifetime))
 }
+
+// PresenceOptions defines possible configurations for presence events.
+type PresenceOptions struct {
+	// Whether inbound presence events are allowed
+	EnableInbound bool `yaml:"enable_inbound"`
+	// Whether outbound presence events are allowed
+	EnableOutbound bool `yaml:"enable_outbound"`
+}

setup/config/config_test.go

@@ -107,18 +107,6 @@ federation_api:
     connect: http://localhost:7772
   external_api:
     listen: http://[::]:8072
-  federation_certificates: []
-federation_sender:
-  internal_api:
-    listen: http://localhost:7775
-    connect: http://localhost:7775
-  database:
-    connection_string: file:federationapi.db
-    max_open_conns: 100
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-  send_max_retries: 16
-  disable_tls_validation: false
 key_server:
   internal_api:
     listen: http://localhost:7779

setup/jetstream/nats.go

@@ -13,12 +13,22 @@ import (
 	"github.com/sirupsen/logrus"
 
 	natsserver "github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
 	natsclient "github.com/nats-io/nats.go"
 )
 
 var natsServer *natsserver.Server
 var natsServerMutex sync.Mutex
 
+func PrepareForTests() (*process.ProcessContext, nats.JetStreamContext, *nats.Conn) {
+	cfg := &config.Dendrite{}
+	cfg.Defaults(true)
+	cfg.Global.JetStream.InMemory = true
+	pc := process.NewProcessContext()
+	js, jc := Prepare(pc, &cfg.Global.JetStream)
+	return pc, js, jc
+}
+
 func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) {
 	// check if we need an in-process NATS Server
 	if len(cfg.Addresses) != 0 {

setup/jetstream/streams.go

@@ -25,6 +25,8 @@ var (
 	OutputReceiptEvent      = "OutputReceiptEvent"
 	OutputStreamEvent       = "OutputStreamEvent"
 	OutputReadUpdate        = "OutputReadUpdate"
+	RequestPresence         = "GetPresence"
+	OutputPresenceEvent     = "OutputPresenceEvent"
 )
 
 var safeCharacters = regexp.MustCompile("[^A-Za-z0-9$]+")
@@ -89,4 +91,10 @@ var streams = []*nats.StreamConfig{
 		Retention: nats.InterestPolicy,
 		Storage:   nats.FileStorage,
 	},
+	{
+		Name:      OutputPresenceEvent,
+		Retention: nats.InterestPolicy,
+		Storage:   nats.MemoryStorage,
+		MaxAge:    time.Minute * 5,
+	},
 }

show-expected-fail-tests.sh

@@ -89,17 +89,17 @@ if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then
 fi
 
 if [ -n "${tests_to_add}" ]; then
-	echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:"
-    echo "\`\`\`"
-    echo -e "${tests_to_add}"
-    echo "\`\`\`"
+	echo "::error::The following tests passed but are not present in \`$2\`. Please append them to the file:"
+	echo "::group::Passing tests"
+  echo -e "${tests_to_add}"
+  echo "::endgroup::"
 fi
 
 if [ -n "${already_in_whitelist}" ]; then
-	echo "**WARN**: Tests in the whitelist still marked as **expected fail**:"
-    echo "\`\`\`"
-    echo -e "${already_in_whitelist}"
-    echo "\`\`\`"
+	echo "::warning::Tests in the whitelist still marked as **expected fail**:"
+	echo "::group::Still marked as expected fail"
+  echo -e "${already_in_whitelist}"
+  echo "::endgroup::"
 fi
 
 exit ${fail_build}

syncapi/consumers/clientapi.go

@@ -119,6 +119,15 @@ func (s *OutputClientDataConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 		return false
 	}
 
+	if output.IgnoredUsers != nil {
+		if err := s.db.UpdateIgnoresForUser(ctx, userID, output.IgnoredUsers); err != nil {
+			log.WithError(err).WithFields(logrus.Fields{
+				"user_id": userID,
+			}).Errorf("Failed to update ignored users")
+			sentry.CaptureException(err)
+		}
+	}
+
 	s.stream.Advance(streamPos)
 	s.notifier.OnNewAccountData(userID, types.StreamingToken{AccountDataPosition: streamPos})
 

syncapi/consumers/presence.go

@@ -0,0 +1,158 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/syncapi/notifier"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
+)
+
+// OutputTypingEventConsumer consumes events that originated in the EDU server.
+type PresenceConsumer struct {
+	ctx           context.Context
+	jetstream     nats.JetStreamContext
+	nats          *nats.Conn
+	durable       string
+	requestTopic  string
+	presenceTopic string
+	db            storage.Database
+	stream        types.StreamProvider
+	notifier      *notifier.Notifier
+	deviceAPI     api.UserDeviceAPI
+	cfg           *config.SyncAPI
+}
+
+// NewPresenceConsumer creates a new PresenceConsumer.
+// Call Start() to begin consuming events.
+func NewPresenceConsumer(
+	process *process.ProcessContext,
+	cfg *config.SyncAPI,
+	js nats.JetStreamContext,
+	nats *nats.Conn,
+	db storage.Database,
+	notifier *notifier.Notifier,
+	stream types.StreamProvider,
+	deviceAPI api.UserDeviceAPI,
+) *PresenceConsumer {
+	return &PresenceConsumer{
+		ctx:           process.Context(),
+		nats:          nats,
+		jetstream:     js,
+		durable:       cfg.Matrix.JetStream.Durable("SyncAPIPresenceConsumer"),
+		presenceTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		requestTopic:  cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence),
+		db:            db,
+		notifier:      notifier,
+		stream:        stream,
+		deviceAPI:     deviceAPI,
+		cfg:           cfg,
+	}
+}
+
+// Start consuming typing events.
+func (s *PresenceConsumer) Start() error {
+	// Normal NATS subscription, used by Request/Reply
+	_, err := s.nats.Subscribe(s.requestTopic, func(msg *nats.Msg) {
+		userID := msg.Header.Get(jetstream.UserID)
+		presence, err := s.db.GetPresence(context.Background(), userID)
+		m := &nats.Msg{
+			Header: nats.Header{},
+		}
+		if err != nil {
+			m.Header.Set("error", err.Error())
+			if err = msg.RespondMsg(m); err != nil {
+				logrus.WithError(err).Error("Unable to respond to messages")
+			}
+			return
+		}
+
+		deviceRes := api.QueryDevicesResponse{}
+		if err = s.deviceAPI.QueryDevices(s.ctx, &api.QueryDevicesRequest{UserID: userID}, &deviceRes); err != nil {
+			m.Header.Set("error", err.Error())
+			if err = msg.RespondMsg(m); err != nil {
+				logrus.WithError(err).Error("Unable to respond to messages")
+			}
+			return
+		}
+
+		for i := range deviceRes.Devices {
+			if int64(presence.LastActiveTS) < deviceRes.Devices[i].LastSeenTS {
+				presence.LastActiveTS = gomatrixserverlib.Timestamp(deviceRes.Devices[i].LastSeenTS)
+			}
+		}
+
+		m.Header.Set(jetstream.UserID, presence.UserID)
+		m.Header.Set("presence", presence.ClientFields.Presence)
+		m.Header.Set("status_msg", *presence.ClientFields.StatusMsg)
+		m.Header.Set("last_active_ts", strconv.Itoa(int(presence.LastActiveTS)))
+
+		if err = msg.RespondMsg(m); err != nil {
+			logrus.WithError(err).Error("Unable to respond to messages")
+			return
+		}
+	})
+	if err != nil {
+		return err
+	}
+	if !s.cfg.Matrix.Presence.EnableInbound && !s.cfg.Matrix.Presence.EnableOutbound {
+		return nil
+	}
+	return jetstream.JetStreamConsumer(
+		s.ctx, s.jetstream, s.presenceTopic, s.durable, s.onMessage,
+		nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(),
+	)
+}
+
+func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+	userID := msg.Header.Get(jetstream.UserID)
+	presence := msg.Header.Get("presence")
+	timestamp := msg.Header.Get("last_active_ts")
+	fromSync, _ := strconv.ParseBool(msg.Header.Get("from_sync"))
+
+	logrus.Debugf("syncAPI received presence event: %+v", msg.Header)
+
+	ts, err := strconv.Atoi(timestamp)
+	if err != nil {
+		return true
+	}
+
+	var statusMsg *string = nil
+	if data, ok := msg.Header["status_msg"]; ok && len(data) > 0 {
+		newMsg := msg.Header.Get("status_msg")
+		statusMsg = &newMsg
+	}
+	// OK is already checked, so no need to do it again
+	p, _ := types.PresenceFromString(presence)
+	pos, err := s.db.UpdatePresence(ctx, userID, p, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync)
+	if err != nil {
+		return true
+	}
+
+	s.stream.Advance(pos)
+	s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID)
+
+	return true
+}

syncapi/notifier/notifier.go

@@ -25,40 +25,53 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// NOTE: ALL FUNCTIONS IN THIS FILE PREFIXED WITH _ ARE NOT THREAD-SAFE
+// AND MUST ONLY BE CALLED WHEN THE NOTIFIER LOCK IS HELD!
+
 // Notifier will wake up sleeping requests when there is some new data.
 // It does not tell requests what that data is, only the sync position which
 // they can use to get at it. This is done to prevent races whereby we tell the caller
 // the event, but the token has already advanced by the time they fetch it, resulting
 // in missed events.
 type Notifier struct {
+	lock *sync.RWMutex
 	// A map of RoomID => Set<UserID> : Must only be accessed by the OnNewEvent goroutine
-	roomIDToJoinedUsers map[string]userIDSet
+	roomIDToJoinedUsers map[string]*userIDSet
 	// A map of RoomID => Set<UserID> : Must only be accessed by the OnNewEvent goroutine
 	roomIDToPeekingDevices map[string]peekingDeviceSet
-	// Protects currPos and userStreams.
-	streamLock *sync.Mutex
 	// The latest sync position
 	currPos types.StreamingToken
 	// A map of user_id => device_id => UserStream which can be used to wake a given user's /sync request.
 	userDeviceStreams map[string]map[string]*UserDeviceStream
 	// The last time we cleaned out stale entries from the userStreams map
 	lastCleanUpTime time.Time
+	// This map is reused to prevent allocations and GC pressure in SharedUsers.
+	_sharedUserMap map[string]struct{}
 }
 
 // NewNotifier creates a new notifier set to the given sync position.
 // In order for this to be of any use, the Notifier needs to be told all rooms and
 // the joined users within each of them by calling Notifier.Load(*storage.SyncServerDatabase).
-func NewNotifier(currPos types.StreamingToken) *Notifier {
+func NewNotifier() *Notifier {
 	return &Notifier{
-		currPos:                currPos,
-		roomIDToJoinedUsers:    make(map[string]userIDSet),
+		roomIDToJoinedUsers:    make(map[string]*userIDSet),
 		roomIDToPeekingDevices: make(map[string]peekingDeviceSet),
 		userDeviceStreams:      make(map[string]map[string]*UserDeviceStream),
-		streamLock:             &sync.Mutex{},
+		lock:                   &sync.RWMutex{},
 		lastCleanUpTime:        time.Now(),
+		_sharedUserMap:         map[string]struct{}{},
 	}
 }
 
+// SetCurrentPosition sets the current streaming positions.
+// This must be called directly after NewNotifier and initialising the streams.
+func (n *Notifier) SetCurrentPosition(currPos types.StreamingToken) {
+	n.lock.Lock()
+	defer n.lock.Unlock()
+
+	n.currPos = currPos
+}
+
 // OnNewEvent is called when a new event is received from the room server. Must only be
 // called from a single goroutine, to avoid races between updates which could set the
 // current sync position incorrectly.
@@ -75,17 +88,16 @@ func (n *Notifier) OnNewEvent(
 ) {
 	// update the current position then notify relevant /sync streams.
 	// This needs to be done PRIOR to waking up users as they will read this value.
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
-
+	n.lock.Lock()
+	defer n.lock.Unlock()
 	n.currPos.ApplyUpdates(posUpdate)
-	n.removeEmptyUserStreams()
+	n._removeEmptyUserStreams()
 
 	if ev != nil {
 		// Map this event's room_id to a list of joined users, and wake them up.
-		usersToNotify := n.joinedUsers(ev.RoomID())
+		usersToNotify := n._joinedUsers(ev.RoomID())
 		// Map this event's room_id to a list of peeking devices, and wake them up.
-		peekingDevicesToNotify := n.PeekingDevices(ev.RoomID())
+		peekingDevicesToNotify := n._peekingDevices(ev.RoomID())
 		// If this is an invite, also add in the invitee to this list.
 		if ev.Type() == "m.room.member" && ev.StateKey() != nil {
 			targetUserID := *ev.StateKey()
@@ -103,20 +115,20 @@ func (n *Notifier) OnNewEvent(
 					// Manually append the new user's ID so they get notified
 					// along all members in the room
 					usersToNotify = append(usersToNotify, targetUserID)
-					n.addJoinedUser(ev.RoomID(), targetUserID)
+					n._addJoinedUser(ev.RoomID(), targetUserID)
 				case gomatrixserverlib.Leave:
 					fallthrough
 				case gomatrixserverlib.Ban:
-					n.removeJoinedUser(ev.RoomID(), targetUserID)
+					n._removeJoinedUser(ev.RoomID(), targetUserID)
 				}
 			}
 		}
 
-		n.wakeupUsers(usersToNotify, peekingDevicesToNotify, n.currPos)
+		n._wakeupUsers(usersToNotify, peekingDevicesToNotify, n.currPos)
 	} else if roomID != "" {
-		n.wakeupUsers(n.joinedUsers(roomID), n.PeekingDevices(roomID), n.currPos)
+		n._wakeupUsers(n._joinedUsers(roomID), n._peekingDevices(roomID), n.currPos)
 	} else if len(userIDs) > 0 {
-		n.wakeupUsers(userIDs, nil, n.currPos)
+		n._wakeupUsers(userIDs, nil, n.currPos)
 	} else {
 		log.WithFields(log.Fields{
 			"posUpdate": posUpdate.String,
@@ -127,22 +139,22 @@ func (n *Notifier) OnNewEvent(
 func (n *Notifier) OnNewAccountData(
 	userID string, posUpdate types.StreamingToken,
 ) {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
 	n.currPos.ApplyUpdates(posUpdate)
-	n.wakeupUsers([]string{userID}, nil, posUpdate)
+	n._wakeupUsers([]string{userID}, nil, posUpdate)
 }
 
 func (n *Notifier) OnNewPeek(
 	roomID, userID, deviceID string,
 	posUpdate types.StreamingToken,
 ) {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
 	n.currPos.ApplyUpdates(posUpdate)
-	n.addPeekingDevice(roomID, userID, deviceID)
+	n._addPeekingDevice(roomID, userID, deviceID)
 
 	// we don't wake up devices here given the roomserver consumer will do this shortly afterwards
 	// by calling OnNewEvent.
@@ -152,11 +164,11 @@ func (n *Notifier) OnRetirePeek(
 	roomID, userID, deviceID string,
 	posUpdate types.StreamingToken,
 ) {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
 	n.currPos.ApplyUpdates(posUpdate)
-	n.removePeekingDevice(roomID, userID, deviceID)
+	n._removePeekingDevice(roomID, userID, deviceID)
 
 	// we don't wake up devices here given the roomserver consumer will do this shortly afterwards
 	// by calling OnRetireEvent.
@@ -166,11 +178,11 @@ func (n *Notifier) OnNewSendToDevice(
 	userID string, deviceIDs []string,
 	posUpdate types.StreamingToken,
 ) {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
 	n.currPos.ApplyUpdates(posUpdate)
-	n.wakeupUserDevice(userID, deviceIDs, n.currPos)
+	n._wakeupUserDevice(userID, deviceIDs, n.currPos)
 }
 
 // OnNewReceipt updates the current position
@@ -178,11 +190,11 @@ func (n *Notifier) OnNewTyping(
 	roomID string,
 	posUpdate types.StreamingToken,
 ) {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
 	n.currPos.ApplyUpdates(posUpdate)
-	n.wakeupUsers(n.joinedUsers(roomID), nil, n.currPos)
+	n._wakeupUsers(n._joinedUsers(roomID), nil, n.currPos)
 }
 
 // OnNewReceipt updates the current position
@@ -190,42 +202,96 @@ func (n *Notifier) OnNewReceipt(
 	roomID string,
 	posUpdate types.StreamingToken,
 ) {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
 	n.currPos.ApplyUpdates(posUpdate)
-	n.wakeupUsers(n.joinedUsers(roomID), nil, n.currPos)
+	n._wakeupUsers(n._joinedUsers(roomID), nil, n.currPos)
 }
 
 func (n *Notifier) OnNewKeyChange(
 	posUpdate types.StreamingToken, wakeUserID, keyChangeUserID string,
 ) {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
 	n.currPos.ApplyUpdates(posUpdate)
-	n.wakeupUsers([]string{wakeUserID}, nil, n.currPos)
+	n._wakeupUsers([]string{wakeUserID}, nil, n.currPos)
 }
 
 func (n *Notifier) OnNewInvite(
 	posUpdate types.StreamingToken, wakeUserID string,
 ) {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
 	n.currPos.ApplyUpdates(posUpdate)
-	n.wakeupUsers([]string{wakeUserID}, nil, n.currPos)
+	n._wakeupUsers([]string{wakeUserID}, nil, n.currPos)
 }
 
 func (n *Notifier) OnNewNotificationData(
 	userID string,
 	posUpdate types.StreamingToken,
 ) {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
 	n.currPos.ApplyUpdates(posUpdate)
-	n.wakeupUsers([]string{userID}, nil, n.currPos)
+	n._wakeupUsers([]string{userID}, nil, n.currPos)
+}
+
+func (n *Notifier) OnNewPresence(
+	posUpdate types.StreamingToken, userID string,
+) {
+	n.lock.Lock()
+	defer n.lock.Unlock()
+
+	n.currPos.ApplyUpdates(posUpdate)
+	sharedUsers := n._sharedUsers(userID)
+	sharedUsers = append(sharedUsers, userID)
+
+	n._wakeupUsers(sharedUsers, nil, n.currPos)
+}
+
+func (n *Notifier) SharedUsers(userID string) []string {
+	n.lock.RLock()
+	defer n.lock.RUnlock()
+	return n._sharedUsers(userID)
+}
+
+func (n *Notifier) _sharedUsers(userID string) []string {
+	n._sharedUserMap[userID] = struct{}{}
+	for roomID, users := range n.roomIDToJoinedUsers {
+		if ok := users.isIn(userID); !ok {
+			continue
+		}
+		for _, userID := range n._joinedUsers(roomID) {
+			n._sharedUserMap[userID] = struct{}{}
+		}
+	}
+	sharedUsers := make([]string, 0, len(n._sharedUserMap)+1)
+	for userID := range n._sharedUserMap {
+		sharedUsers = append(sharedUsers, userID)
+		delete(n._sharedUserMap, userID)
+	}
+	return sharedUsers
+}
+
+func (n *Notifier) IsSharedUser(userA, userB string) bool {
+	n.lock.RLock()
+	defer n.lock.RUnlock()
+	var okA, okB bool
+	for _, users := range n.roomIDToJoinedUsers {
+		okA = users.isIn(userA)
+		if !okA {
+			continue
+		}
+		okB = users.isIn(userB)
+		if okA && okB {
+			return true
+		}
+	}
+	return false
 }
 
 // GetListener returns a UserStreamListener that can be used to wait for
@@ -240,16 +306,18 @@ func (n *Notifier) GetListener(req types.SyncRequest) UserDeviceStreamListener {
 	// TODO: v1 /events 'peeking' has an 'explicit room ID' which is also tracked,
 	//       but given we don't do /events, let's pretend it doesn't exist.
 
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
-	n.removeEmptyUserStreams()
+	n._removeEmptyUserStreams()
 
-	return n.fetchUserDeviceStream(req.Device.UserID, req.Device.ID, true).GetListener(req.Context)
+	return n._fetchUserDeviceStream(req.Device.UserID, req.Device.ID, true).GetListener(req.Context)
 }
 
 // Load the membership states required to notify users correctly.
 func (n *Notifier) Load(ctx context.Context, db storage.Database) error {
+	n.lock.Lock()
+	defer n.lock.Unlock()
 	roomToUsers, err := db.AllJoinedUsersInRooms(ctx)
 	if err != nil {
 		return err
@@ -267,8 +335,8 @@ func (n *Notifier) Load(ctx context.Context, db storage.Database) error {
 
 // CurrentPosition returns the current sync position
 func (n *Notifier) CurrentPosition() types.StreamingToken {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.RLock()
+	defer n.lock.RUnlock()
 
 	return n.currPos
 }
@@ -280,11 +348,12 @@ func (n *Notifier) setUsersJoinedToRooms(roomIDToUserIDs map[string][]string) {
 	// This is just the bulk form of addJoinedUser
 	for roomID, userIDs := range roomIDToUserIDs {
 		if _, ok := n.roomIDToJoinedUsers[roomID]; !ok {
-			n.roomIDToJoinedUsers[roomID] = make(userIDSet)
+			n.roomIDToJoinedUsers[roomID] = newUserIDSet(len(userIDs))
 		}
 		for _, userID := range userIDs {
 			n.roomIDToJoinedUsers[roomID].add(userID)
 		}
+		n.roomIDToJoinedUsers[roomID].precompute()
 	}
 }
 
@@ -295,7 +364,7 @@ func (n *Notifier) setPeekingDevices(roomIDToPeekingDevices map[string][]types.P
 	// This is just the bulk form of addPeekingDevice
 	for roomID, peekingDevices := range roomIDToPeekingDevices {
 		if _, ok := n.roomIDToPeekingDevices[roomID]; !ok {
-			n.roomIDToPeekingDevices[roomID] = make(peekingDeviceSet)
+			n.roomIDToPeekingDevices[roomID] = make(peekingDeviceSet, len(peekingDevices))
 		}
 		for _, peekingDevice := range peekingDevices {
 			n.roomIDToPeekingDevices[roomID].add(peekingDevice)
@@ -303,11 +372,11 @@ func (n *Notifier) setPeekingDevices(roomIDToPeekingDevices map[string][]types.P
 	}
 }
 
-// wakeupUsers will wake up the sync strems for all of the devices for all of the
+// _wakeupUsers will wake up the sync strems for all of the devices for all of the
 // specified user IDs, and also the specified peekingDevices
-func (n *Notifier) wakeupUsers(userIDs []string, peekingDevices []types.PeekingDevice, newPos types.StreamingToken) {
+func (n *Notifier) _wakeupUsers(userIDs []string, peekingDevices []types.PeekingDevice, newPos types.StreamingToken) {
 	for _, userID := range userIDs {
-		for _, stream := range n.fetchUserStreams(userID) {
+		for _, stream := range n._fetchUserStreams(userID) {
 			if stream == nil {
 				continue
 			}
@@ -317,28 +386,27 @@ func (n *Notifier) wakeupUsers(userIDs []string, peekingDevices []types.PeekingD
 
 	for _, peekingDevice := range peekingDevices {
 		// TODO: don't bother waking up for devices whose users we already woke up
-		if stream := n.fetchUserDeviceStream(peekingDevice.UserID, peekingDevice.DeviceID, false); stream != nil {
+		if stream := n._fetchUserDeviceStream(peekingDevice.UserID, peekingDevice.DeviceID, false); stream != nil {
 			stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream
 		}
 	}
 }
 
-// wakeupUserDevice will wake up the sync stream for a specific user device. Other
+// _wakeupUserDevice will wake up the sync stream for a specific user device. Other
 // device streams will be left alone.
 // nolint:unused
-func (n *Notifier) wakeupUserDevice(userID string, deviceIDs []string, newPos types.StreamingToken) {
+func (n *Notifier) _wakeupUserDevice(userID string, deviceIDs []string, newPos types.StreamingToken) {
 	for _, deviceID := range deviceIDs {
-		if stream := n.fetchUserDeviceStream(userID, deviceID, false); stream != nil {
+		if stream := n._fetchUserDeviceStream(userID, deviceID, false); stream != nil {
 			stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream
 		}
 	}
 }
 
-// fetchUserDeviceStream retrieves a stream unique to the given device. If makeIfNotExists is true,
+// _fetchUserDeviceStream retrieves a stream unique to the given device. If makeIfNotExists is true,
 // a stream will be made for this device if one doesn't exist and it will be returned. This
 // function does not wait for data to be available on the stream.
-// NB: Callers should have locked the mutex before calling this function.
-func (n *Notifier) fetchUserDeviceStream(userID, deviceID string, makeIfNotExists bool) *UserDeviceStream {
+func (n *Notifier) _fetchUserDeviceStream(userID, deviceID string, makeIfNotExists bool) *UserDeviceStream {
 	_, ok := n.userDeviceStreams[userID]
 	if !ok {
 		if !makeIfNotExists {
@@ -359,57 +427,58 @@ func (n *Notifier) fetchUserDeviceStream(userID, deviceID string, makeIfNotExist
 	return stream
 }
 
-// fetchUserStreams retrieves all streams for the given user. If makeIfNotExists is true,
+// _fetchUserStreams retrieves all streams for the given user. If makeIfNotExists is true,
 // a stream will be made for this user if one doesn't exist and it will be returned. This
 // function does not wait for data to be available on the stream.
-// NB: Callers should have locked the mutex before calling this function.
-func (n *Notifier) fetchUserStreams(userID string) []*UserDeviceStream {
+func (n *Notifier) _fetchUserStreams(userID string) []*UserDeviceStream {
 	user, ok := n.userDeviceStreams[userID]
 	if !ok {
 		return []*UserDeviceStream{}
 	}
-	streams := []*UserDeviceStream{}
+	streams := make([]*UserDeviceStream, 0, len(user))
 	for _, stream := range user {
 		streams = append(streams, stream)
 	}
 	return streams
 }
 
-// Not thread-safe: must be called on the OnNewEvent goroutine only
-func (n *Notifier) addJoinedUser(roomID, userID string) {
+func (n *Notifier) _addJoinedUser(roomID, userID string) {
 	if _, ok := n.roomIDToJoinedUsers[roomID]; !ok {
-		n.roomIDToJoinedUsers[roomID] = make(userIDSet)
+		n.roomIDToJoinedUsers[roomID] = newUserIDSet(8)
 	}
 	n.roomIDToJoinedUsers[roomID].add(userID)
+	n.roomIDToJoinedUsers[roomID].precompute()
 }
 
-// Not thread-safe: must be called on the OnNewEvent goroutine only
-func (n *Notifier) removeJoinedUser(roomID, userID string) {
+func (n *Notifier) _removeJoinedUser(roomID, userID string) {
 	if _, ok := n.roomIDToJoinedUsers[roomID]; !ok {
-		n.roomIDToJoinedUsers[roomID] = make(userIDSet)
+		n.roomIDToJoinedUsers[roomID] = newUserIDSet(8)
 	}
 	n.roomIDToJoinedUsers[roomID].remove(userID)
+	n.roomIDToJoinedUsers[roomID].precompute()
 }
 
-// Not thread-safe: must be called on the OnNewEvent goroutine only
-func (n *Notifier) joinedUsers(roomID string) (userIDs []string) {
+func (n *Notifier) JoinedUsers(roomID string) (userIDs []string) {
+	n.lock.RLock()
+	defer n.lock.RUnlock()
+	return n._joinedUsers(roomID)
+}
+
+func (n *Notifier) _joinedUsers(roomID string) (userIDs []string) {
 	if _, ok := n.roomIDToJoinedUsers[roomID]; !ok {
 		return
 	}
 	return n.roomIDToJoinedUsers[roomID].values()
 }
 
-// Not thread-safe: must be called on the OnNewEvent goroutine only
-func (n *Notifier) addPeekingDevice(roomID, userID, deviceID string) {
+func (n *Notifier) _addPeekingDevice(roomID, userID, deviceID string) {
 	if _, ok := n.roomIDToPeekingDevices[roomID]; !ok {
 		n.roomIDToPeekingDevices[roomID] = make(peekingDeviceSet)
 	}
 	n.roomIDToPeekingDevices[roomID].add(types.PeekingDevice{UserID: userID, DeviceID: deviceID})
 }
 
-// Not thread-safe: must be called on the OnNewEvent goroutine only
-// nolint:unused
-func (n *Notifier) removePeekingDevice(roomID, userID, deviceID string) {
+func (n *Notifier) _removePeekingDevice(roomID, userID, deviceID string) {
 	if _, ok := n.roomIDToPeekingDevices[roomID]; !ok {
 		n.roomIDToPeekingDevices[roomID] = make(peekingDeviceSet)
 	}
@@ -417,22 +486,26 @@ func (n *Notifier) removePeekingDevice(roomID, userID, deviceID string) {
 	n.roomIDToPeekingDevices[roomID].remove(types.PeekingDevice{UserID: userID, DeviceID: deviceID})
 }
 
-// Not thread-safe: must be called on the OnNewEvent goroutine only
 func (n *Notifier) PeekingDevices(roomID string) (peekingDevices []types.PeekingDevice) {
+	n.lock.RLock()
+	defer n.lock.RUnlock()
+	return n._peekingDevices(roomID)
+}
+
+func (n *Notifier) _peekingDevices(roomID string) (peekingDevices []types.PeekingDevice) {
 	if _, ok := n.roomIDToPeekingDevices[roomID]; !ok {
 		return
 	}
 	return n.roomIDToPeekingDevices[roomID].values()
 }
 
-// removeEmptyUserStreams iterates through the user stream map and removes any
+// _removeEmptyUserStreams iterates through the user stream map and removes any
 // that have been empty for a certain amount of time. This is a crude way of
 // ensuring that the userStreams map doesn't grow forver.
 // This should be called when the notifier gets called for whatever reason,
 // the function itself is responsible for ensuring it doesn't iterate too
 // often.
-// NB: Callers should have locked the mutex before calling this function.
-func (n *Notifier) removeEmptyUserStreams() {
+func (n *Notifier) _removeEmptyUserStreams() {
 	// Only clean up  now and again
 	now := time.Now()
 	if n.lastCleanUpTime.Add(time.Minute).After(now) {
@@ -454,18 +527,52 @@ func (n *Notifier) removeEmptyUserStreams() {
 }
 
 // A string set, mainly existing for improving clarity of structs in this file.
-type userIDSet map[string]bool
-
-func (s userIDSet) add(str string) {
-	s[str] = true
+type userIDSet struct {
+	sync.Mutex
+	set         map[string]struct{}
+	precomputed []string
 }
 
-func (s userIDSet) remove(str string) {
-	delete(s, str)
+func newUserIDSet(cap int) *userIDSet {
+	return &userIDSet{
+		set:         make(map[string]struct{}, cap),
+		precomputed: nil,
+	}
 }
 
-func (s userIDSet) values() (vals []string) {
-	for str := range s {
+func (s *userIDSet) add(str string) {
+	s.Lock()
+	defer s.Unlock()
+	s.set[str] = struct{}{}
+	s.precomputed = s.precomputed[:0] // invalidate cache
+}
+
+func (s *userIDSet) remove(str string) {
+	s.Lock()
+	defer s.Unlock()
+	delete(s.set, str)
+	s.precomputed = s.precomputed[:0] // invalidate cache
+}
+
+func (s *userIDSet) precompute() {
+	s.Lock()
+	defer s.Unlock()
+	s.precomputed = s.values()
+}
+
+func (s *userIDSet) isIn(str string) bool {
+	s.Lock()
+	defer s.Unlock()
+	_, ok := s.set[str]
+	return ok
+}
+
+func (s *userIDSet) values() (vals []string) {
+	if len(s.precomputed) > 0 {
+		return s.precomputed // only return if not invalidated
+	}
+	vals = make([]string, 0, len(s.set))
+	for str := range s.set {
 		vals = append(vals, str)
 	}
 	return
@@ -473,10 +580,10 @@ func (s userIDSet) values() (vals []string) {
 
 // A set of PeekingDevices, similar to userIDSet
 
-type peekingDeviceSet map[types.PeekingDevice]bool
+type peekingDeviceSet map[types.PeekingDevice]struct{}
 
 func (s peekingDeviceSet) add(d types.PeekingDevice) {
-	s[d] = true
+	s[d] = struct{}{}
 }
 
 // nolint:unused
@@ -485,6 +592,7 @@ func (s peekingDeviceSet) remove(d types.PeekingDevice) {
 }
 
 func (s peekingDeviceSet) values() (vals []types.PeekingDevice) {
+	vals = make([]types.PeekingDevice, 0, len(s))
 	for d := range s {
 		vals = append(vals, d)
 	}

syncapi/notifier/notifier_test.go

@@ -107,7 +107,8 @@ func mustEqualPositions(t *testing.T, got, want types.StreamingToken) {
 
 // Test that the current position is returned if a request is already behind.
 func TestImmediateNotification(t *testing.T) {
-	n := NewNotifier(syncPositionBefore)
+	n := NewNotifier()
+	n.SetCurrentPosition(syncPositionBefore)
 	pos, err := waitForEvents(n, newTestSyncRequest(alice, aliceDev, syncPositionVeryOld))
 	if err != nil {
 		t.Fatalf("TestImmediateNotification error: %s", err)
@@ -117,7 +118,8 @@ func TestImmediateNotification(t *testing.T) {
 
 // Test that new events to a joined room unblocks the request.
 func TestNewEventAndJoinedToRoom(t *testing.T) {
-	n := NewNotifier(syncPositionBefore)
+	n := NewNotifier()
+	n.SetCurrentPosition(syncPositionBefore)
 	n.setUsersJoinedToRooms(map[string][]string{
 		roomID: {alice, bob},
 	})
@@ -142,7 +144,8 @@ func TestNewEventAndJoinedToRoom(t *testing.T) {
 }
 
 func TestCorrectStream(t *testing.T) {
-	n := NewNotifier(syncPositionBefore)
+	n := NewNotifier()
+	n.SetCurrentPosition(syncPositionBefore)
 	stream := lockedFetchUserStream(n, bob, bobDev)
 	if stream.UserID != bob {
 		t.Fatalf("expected user %q, got %q", bob, stream.UserID)
@@ -153,7 +156,8 @@ func TestCorrectStream(t *testing.T) {
 }
 
 func TestCorrectStreamWakeup(t *testing.T) {
-	n := NewNotifier(syncPositionBefore)
+	n := NewNotifier()
+	n.SetCurrentPosition(syncPositionBefore)
 	awoken := make(chan string)
 
 	streamone := lockedFetchUserStream(n, alice, "one")
@@ -161,9 +165,9 @@ func TestCorrectStreamWakeup(t *testing.T) {
 
 	go func() {
 		select {
-		case <-streamone.signalChannel:
+		case <-streamone.ch():
 			awoken <- "one"
-		case <-streamtwo.signalChannel:
+		case <-streamtwo.ch():
 			awoken <- "two"
 		}
 	}()
@@ -171,7 +175,7 @@ func TestCorrectStreamWakeup(t *testing.T) {
 	time.Sleep(1 * time.Second)
 
 	wake := "two"
-	n.wakeupUserDevice(alice, []string{wake}, syncPositionAfter)
+	n._wakeupUserDevice(alice, []string{wake}, syncPositionAfter)
 
 	if result := <-awoken; result != wake {
 		t.Fatalf("expected to wake %q, got %q", wake, result)
@@ -180,7 +184,8 @@ func TestCorrectStreamWakeup(t *testing.T) {
 
 // Test that an invite unblocks the request
 func TestNewInviteEventForUser(t *testing.T) {
-	n := NewNotifier(syncPositionBefore)
+	n := NewNotifier()
+	n.SetCurrentPosition(syncPositionBefore)
 	n.setUsersJoinedToRooms(map[string][]string{
 		roomID: {alice, bob},
 	})
@@ -236,7 +241,8 @@ func TestEDUWakeup(t *testing.T) {
 
 // Test that all blocked requests get woken up on a new event.
 func TestMultipleRequestWakeup(t *testing.T) {
-	n := NewNotifier(syncPositionBefore)
+	n := NewNotifier()
+	n.SetCurrentPosition(syncPositionBefore)
 	n.setUsersJoinedToRooms(map[string][]string{
 		roomID: {alice, bob},
 	})
@@ -272,7 +278,8 @@ func TestMultipleRequestWakeup(t *testing.T) {
 func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) {
 	// listen as bob. Make bob leave room. Make alice send event to room.
 	// Make sure alice gets woken up only and not bob as well.
-	n := NewNotifier(syncPositionBefore)
+	n := NewNotifier()
+	n.SetCurrentPosition(syncPositionBefore)
 	n.setUsersJoinedToRooms(map[string][]string{
 		roomID: {alice, bob},
 	})
@@ -352,10 +359,10 @@ func waitForBlocking(s *UserDeviceStream, numBlocking uint) {
 // lockedFetchUserStream invokes Notifier.fetchUserStream, respecting Notifier.streamLock.
 // A new stream is made if it doesn't exist already.
 func lockedFetchUserStream(n *Notifier, userID, deviceID string) *UserDeviceStream {
-	n.streamLock.Lock()
-	defer n.streamLock.Unlock()
+	n.lock.Lock()
+	defer n.lock.Unlock()
 
-	return n.fetchUserDeviceStream(userID, deviceID, true)
+	return n._fetchUserDeviceStream(userID, deviceID, true)
 }
 
 func newTestSyncRequest(userID, deviceID string, since types.StreamingToken) types.SyncRequest {

syncapi/notifier/userstream.go

@@ -118,6 +118,12 @@ func (s *UserDeviceStream) TimeOfLastNonEmpty() time.Time {
 	return s.timeOfLastChannel
 }
 
+func (s *UserDeviceStream) ch() <-chan struct{} {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	return s.signalChannel
+}
+
 // GetSyncPosition returns last sync position which the UserStream was
 // notified about
 func (s *UserDeviceStreamListener) GetSyncPosition() types.StreamingToken {

syncapi/producers/federationapi_presence.go

@@ -0,0 +1,48 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package producers
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+)
+
+// FederationAPIPresenceProducer produces events for the federation API server to consume
+type FederationAPIPresenceProducer struct {
+	Topic     string
+	JetStream nats.JetStreamContext
+}
+
+func (f *FederationAPIPresenceProducer) SendPresence(
+	userID string, presence types.Presence, statusMsg *string,
+) error {
+	msg := nats.NewMsg(f.Topic)
+	msg.Header.Set(jetstream.UserID, userID)
+	msg.Header.Set("presence", presence.String())
+	msg.Header.Set("from_sync", "true") // only update last_active_ts and presence
+	msg.Header.Set("last_active_ts", strconv.Itoa(int(gomatrixserverlib.AsTimestamp(time.Now()))))
+
+	if statusMsg != nil {
+		msg.Header.Set("status_msg", *statusMsg)
+	}
+
+	_, err := f.JetStream.PublishMsg(msg)
+	return err
+}

syncapi/routing/context.go

@@ -22,6 +22,7 @@ import (
 	"strconv"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/caching"
 	roomserver "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -44,6 +45,7 @@ func Context(
 	rsAPI roomserver.RoomserverInternalAPI,
 	syncDB storage.Database,
 	roomID, eventID string,
+	lazyLoadCache *caching.LazyLoadCache,
 ) util.JSONResponse {
 	filter, err := parseRoomEventFilter(req)
 	if err != nil {
@@ -60,7 +62,9 @@ func Context(
 			Headers: nil,
 		}
 	}
-	filter.Rooms = append(filter.Rooms, roomID)
+	if filter.Rooms != nil {
+		*filter.Rooms = append(*filter.Rooms, roomID)
+	}
 
 	ctx := req.Context()
 	membershipRes := roomserver.QueryMembershipForUserResponse{}
@@ -127,7 +131,7 @@ func Context(
 
 	eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatAll)
 	eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatAll)
-	newState := applyLazyLoadMembers(filter, eventsAfterClient, eventsBeforeClient, state)
+	newState := applyLazyLoadMembers(device, filter, eventsAfterClient, eventsBeforeClient, state, lazyLoadCache)
 
 	response := ContextRespsonse{
 		Event:        gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll),
@@ -146,15 +150,25 @@ func Context(
 	}
 }
 
-func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent, state []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
+func applyLazyLoadMembers(
+	device *userapi.Device,
+	filter *gomatrixserverlib.RoomEventFilter,
+	eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent,
+	state []*gomatrixserverlib.HeaderedEvent,
+	lazyLoadCache *caching.LazyLoadCache,
+) []*gomatrixserverlib.HeaderedEvent {
 	if filter == nil || !filter.LazyLoadMembers {
 		return state
 	}
 	allEvents := append(eventsBefore, eventsAfter...)
-	x := make(map[string]bool)
+	x := make(map[string]struct{})
 	// get members who actually send an event
 	for _, e := range allEvents {
-		x[e.Sender] = true
+		// Don't add membership events the client should already know about
+		if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, e.RoomID, e.Sender); cached {
+			continue
+		}
+		x[e.Sender] = struct{}{}
 	}
 
 	newState := []*gomatrixserverlib.HeaderedEvent{}
@@ -164,8 +178,9 @@ func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter
 			newState = append(newState, event)
 		} else {
 			// did the user send an event?
-			if x[event.Sender()] {
+			if _, ok := x[event.Sender()]; ok {
 				membershipEvents = append(membershipEvents, event)
+				lazyLoadCache.StoreLazyLoadedUser(device, event.RoomID(), event.Sender(), event.EventID())
 			}
 		}
 	}

syncapi/routing/messages.go

@@ -21,6 +21,7 @@ import (
 	"sort"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage"
@@ -64,6 +65,7 @@ func OnIncomingMessagesRequest(
 	rsAPI api.RoomserverInternalAPI,
 	cfg *config.SyncAPI,
 	srp *sync.RequestPool,
+	lazyLoadCache *caching.LazyLoadCache,
 ) util.JSONResponse {
 	var err error
 
@@ -200,6 +202,10 @@ func OnIncomingMessagesRequest(
 	if filter.LazyLoadMembers {
 		membershipToUser := make(map[string]*gomatrixserverlib.HeaderedEvent)
 		for _, evt := range clientEvents {
+			// Don't add membership events the client should already know about
+			if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, roomID, evt.Sender); cached {
+				continue
+			}
 			membership, err := db.GetStateEvent(req.Context(), roomID, gomatrixserverlib.MRoomMember, evt.Sender)
 			if err != nil {
 				util.GetLogger(req.Context()).WithError(err).Error("failed to get membership event for user")
@@ -207,10 +213,11 @@ func OnIncomingMessagesRequest(
 			}
 			if membership != nil {
 				membershipToUser[evt.Sender] = membership
+				lazyLoadCache.StoreLazyLoadedUser(device, roomID, evt.Sender, membership.EventID())
 			}
 		}
 		for _, evt := range membershipToUser {
-			state = append(state, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatAll))
+			state = append(state, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatSync))
 		}
 	}
 
@@ -262,12 +269,8 @@ func (r *messagesReq) retrieveEvents() (
 	clientEvents []gomatrixserverlib.ClientEvent, start,
 	end types.TopologyToken, err error,
 ) {
-	eventFilter := r.filter
-
 	// Retrieve the events from the local database.
-	streamEvents, err := r.db.GetEventsInTopologicalRange(
-		r.ctx, r.from, r.to, r.roomID, eventFilter.Limit, r.backwardOrdering,
-	)
+	streamEvents, err := r.db.GetEventsInTopologicalRange(r.ctx, r.from, r.to, r.roomID, r.filter, r.backwardOrdering)
 	if err != nil {
 		err = fmt.Errorf("GetEventsInRange: %w", err)
 		return