Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/prometheus

.github/workflows/dendrite.yml

@@ -73,6 +73,26 @@ jobs:
     timeout-minutes: 5
     name: Unit tests (Go ${{ matrix.go }})
     runs-on: ubuntu-latest
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:13-alpine
+        # Provide the password for postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     strategy:
       fail-fast: false
       matrix:
@@ -92,6 +112,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go${{ matrix.go }}-test-
       - run: go test ./...
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
 
   # build Dendrite for linux with different architectures and go versions
   build:
@@ -233,7 +258,14 @@ jobs:
       - name: Summarise results.tap
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-
+      - name: Sytest List Maintenance
+        if: ${{ always() }}
+        run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
+        continue-on-error: true # not fatal
+      - name: Are We Synapse Yet?
+        if: ${{ always() }}
+        run: /src/are-we-synapse-yet.py /logs/results.tap -v
+        continue-on-error: true # not fatal
       - name: Upload Sytest logs
         uses: actions/upload-artifact@v2
         if: ${{ always() }}

CHANGES.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## Dendrite 0.8.1 (2022-04-07)
+
+### Fixes
+
+* A bug which could result in the sync API deadlocking due to lock contention in the notifier has been fixed
+
+## Dendrite 0.8.0 (2022-04-07)
+
+### Features
+
+* Support for presence has been added
+  * Presence is not enabled by default
+  * The `global.presence.enable_inbound` and `global.presence.enable_outbound` configuration options allow configuring inbound and outbound presence separately
+* Support for room upgrades via the `/room/{roomID}/upgrade` endpoint has been added (contributed by [DavidSpenler](https://github.com/DavidSpenler), [alexkursell](https://github.com/alexkursell))
+* Support for ignoring users has been added
+* Joined and invite user counts are now sent in the `/sync` room summaries
+* Queued federation and stale device list updates will now be staggered at startup over an up-to 2 minute warm-up period, rather than happening all at once
+* Memory pressure created by the sync notifier has been reduced
+* The EDU server component has now been removed, with the work being moved to more relevant components
+
+### Fixes
+
+* It is now possible to set the `power_level_content_override` when creating a room to include power levels over 100
+* `/send_join` and `/state` responses will now not unmarshal the JSON twice
+* The stream event consumer for push notifications will no longer request membership events that are irrelevant
+* Appservices will no longer incorrectly receive state events twice
+
 ## Dendrite 0.7.0 (2022-03-25)
 
 ### Features

README.md

@@ -82,20 +82,17 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of March 2022 we're at around 76% CS API coverage and 95% Federation coverage, though check
+updates with CI. As of April 2022 we're at around 83% CS API coverage and 95% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
-servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably:
+servers such as matrix.org reasonably well, although there are still some missing features (like Search).
-
-- Search
-- User Directory
-- Presence
 
 We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather
 than features that massive deployments may be interested in (User Directory, OpenID, Guests, Admin APIs, AS API).
 This means Dendrite supports amongst others:
 
 - Core room functionality (creating rooms, invites, auth rules)
-- Federation in rooms v1-v7
+- Full support for room versions 1 to 7
+- Experimental support for room versions 8 to 9
 - Backfilling locally and via federation
 - Accounts, Profiles and Devices
 - Published room lists
@@ -108,6 +105,8 @@ This means Dendrite supports amongst others:
 - Receipts
 - Push
 - Guests
+- User Directory
+- Presence
 
 ## Contributing
 

appservice/consumers/roomserver.go

@@ -83,20 +83,29 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 		return true
 	}
 
-	if output.Type != api.OutputTypeNewRoomEvent {
+	if output.Type != api.OutputTypeNewRoomEvent || output.NewRoomEvent == nil {
 		return true
 	}
 
-	events := []*gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
+	newEventID := output.NewRoomEvent.Event.EventID()
+	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(output.NewRoomEvent.AddsStateEventIDs))
+	events = append(events, output.NewRoomEvent.Event)
 	if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
 		eventsReq := &api.QueryEventsByIDRequest{
-			EventIDs: output.NewRoomEvent.AddsStateEventIDs,
+			EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
 		}
 		eventsRes := &api.QueryEventsByIDResponse{}
-		if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
+		for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
-			return false
+			if eventID != newEventID {
+				eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
+			}
+		}
+		if len(eventsReq.EventIDs) > 0 {
+			if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
+				return false
+			}
+			events = append(events, eventsRes.Events...)
 		}
-		events = append(events, eventsRes.Events...)
 	}
 
 	// Send event to any relevant application services

are-we-synapse-yet.list

@@ -106,10 +106,13 @@ rst Users cannot set notifications powerlevel higher than their own (2 subtests)
 rst Both GET and PUT work
 rct POST /rooms/:room_id/receipt can create receipts 
 red POST /rooms/:room_id/read_markers can create read marker
+med POST /media/v3/upload can create an upload
 med POST /media/r0/upload can create an upload 
+med GET /media/v3/download can fetch the value again
 med GET /media/r0/download can fetch the value again 
 cap GET /capabilities is present and well formed for registered user 
 cap GET /r0/capabilities is not public
+cap GET /v3/capabilities is not public
 reg Register with a recaptcha 
 reg registration is idempotent, without username specified 
 reg registration is idempotent, with username specified 
@@ -174,7 +177,9 @@ eph Ephemeral messages received from clients are correctly expired
 ali Room aliases can contain Unicode
 f,ali Remote room alias queries can handle Unicode
 ali Canonical alias can be set
+ali Canonical alias can be set (3 subtests)
 ali Canonical alias can include alt_aliases
+ali Canonical alias can include alt_aliases (4 subtests)
 ali Regular users can add and delete aliases in the default room configuration
 ali Regular users can add and delete aliases when m.room.aliases is restricted
 ali Deleting a non-existent alias should return a 404
@@ -478,6 +483,30 @@ rmv Inbound federation rejects invites which include invalid JSON for room versi
 rmv Outbound federation rejects invite response which include invalid JSON for room version 6
 rmv Inbound federation rejects invite rejections which include invalid JSON for room version 6
 rmv Server rejects invalid JSON in a version 6 room
+rmv User can create and send/receive messages in a room with version 7 (2 subtests)
+rmv local user can join room with version 7
+rmv User can invite local user to room with version 7
+rmv remote user can join room with version 7
+rmv User can invite remote user to room with version 7
+rmv Remote user can backfill in a room with version 7
+rmv Can reject invites over federation for rooms with version 7
+rmv Can receive redactions from regular users over federation in room version 7
+rmv User can create and send/receive messages in a room with version 8 (2 subtests)
+rmv local user can join room with version 8
+rmv User can invite local user to room with version 8
+rmv remote user can join room with version 8
+rmv User can invite remote user to room with version 8
+rmv Remote user can backfill in a room with version 8
+rmv Can reject invites over federation for rooms with version 8
+rmv Can receive redactions from regular users over federation in room version 8
+rmv User can create and send/receive messages in a room with version 9 (2 subtests)
+rmv local user can join room with version 9
+rmv User can invite local user to room with version 9
+rmv remote user can join room with version 9
+rmv User can invite remote user to room with version 9
+rmv Remote user can backfill in a room with version 9
+rmv Can reject invites over federation for rooms with version 9
+rmv Can receive redactions from regular users over federation in room version 9
 pre Presence changes are reported to local room members 
 f,pre Presence changes are also reported to remote room members 
 pre Presence changes to UNAVAILABLE are reported to local room members 
@@ -772,12 +801,15 @@ app AS can make room aliases
 app Regular users cannot create room aliases within the AS namespace
 app AS-ghosted users can use rooms via AS 
 app AS-ghosted users can use rooms themselves 
+app AS-ghosted users can use rooms via AS (2 subtests)
+app AS-ghosted users can use rooms themselves (3 subtests)
 app Ghost user must register before joining room 
 app AS can set avatar for ghosted users 
 app AS can set displayname for ghosted users 
 app AS can't set displayname for random users
 app Inviting an AS-hosted user asks the AS server 
 app Accesing an AS-hosted room alias asks the AS server 
+app Accesing an AS-hosted room alias asks the AS server (2 subtests)
 app Events in rooms with AS-hosted room aliases are sent to AS server 
 app AS user (not ghost) can join room without registering 
 app AS user (not ghost) can join room without registering, with user_id query param
@@ -789,6 +821,8 @@ app AS can publish rooms in their own list
 app AS and main public room lists are separate 
 app AS can deactivate a user 
 psh Test that a message is pushed 
+psh Test that a message is pushed (6 subtests)
+psh Test that rejected pushers are removed. (4 subtests)
 psh Invites are pushed 
 psh Rooms with names are correctly named in pushed 
 psh Rooms with canonical alias are correctly named in pushed 
@@ -857,9 +891,12 @@ pre Left room members do not cause problems for presence
 crm Rooms can be created with an initial invite list (SYN-205) (1 subtests)
 typ Typing notifications don't leak 
 ban Non-present room members cannot ban others 
+ban Non-present room members cannot ban others (3 subtests)
 psh Getting push rules doesn't corrupt the cache SYN-390 
+psh Getting push rules doesn't corrupt the cache SYN-390 (3 subtests)
 inv Test that we can be reinvited to a room we created 
 syn Multiple calls to /sync should not cause 500 errors 
+syn Multiple calls to /sync should not cause 500 errors (6 subtests)
 gst Guest user can call /events on another world_readable room (SYN-606) 
 gst Real user can call /events on another world_readable room (SYN-606) 
 gst Events come down the correct room

are-we-synapse-yet.py

@@ -3,7 +3,7 @@
 from __future__ import division
 import argparse
 import re
-import sys
+import os
 
 # Usage: $ ./are-we-synapse-yet.py [-v] results.tap
 # This script scans a results.tap file from Dendrite's CI process and spits out
@@ -156,6 +156,7 @@ def parse_test_line(line):
 #    ✓ POST /register downcases capitals in usernames
 #    ...
 def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
+    ci = os.getenv("CI") # When running from GHA, this groups the subsections
     subsections = [] # Registration: 100% (13/13 tests)
     subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"]
     total_passing = 0
@@ -169,7 +170,7 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
         for name, passing in tests.items():
             if passing:
                 group_passing += 1
-            test_names_and_marks.append(f"{'✓' if passing else '×'} {name}")
+            test_names_and_marks.append(f"{'✅' if passing else '❌'} {name}")
             
         total_tests += group_total
         total_passing += group_passing
@@ -186,11 +187,11 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
     print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests))
     print("-" * (len(header_name)+1))
     for line in subsections:
-        print("  %s" % (line,))
+        print("%s%s" % ("::group::" if ci and verbose else "", line,))
         if verbose:
             for test_name_and_pass_mark in subsection_test_names[line]:
                 print("    %s" % (test_name_and_pass_mark,))
-            print("")
+            print("%s" % ("::endgroup::" if ci else ""))
     print("")
 
 def main(results_tap_path, verbose):

build.cmd

@@ -0,0 +1,51 @@
+@echo off
+
+:ENTRY_POINT
+    setlocal EnableDelayedExpansion
+
+    REM script base dir
+    set SCRIPTDIR=%~dp0
+    set PROJDIR=%SCRIPTDIR:~0,-1%
+
+    REM Put installed packages into ./bin
+    set GOBIN=%PROJDIR%\bin
+
+    set FLAGS=
+
+    REM Check if sources are under Git control
+    if not exist ".git" goto :CHECK_BIN
+
+    REM set BUILD=`git rev-parse --short HEAD \\ ""`
+    FOR /F "tokens=*" %%X IN ('git rev-parse --short HEAD') DO (
+        set BUILD=%%X
+    )
+
+    REM set BRANCH=`(git symbolic-ref --short HEAD \ tr -d \/ ) \\ ""`
+    FOR /F "tokens=*" %%X IN ('git symbolic-ref --short HEAD') DO (
+        set BRANCHRAW=%%X
+        set BRANCH=!BRANCHRAW:/=!
+    )
+    if "%BRANCH%" == "main" set BRANCH=
+
+    set FLAGS=-X github.com/matrix-org/dendrite/internal.branch=%BRANCH% -X github.com/matrix-org/dendrite/internal.build=%BUILD%
+
+:CHECK_BIN
+    if exist "bin" goto :ALL_SET
+    mkdir "bin"
+
+:ALL_SET
+    set CGO_ENABLED=1
+    for /D %%P in (cmd\*) do (
+        go build -trimpath -ldflags "%FLAGS%" -v -o ".\bin" ".\%%P"
+    )
+
+    set CGO_ENABLED=0
+    set GOOS=js
+    set GOARCH=wasm
+    go build -trimpath -ldflags "%FLAGS%" -o bin\main.wasm .\cmd\dendritejs-pinecone
+
+    goto :DONE
+
+:DONE
+    echo Done
+    endlocal

build/docker/config/dendrite.yaml

@@ -62,6 +62,17 @@ global:
     - matrix.org
     - vector.im
 
+  # Disables federation. Dendrite will not be able to make any outbound HTTP requests
+  # to other servers and the federation API will not be exposed.
+  disable_federation: false
+
+  # Configures the handling of presence events.
+  presence:
+    # Whether inbound presence events are allowed, e.g. receiving presence events from other servers
+    enable_inbound: false
+    # Whether outbound presence events are allowed, e.g. sending presence events to other servers
+    enable_outbound: false
+
   # Configuration for NATS JetStream
   jetstream:
     # A list of NATS Server addresses to connect to. If none are specified, an
@@ -160,12 +171,6 @@ client_api:
     threshold: 5
     cooloff_ms: 500
 
-# Configuration for the EDU server.
-edu_server:
-  internal_api:
-    listen: http://0.0.0.0:7778
-    connect: http://edu_server:7778
-
 # Configuration for the Federation API.
 federation_api:
   internal_api:
@@ -179,12 +184,6 @@ federation_api:
     max_idle_conns: 2
     conn_max_lifetime: -1
 
-  # List of paths to X.509 certificates to be used by the external federation listeners.
-  # These certificates will be used to calculate the TLS fingerprints and other servers
-  # will expect the certificate to match these fingerprints. Certificates must be in PEM
-  # format.
-  federation_certificates: []
-
   # How many times we will try to resend a failed transaction to a specific server. The
   # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc.
   send_max_retries: 16

build/docker/docker-compose.polylith.yml

@@ -84,18 +84,6 @@ services:
       - internal
     restart: unless-stopped
 
-  edu_server:
-    hostname: edu_server
-    image: matrixdotorg/dendrite-polylith:latest
-    command: eduserver
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-    networks:
-      - internal
-    restart: unless-stopped
-
   federation_api:
     hostname: federation_api
     image: matrixdotorg/dendrite-polylith:latest

build/gobind-pinecone/monolith.go

@@ -1,3 +1,17 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package gobind
 
 import (
@@ -9,7 +23,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"os"
@@ -22,11 +35,9 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
@@ -80,7 +91,7 @@ func (m *DendriteMonolith) PeerCount(peertype int) int {
 }
 
 func (m *DendriteMonolith) SessionCount() int {
-	return len(m.PineconeQUIC.Sessions())
+	return len(m.PineconeQUIC.Protocol("matrix").Sessions())
 }
 
 func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
@@ -259,7 +270,7 @@ func (m *DendriteMonolith) Start() {
 		pk = sk.Public().(ed25519.PublicKey)
 	}
 
-	m.listener, err = net.Listen("tcp", "localhost:65432")
+	m.listener, err = net.Listen("tcp", ":65432")
 	if err != nil {
 		panic(err)
 	}
@@ -270,10 +281,9 @@ func (m *DendriteMonolith) Start() {
 	m.logger.SetOutput(BindLogger{})
 	logrus.SetOutput(BindLogger{})
 
-	logger := log.New(os.Stdout, "PINECONE: ", 0)
+	m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
-	m.PineconeRouter = pineconeRouter.NewRouter(logger, sk, false)
+	m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
-	m.PineconeQUIC = pineconeSessions.NewSessions(logger, m.PineconeRouter)
+	m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
-	m.PineconeMulticast = pineconeMulticast.NewMulticast(logger, m.PineconeRouter)
 
 	prefix := hex.EncodeToString(pk)
 	cfg := &config.Dendrite{}
@@ -281,6 +291,7 @@ func (m *DendriteMonolith) Start() {
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Global.JetStream.InMemory = true
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/%s", m.StorageDirectory, prefix))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-account.db", m.StorageDirectory, prefix))
 	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory))
@@ -315,16 +326,15 @@ func (m *DendriteMonolith) Start() {
 	m.userAPI = userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(m.userAPI)
 
-	eduInputAPI := eduserver.NewInternalAPI(
-		base, cache.New(), m.userAPI,
-	)
-
 	asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI)
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
+	userProvider := users.NewPineconeUserProvider(m.PineconeRouter, m.PineconeQUIC, m.userAPI, federation)
+	roomProvider := rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation)
+
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
 		AccountDB: accountDB,
@@ -332,13 +342,13 @@ func (m *DendriteMonolith) Start() {
 		FedClient: federation,
 		KeyRing:   keyRing,
 
-		AppserviceAPI:          asAPI,
+		AppserviceAPI:            asAPI,
-		EDUInternalAPI:         eduInputAPI,
+		FederationAPI:            fsAPI,
-		FederationAPI:          fsAPI,
+		RoomserverAPI:            rsAPI,
-		RoomserverAPI:          rsAPI,
+		UserAPI:                  m.userAPI,
-		UserAPI:                m.userAPI,
+		KeyAPI:                   keyAPI,
-		KeyAPI:                 keyAPI,
+		ExtPublicRoomsProvider:   roomProvider,
-		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation),
+		ExtUserDirectoryProvider: userProvider,
 	}
 	monolith.AddAllPublicRoutes(
 		base.ProcessContext,
@@ -354,12 +364,15 @@ func (m *DendriteMonolith) Start() {
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
 
 	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
 	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
 	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 
-	pHTTP := m.PineconeQUIC.HTTP()
+	pHTTP := m.PineconeQUIC.Protocol("matrix").HTTP()
+	pHTTP.Mux().Handle(users.PublicURL, pMux)
 	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
 	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
 
@@ -384,28 +397,21 @@ func (m *DendriteMonolith) Start() {
 
 	go func() {
 		m.logger.Info("Listening on ", cfg.Global.ServerName)
-		m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC))
+		m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")))
 	}()
 	go func() {
 		logrus.Info("Listening on ", m.listener.Addr())
 		logrus.Fatal(http.Serve(m.listener, httpRouter))
 	}()
-	go func() {
-		logrus.Info("Sending wake-up message to known nodes")
-		req := &api.PerformBroadcastEDURequest{}
-		res := &api.PerformBroadcastEDUResponse{}
-		if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil {
-			logrus.WithError(err).Error("Failed to send wake-up message to known nodes")
-		}
-	}()
 }
 
 func (m *DendriteMonolith) Stop() {
+	m.processContext.ShutdownDendrite()
 	_ = m.listener.Close()
 	m.PineconeMulticast.Stop()
 	_ = m.PineconeQUIC.Close()
-	m.processContext.ShutdownDendrite()
 	_ = m.PineconeRouter.Close()
+	m.processContext.WaitForComponentsToFinish()
 }
 
 const MaxFrameSize = types.MaxFrameSize

build/gobind-pinecone/platform_ios.go

@@ -1,3 +1,17 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //go:build ios
 // +build ios
 

build/gobind-pinecone/platform_other.go

@@ -1,3 +1,17 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //go:build !ios
 // +build !ios
 

build/gobind-yggdrasil/monolith.go

@@ -13,8 +13,6 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms"
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -23,6 +21,7 @@ import (
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
@@ -36,6 +35,7 @@ type DendriteMonolith struct {
 	StorageDirectory string
 	listener         net.Listener
 	httpServer       *http.Server
+	processContext   *process.ProcessContext
 }
 
 func (m *DendriteMonolith) BaseURL() string {
@@ -87,6 +87,7 @@ func (m *DendriteMonolith) Start() {
 	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory))
+	cfg.Global.JetStream.InMemory = true
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-account.db", m.StorageDirectory))
 	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory))
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-syncapi.db", m.StorageDirectory))
@@ -101,6 +102,7 @@ func (m *DendriteMonolith) Start() {
 	}
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
+	m.processContext = base.ProcessContext
 	defer base.Close() // nolint: errcheck
 
 	accountDB := base.CreateAccountsDB()
@@ -119,10 +121,6 @@ func (m *DendriteMonolith) Start() {
 	userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
-	eduInputAPI := eduserver.NewInternalAPI(
-		base, cache.New(), userAPI,
-	)
-
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 
@@ -137,12 +135,11 @@ func (m *DendriteMonolith) Start() {
 		FedClient: federation,
 		KeyRing:   keyRing,
 
-		AppserviceAPI:  asAPI,
+		AppserviceAPI: asAPI,
-		EDUInternalAPI: eduInputAPI,
+		FederationAPI: fsAPI,
-		FederationAPI:  fsAPI,
+		RoomserverAPI: rsAPI,
-		RoomserverAPI:  rsAPI,
+		UserAPI:       userAPI,
-		UserAPI:        userAPI,
+		KeyAPI:        keyAPI,
-		KeyAPI:         keyAPI,
 		ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider(
 			ygg, fsAPI, federation,
 		),
@@ -197,9 +194,12 @@ func (m *DendriteMonolith) Start() {
 	}()
 }
 
-func (m *DendriteMonolith) Suspend() {
+func (m *DendriteMonolith) Stop() {
-	m.logger.Info("Suspending monolith")
 	if err := m.httpServer.Close(); err != nil {
 		m.logger.Warn("Error stopping HTTP server:", err)
 	}
+	if m.processContext != nil {
+		m.processContext.ShutdownDendrite()
+		m.processContext.WaitForComponentsToFinish()
+	}
 }

clientapi/auth/authtypes/profile.go

@@ -17,6 +17,7 @@ package authtypes
 // Profile represents the profile for a Matrix account.
 type Profile struct {
 	Localpart   string `json:"local_part"`
+	ServerName  string `json:"server_name,omitempty"` // NOTSPEC: only set by Pinecone user provider
 	DisplayName string `json:"display_name"`
 	AvatarURL   string `json:"avatar_url"`
 }

clientapi/clientapi.go

@@ -20,7 +20,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/clientapi/routing"
-	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
@@ -40,26 +39,32 @@ func AddPublicRoutes(
 	cfg *config.ClientAPI,
 	federation *gomatrixserverlib.FederationClient,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	eduInputAPI eduServerAPI.EDUServerInputAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	transactionsCache *transactions.Cache,
 	fsAPI federationAPI.FederationInternalAPI,
 	userAPI userapi.UserInternalAPI,
+	userDirectoryProvider userapi.UserDirectoryProvider,
 	keyAPI keyserverAPI.KeyInternalAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	mscCfg *config.MSCs,
 ) {
-	js, _ := jetstream.Prepare(process, &cfg.Matrix.JetStream)
+	js, natsClient := jetstream.Prepare(process, &cfg.Matrix.JetStream)
 
 	syncProducer := &producers.SyncAPIProducer{
-		JetStream: js,
+		JetStream:              js,
-		Topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
+		TopicClientData:        cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
+		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		UserAPI:                userAPI,
+		ServerName:             cfg.Matrix.ServerName,
 	}
 
 	routing.Setup(
-		router, synapseAdminRouter, cfg, eduInputAPI, rsAPI, asAPI,
+		router, synapseAdminRouter, cfg, rsAPI, asAPI,
-		userAPI, federation,
+		userAPI, userDirectoryProvider, federation,
 		syncProducer, transactionsCache, fsAPI, keyAPI,
-		extRoomsProvider, mscCfg,
+		extRoomsProvider, mscCfg, natsClient,
 	)
 }

clientapi/producers/syncapi.go

@@ -15,32 +15,45 @@
 package producers
 
 import (
+	"context"
 	"encoding/json"
+	"strconv"
+	"time"
 
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 )
 
 // SyncAPIProducer produces events for the sync API server to consume
 type SyncAPIProducer struct {
-	Topic     string
+	TopicClientData        string
-	JetStream nats.JetStreamContext
+	TopicReceiptEvent      string
+	TopicSendToDeviceEvent string
+	TopicTypingEvent       string
+	TopicPresenceEvent     string
+	JetStream              nats.JetStreamContext
+	ServerName             gomatrixserverlib.ServerName
+	UserAPI                userapi.UserInternalAPI
 }
 
 // SendData sends account data to the sync API server
-func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON) error {
+func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON, ignoredUsers *types.IgnoredUsers) error {
 	m := &nats.Msg{
-		Subject: p.Topic,
+		Subject: p.TopicClientData,
 		Header:  nats.Header{},
 	}
 	m.Header.Set(jetstream.UserID, userID)
 
 	data := eventutil.AccountData{
-		RoomID:     roomID,
+		RoomID:       roomID,
-		Type:       dataType,
+		Type:         dataType,
-		ReadMarker: readMarker,
+		ReadMarker:   readMarker,
+		IgnoredUsers: ignoredUsers,
 	}
 	var err error
 	m.Data, err = json.Marshal(data)
@@ -52,8 +65,130 @@ func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string
 		"user_id":   userID,
 		"room_id":   roomID,
 		"data_type": dataType,
-	}).Tracef("Producing to topic '%s'", p.Topic)
+	}).Tracef("Producing to topic '%s'", p.TopicClientData)
 
 	_, err = p.JetStream.PublishMsg(m)
 	return err
 }
+
+func (p *SyncAPIProducer) SendReceipt(
+	ctx context.Context,
+	userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp,
+) error {
+	m := &nats.Msg{
+		Subject: p.TopicReceiptEvent,
+		Header:  nats.Header{},
+	}
+	m.Header.Set(jetstream.UserID, userID)
+	m.Header.Set(jetstream.RoomID, roomID)
+	m.Header.Set(jetstream.EventID, eventID)
+	m.Header.Set("type", receiptType)
+	m.Header.Set("timestamp", strconv.Itoa(int(timestamp)))
+
+	log.WithFields(log.Fields{}).Tracef("Producing to topic '%s'", p.TopicReceiptEvent)
+	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}
+
+func (p *SyncAPIProducer) SendToDevice(
+	ctx context.Context, sender, userID, deviceID, eventType string,
+	message interface{},
+) error {
+	devices := []string{}
+	_, domain, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return err
+	}
+
+	// If the event is targeted locally then we want to expand the wildcard
+	// out into individual device IDs so that we can send them to each respective
+	// device. If the event isn't targeted locally then we can't expand the
+	// wildcard as we don't know about the remote devices, so instead we leave it
+	// as-is, so that the federation sender can send it on with the wildcard intact.
+	if domain == p.ServerName && deviceID == "*" {
+		var res userapi.QueryDevicesResponse
+		err = p.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{
+			UserID: userID,
+		}, &res)
+		if err != nil {
+			return err
+		}
+		for _, dev := range res.Devices {
+			devices = append(devices, dev.ID)
+		}
+	} else {
+		devices = append(devices, deviceID)
+	}
+
+	js, err := json.Marshal(message)
+	if err != nil {
+		return err
+	}
+
+	log.WithFields(log.Fields{
+		"user_id":     userID,
+		"num_devices": len(devices),
+		"type":        eventType,
+	}).Tracef("Producing to topic '%s'", p.TopicSendToDeviceEvent)
+	for _, device := range devices {
+		ote := &types.OutputSendToDeviceEvent{
+			UserID:   userID,
+			DeviceID: device,
+			SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{
+				Sender:  sender,
+				Type:    eventType,
+				Content: js,
+			},
+		}
+
+		eventJSON, err := json.Marshal(ote)
+		if err != nil {
+			log.WithError(err).Error("sendToDevice failed json.Marshal")
+			return err
+		}
+		m := &nats.Msg{
+			Subject: p.TopicSendToDeviceEvent,
+			Data:    eventJSON,
+			Header:  nats.Header{},
+		}
+		m.Header.Set("sender", sender)
+		m.Header.Set(jetstream.UserID, userID)
+		if _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)); err != nil {
+			log.WithError(err).Error("sendToDevice failed t.Producer.SendMessage")
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *SyncAPIProducer) SendTyping(
+	ctx context.Context, userID, roomID string, typing bool, timeoutMS int64,
+) error {
+	m := &nats.Msg{
+		Subject: p.TopicTypingEvent,
+		Header:  nats.Header{},
+	}
+	m.Header.Set(jetstream.UserID, userID)
+	m.Header.Set(jetstream.RoomID, roomID)
+	m.Header.Set("typing", strconv.FormatBool(typing))
+	m.Header.Set("timeout_ms", strconv.Itoa(int(timeoutMS)))
+
+	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}
+
+func (p *SyncAPIProducer) SendPresence(
+	ctx context.Context, userID string, presence types.Presence, statusMsg *string,
+) error {
+	m := nats.NewMsg(p.TopicPresenceEvent)
+	m.Header.Set(jetstream.UserID, userID)
+	m.Header.Set("presence", presence.String())
+	if statusMsg != nil {
+		m.Header.Set("status_msg", *statusMsg)
+	}
+
+	m.Header.Set("last_active_ts", strconv.Itoa(int(gomatrixserverlib.AsTimestamp(time.Now()))))
+
+	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}

clientapi/routing/account_data.go

@@ -23,9 +23,9 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
-	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/util"
@@ -95,10 +95,10 @@ func SaveAccountData(
 		}
 	}
 
-	if dataType == "m.fully_read" {
+	if dataType == "m.fully_read" || dataType == "m.push_rules" {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("Unable to set read marker"),
+			JSON: jsonerror.Forbidden(fmt.Sprintf("Unable to modify %q using this API", dataType)),
 		}
 	}
 
@@ -127,8 +127,14 @@ func SaveAccountData(
 		return util.ErrorResponse(err)
 	}
 
+	var ignoredUsers *types.IgnoredUsers
+	if dataType == "m.ignored_user_list" {
+		ignoredUsers = &types.IgnoredUsers{}
+		_ = json.Unmarshal(body, ignoredUsers)
+	}
+
 	// TODO: user API should do this since it's account data
-	if err := syncProducer.SendData(userID, roomID, dataType, nil); err != nil {
+	if err := syncProducer.SendData(userID, roomID, dataType, nil, ignoredUsers); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
 		return jsonerror.InternalServerError()
 	}
@@ -146,7 +152,7 @@ type fullyReadEvent struct {
 // SaveReadMarker implements POST /rooms/{roomId}/read_markers
 func SaveReadMarker(
 	req *http.Request,
-	userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI,
+	userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
 	syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
 ) util.JSONResponse {
 	// Verify that the user is a member of this room
@@ -185,14 +191,14 @@ func SaveReadMarker(
 		return util.ErrorResponse(err)
 	}
 
-	if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r); err != nil {
+	if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r, nil); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
 		return jsonerror.InternalServerError()
 	}
 
 	// Handle the read receipt that may be included in the read marker
 	if r.Read != "" {
-		return SetReceipt(req, eduAPI, device, roomID, "m.read", r.Read)
+		return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read)
 	}
 
 	return util.JSONResponse{

clientapi/routing/presence.go

@@ -0,0 +1,137 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+)
+
+type presenceReq struct {
+	Presence  string  `json:"presence"`
+	StatusMsg *string `json:"status_msg,omitempty"`
+}
+
+func SetPresence(
+	req *http.Request,
+	cfg *config.ClientAPI,
+	device *api.Device,
+	producer *producers.SyncAPIProducer,
+	userID string,
+) util.JSONResponse {
+	if !cfg.Matrix.Presence.EnableOutbound {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: struct{}{},
+		}
+	}
+	if device.UserID != userID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Unable to set presence for other user."),
+		}
+	}
+	var presence presenceReq
+	parseErr := httputil.UnmarshalJSONRequest(req, &presence)
+	if parseErr != nil {
+		return *parseErr
+	}
+
+	presenceStatus, ok := types.PresenceFromString(presence.Presence)
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown(fmt.Sprintf("Unknown presence '%s'.", presence.Presence)),
+		}
+	}
+	err := producer.SendPresence(req.Context(), userID, presenceStatus, presence.StatusMsg)
+	if err != nil {
+		log.WithError(err).Errorf("failed to update presence")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func GetPresence(
+	req *http.Request,
+	device *api.Device,
+	natsClient *nats.Conn,
+	presenceTopic string,
+	userID string,
+) util.JSONResponse {
+	msg := nats.NewMsg(presenceTopic)
+	msg.Header.Set(jetstream.UserID, userID)
+
+	presence, err := natsClient.RequestMsg(msg, time.Second*10)
+	if err != nil {
+		log.WithError(err).Errorf("unable to get presence")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+
+	statusMsg := presence.Header.Get("status_msg")
+	e := presence.Header.Get("error")
+	if e != "" {
+		log.Errorf("received error msg from nats: %s", e)
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: types.PresenceClientResponse{
+				Presence: types.PresenceUnavailable.String(),
+			},
+		}
+	}
+	lastActive, err := strconv.Atoi(presence.Header.Get("last_active_ts"))
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+
+	p := types.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(lastActive)}
+	currentlyActive := p.CurrentlyActive()
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: types.PresenceClientResponse{
+			CurrentlyActive: &currentlyActive,
+			LastActiveAgo:   p.LastActiveAgo(),
+			Presence:        presence.Header.Get("presence"),
+			StatusMsg:       &statusMsg,
+		},
+	}
+}

clientapi/routing/profile.go

@@ -180,7 +180,7 @@ func SetAvatarURL(
 		return jsonerror.InternalServerError()
 	}
 
-	if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, false); err != nil {
+	if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, true); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/receipt.go

@@ -19,21 +19,20 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/gomatrixserverlib"
 
-	"github.com/matrix-org/dendrite/eduserver/api"
-
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )
 
-func SetReceipt(req *http.Request, eduAPI api.EDUServerInputAPI, device *userapi.Device, roomId, receiptType, eventId string) util.JSONResponse {
+func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
 	timestamp := gomatrixserverlib.AsTimestamp(time.Now())
 	logrus.WithFields(logrus.Fields{
-		"roomId":      roomId,
+		"roomID":      roomID,
 		"receiptType": receiptType,
-		"eventId":     eventId,
+		"eventID":     eventID,
 		"userId":      device.UserID,
 		"timestamp":   timestamp,
 	}).Debug("Setting receipt")
@@ -43,7 +42,7 @@ func SetReceipt(req *http.Request, eduAPI api.EDUServerInputAPI, device *userapi
 		return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType))
 	}
 
-	if err := api.SendReceipt(req.Context(), eduAPI, device.UserID, roomId, eventId, receiptType, timestamp); err != nil {
+	if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil {
 		return util.ErrorResponse(err)
 	}
 

clientapi/routing/register.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/tokens"
@@ -520,22 +521,37 @@ func Register(
 	userAPI userapi.UserRegisterAPI,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
+	defer req.Body.Close() // nolint: errcheck
+	reqBody, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotJSON("Unable to read request body"),
+		}
+	}
+
 	var r registerRequest
-	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	sessionID := gjson.GetBytes(reqBody, "auth.session").String()
-	if resErr != nil {
+	if sessionID == "" {
+		// Generate a new, random session ID
+		sessionID = util.RandomString(sessionIDLength)
+	} else if data, ok := sessions.getParams(sessionID); ok {
+		// Use the parameters from the session as our defaults.
+		// Some of these might end up being overwritten if the
+		// values are specified again in the request body.
+		r.Username = data.Username
+		r.Password = data.Password
+		r.DeviceID = data.DeviceID
+		r.InitialDisplayName = data.InitialDisplayName
+		r.InhibitLogin = data.InhibitLogin
+	}
+	if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil {
 		return *resErr
 	}
 	if req.URL.Query().Get("kind") == "guest" {
 		return handleGuestRegistration(req, r, cfg, userAPI)
 	}
 
-	// Retrieve or generate the sessionID
-	sessionID := r.Auth.Session
-	if sessionID == "" {
-		// Generate a new, random session ID
-		sessionID = util.RandomString(sessionIDLength)
-	}
-
 	// Don't allow numeric usernames less than MAX_INT64.
 	if _, err := strconv.ParseInt(r.Username, 10, 64); err == nil {
 		return util.JSONResponse{
@@ -563,7 +579,7 @@ func Register(
 	case r.Type == authtypes.LoginTypeApplicationService && accessTokenErr == nil:
 		// Spec-compliant case (the access_token is specified and the login type
 		// is correctly set, so it's an appservice registration)
-		if resErr = validateApplicationServiceUsername(r.Username); resErr != nil {
+		if resErr := validateApplicationServiceUsername(r.Username); resErr != nil {
 			return *resErr
 		}
 	case accessTokenErr == nil:
@@ -576,11 +592,11 @@ func Register(
 	default:
 		// Spec-compliant case (neither the access_token nor the login type are
 		// specified, so it's a normal user registration)
-		if resErr = validateUsername(r.Username); resErr != nil {
+		if resErr := validateUsername(r.Username); resErr != nil {
 			return *resErr
 		}
 	}
-	if resErr = validatePassword(r.Password); resErr != nil {
+	if resErr := validatePassword(r.Password); resErr != nil {
 		return *resErr
 	}
 
@@ -830,24 +846,17 @@ func completeRegistration(
 		}
 	}()
 
-	if data, ok := sessions.getParams(sessionID); ok {
-		username = data.Username
-		password = data.Password
-		deviceID = data.DeviceID
-		displayName = data.InitialDisplayName
-		inhibitLogin = data.InhibitLogin
-	}
 	if username == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("missing username"),
+			JSON: jsonerror.MissingArgument("Missing username"),
 		}
 	}
 	// Blank passwords are only allowed by registered application services
 	if password == "" && appserviceID == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("missing password"),
+			JSON: jsonerror.MissingArgument("Missing password"),
 		}
 	}
 	var accRes userapi.PerformAccountCreationResponse

clientapi/routing/room_tagging.go

@@ -98,7 +98,7 @@ func PutTag(
 		return jsonerror.InternalServerError()
 	}
 
-	if err = syncProducer.SendData(userID, roomID, "m.tag", nil); err != nil {
+	if err = syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
 		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
 	}
 
@@ -151,7 +151,7 @@ func DeleteTag(
 	}
 
 	// TODO: user API should do this since it's account data
-	if err := syncProducer.SendData(userID, roomID, "m.tag", nil); err != nil {
+	if err := syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
 		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
 	}
 

clientapi/routing/routing.go

@@ -26,16 +26,17 @@ import (
 	clientutil "github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
-	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
@@ -48,17 +49,17 @@ import (
 // nolint: gocyclo
 func Setup(
 	publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI,
-	eduAPI eduServerAPI.EDUServerInputAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	userAPI userapi.UserInternalAPI,
+	userDirectoryProvider userapi.UserDirectoryProvider,
 	federation *gomatrixserverlib.FederationClient,
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
 	federationSender federationAPI.FederationInternalAPI,
 	keyAPI keyserverAPI.KeyInternalAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
-	mscCfg *config.MSCs,
+	mscCfg *config.MSCs, natsClient *nats.Conn,
 ) {
 	prometheus.MustRegister(amtRegUsers, sendEventDuration)
 
@@ -469,7 +470,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendTyping(req, device, vars["roomID"], vars["userID"], eduAPI, rsAPI)
+			return SendTyping(req, device, vars["roomID"], vars["userID"], rsAPI, syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/redact/{eventID}",
@@ -498,7 +499,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			txnID := vars["txnID"]
-			return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID)
+			return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
@@ -512,7 +513,7 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			txnID := vars["txnID"]
-			return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID)
+			return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
@@ -783,20 +784,6 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	// Element logs get flooded unless this is handled
-	v3mux.Handle("/presence/{userID}/status",
-		httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
-				return *r
-			}
-			// TODO: Set presence (probably the responsibility of a presence server not clientapi)
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: struct{}{},
-			}
-		}),
-	).Methods(http.MethodPut, http.MethodOptions)
-
 	v3mux.Handle("/voip/turnServer",
 		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
@@ -907,6 +894,7 @@ func Setup(
 				device,
 				userAPI,
 				rsAPI,
+				userDirectoryProvider,
 				cfg.Matrix.ServerName,
 				postContent.SearchString,
 				postContent.Limit,
@@ -943,7 +931,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SaveReadMarker(req, userAPI, rsAPI, eduAPI, syncProducer, device, vars["roomID"])
+			return SaveReadMarker(req, userAPI, rsAPI, syncProducer, device, vars["roomID"])
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
@@ -960,6 +948,16 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
+	v3mux.Handle("/rooms/{roomID}/upgrade",
+		httputil.MakeAuthAPI("rooms_upgrade", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return UpgradeRoom(req, device, cfg, vars["roomID"], userAPI, rsAPI, asAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	v3mux.Handle("/devices",
 		httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetDevicesByLocalpart(req, userAPI, device)
@@ -1298,7 +1296,25 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 
-			return SetReceipt(req, eduAPI, device, vars["roomId"], vars["receiptType"], vars["eventId"])
+			return SetReceipt(req, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"])
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
+	v3mux.Handle("/presence/{userId}/status",
+		httputil.MakeAuthAPI("set_presence", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SetPresence(req, cfg, device, syncProducer, vars["userId"])
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+	v3mux.Handle("/presence/{userId}/status",
+		httputil.MakeAuthAPI("get_presence", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
 }

clientapi/routing/sendevent.go

@@ -268,5 +268,24 @@ func generateSendEvent(
 			JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
 		}
 	}
+
+	// User should not be able to send a tombstone event to the same room.
+	if e.Type() == "m.room.tombstone" {
+		content := make(map[string]interface{})
+		if err = json.Unmarshal(e.Content(), &content); err != nil {
+			util.GetLogger(ctx).WithError(err).Error("Cannot unmarshal the event content.")
+			return nil, &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("Cannot unmarshal the event content."),
+			}
+		}
+		if content["replacement_room"] == e.RoomID() {
+			return nil, &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidParam("Cannot send tombstone event that points to the same room."),
+			}
+		}
+	}
+
 	return e.Event, nil
 }

clientapi/routing/sendtodevice.go

@@ -18,17 +18,17 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/eduserver/api"
+	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
 // SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}
-// sends the device events to the EDU Server
+// sends the device events to the syncapi & federationsender
 func SendToDevice(
 	req *http.Request, device *userapi.Device,
-	eduAPI api.EDUServerInputAPI,
+	syncProducer *producers.SyncAPIProducer,
 	txnCache *transactions.Cache,
 	eventType string, txnID *string,
 ) util.JSONResponse {
@@ -48,8 +48,8 @@ func SendToDevice(
 
 	for userID, byUser := range httpReq.Messages {
 		for deviceID, message := range byUser {
-			if err := api.SendToDevice(
+			if err := syncProducer.SendToDevice(
-				req.Context(), eduAPI, device.UserID, userID, deviceID, eventType, message,
+				req.Context(), device.UserID, userID, deviceID, eventType, message,
 			); err != nil {
 				util.GetLogger(req.Context()).WithError(err).Error("eduProducer.SendToDevice failed")
 				return jsonerror.InternalServerError()

clientapi/routing/sendtyping.go

@@ -17,7 +17,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/eduserver/api"
+	"github.com/matrix-org/dendrite/clientapi/producers"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
@@ -32,9 +32,8 @@ type typingContentJSON struct {
 // sends the typing events to client API typingProducer
 func SendTyping(
 	req *http.Request, device *userapi.Device, roomID string,
-	userID string,
+	userID string, rsAPI roomserverAPI.RoomserverInternalAPI,
-	eduAPI api.EDUServerInputAPI,
+	syncProducer *producers.SyncAPIProducer,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if device.UserID != userID {
 		return util.JSONResponse{
@@ -56,9 +55,7 @@ func SendTyping(
 		return *resErr
 	}
 
-	if err := api.SendTyping(
+	if err := syncProducer.SendTyping(req.Context(), userID, roomID, r.Typing, r.Timeout); err != nil {
-		req.Context(), eduAPI, userID, roomID, r.Typing, r.Timeout,
-	); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("eduProducer.Send failed")
 		return jsonerror.InternalServerError()
 	}

clientapi/routing/upgrade_room.go

@@ -0,0 +1,92 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/dendrite/setup/config"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type upgradeRoomRequest struct {
+	NewVersion string `json:"new_version"`
+}
+
+type upgradeRoomResponse struct {
+	ReplacementRoom string `json:"replacement_room"`
+}
+
+// UpgradeRoom implements /upgrade
+func UpgradeRoom(
+	req *http.Request, device *userapi.Device,
+	cfg *config.ClientAPI,
+	roomID string, profileAPI userapi.UserProfileAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	var r upgradeRoomRequest
+	if rErr := httputil.UnmarshalJSONRequest(req, &r); rErr != nil {
+		return *rErr
+	}
+
+	// Validate that the room version is supported
+	if _, err := version.SupportedRoomVersion(gomatrixserverlib.RoomVersion(r.NewVersion)); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnsupportedRoomVersion("This server does not support that room version"),
+		}
+	}
+
+	upgradeReq := roomserverAPI.PerformRoomUpgradeRequest{
+		UserID:      device.UserID,
+		RoomID:      roomID,
+		RoomVersion: gomatrixserverlib.RoomVersion(r.NewVersion),
+	}
+	upgradeResp := roomserverAPI.PerformRoomUpgradeResponse{}
+
+	rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp)
+
+	if upgradeResp.Error != nil {
+		if upgradeResp.Error.Code == roomserverAPI.PerformErrorNoRoom {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: jsonerror.NotFound("Room does not exist"),
+			}
+		} else if upgradeResp.Error.Code == roomserverAPI.PerformErrorNotAllowed {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden(upgradeResp.Error.Msg),
+			}
+		} else {
+			return jsonerror.InternalServerError()
+		}
+
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: upgradeRoomResponse{
+			ReplacementRoom: upgradeResp.NewRoomID,
+		},
+	}
+}

clientapi/routing/userdirectory.go

@@ -16,6 +16,7 @@ package routing
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@@ -35,6 +36,7 @@ func SearchUserDirectory(
 	device *userapi.Device,
 	userAPI userapi.UserInternalAPI,
 	rsAPI api.RoomserverInternalAPI,
+	provider userapi.UserDirectoryProvider,
 	serverName gomatrixserverlib.ServerName,
 	searchString string,
 	limit int,
@@ -50,13 +52,12 @@ func SearchUserDirectory(
 	}
 
 	// First start searching local users.
-
 	userReq := &userapi.QuerySearchProfilesRequest{
 		SearchString: searchString,
 		Limit:        limit,
 	}
 	userRes := &userapi.QuerySearchProfilesResponse{}
-	if err := userAPI.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
+	if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
 		errRes := util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
 		return &errRes
 	}
@@ -67,7 +68,12 @@ func SearchUserDirectory(
 			break
 		}
 
-		userID := fmt.Sprintf("@%s:%s", user.Localpart, serverName)
+		var userID string
+		if user.ServerName != "" {
+			userID = fmt.Sprintf("@%s:%s", user.Localpart, user.ServerName)
+		} else {
+			userID = fmt.Sprintf("@%s:%s", user.Localpart, serverName)
+		}
 		if _, ok := results[userID]; !ok {
 			results[userID] = authtypes.FullyQualifiedProfile{
 				UserID:      userID,
@@ -87,7 +93,7 @@ func SearchUserDirectory(
 			Limit:        limit - len(results),
 		}
 		stateRes := &api.QueryKnownUsersResponse{}
-		if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil {
+		if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil && err != sql.ErrNoRows {
 			errRes := util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
 			return &errRes
 		}

cmd/dendrite-demo-libp2p/main.go

@@ -29,7 +29,6 @@ import (
 	p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery"
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
-	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver"
@@ -40,8 +39,6 @@ import (
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
-	"github.com/matrix-org/dendrite/eduserver/cache"
-
 	"github.com/sirupsen/logrus"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -152,9 +149,6 @@ func main() {
 	userAPI := userapi.NewInternalAPI(&base.Base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.Base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
-	eduInputAPI := eduserver.NewInternalAPI(
-		&base.Base, cache.New(), userAPI,
-	)
 	asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationapi.NewInternalAPI(
@@ -180,7 +174,6 @@ func main() {
 		KeyRing:   keyRing,
 
 		AppserviceAPI:          asAPI,
-		EDUInternalAPI:         eduInputAPI,
 		FederationAPI:          fsAPI,
 		RoomserverAPI:          rsAPI,
 		UserAPI:                userAPI,

cmd/dendrite-demo-pinecone/conn/client.go

@@ -1,3 +1,17 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package conn
 
 import (
@@ -53,21 +67,22 @@ func (y *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 }
 
 func createTransport(s *pineconeSessions.Sessions) *http.Transport {
+	proto := s.Protocol("matrix")
 	tr := &http.Transport{
 		DisableKeepAlives: false,
-		Dial:              s.Dial,
+		Dial:              proto.Dial,
-		DialContext:       s.DialContext,
+		DialContext:       proto.DialContext,
-		DialTLS:           s.DialTLS,
+		DialTLS:           proto.DialTLS,
-		DialTLSContext:    s.DialTLSContext,
+		DialTLSContext:    proto.DialTLSContext,
 	}
 	tr.RegisterProtocol(
 		"matrix", &RoundTripper{
 			inner: &http.Transport{
 				DisableKeepAlives: false,
-				Dial:              s.Dial,
+				Dial:              proto.Dial,
-				DialContext:       s.DialContext,
+				DialContext:       proto.DialContext,
-				DialTLS:           s.DialTLS,
+				DialTLS:           proto.DialTLS,
-				DialTLSContext:    s.DialTLSContext,
+				DialTLSContext:    proto.DialTLSContext,
 			},
 		},
 	)

cmd/dendrite-demo-pinecone/conn/ws.go

@@ -1,3 +1,17 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package conn
 
 import (

cmd/dendrite-demo-pinecone/defaults/defaults.go

@@ -0,0 +1,21 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package defaults
+
+import "github.com/matrix-org/gomatrixserverlib"
+
+var DefaultServerNames = map[gomatrixserverlib.ServerName]struct{}{
+	"3bf0258d23c60952639cc4c69c71d1508a7d43a0475d9000ff900a1848411ec7": {},
+}

cmd/dendrite-demo-pinecone/embed/embed_elementweb.go

@@ -1,3 +1,17 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //go:build elementweb
 // +build elementweb
 

cmd/dendrite-demo-pinecone/embed/embed_other.go

@@ -1,3 +1,17 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //go:build !elementweb
 // +build !elementweb
 

cmd/dendrite-demo-pinecone/main.go

@@ -1,4 +1,4 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
+// Copyright 2022 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"os"
@@ -35,11 +34,9 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/embed"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver"
@@ -92,8 +89,7 @@ func main() {
 		pk = sk.Public().(ed25519.PublicKey)
 	}
 
-	logger := log.New(os.Stdout, "", 0)
+	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
-	pRouter := pineconeRouter.NewRouter(logger, sk, false)
 
 	go func() {
 		listener, err := net.Listen("tcp", *instanceListen)
@@ -123,8 +119,8 @@ func main() {
 		}
 	}()
 
-	pQUIC := pineconeSessions.NewSessions(logger, pRouter)
+	pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
-	pMulticast := pineconeMulticast.NewMulticast(logger, pRouter)
+	pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
 	pMulticast.Start()
 
 	connectToStaticPeer := func() {
@@ -190,14 +186,13 @@ func main() {
 	userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
-	eduInputAPI := eduserver.NewInternalAPI(
-		base, cache.New(), userAPI,
-	)
-
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 
 	rsComponent.SetFederationAPI(fsAPI, keyRing)
 
+	userProvider := users.NewPineconeUserProvider(pRouter, pQUIC, userAPI, federation)
+	roomProvider := rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation)
+
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
 		AccountDB: accountDB,
@@ -205,13 +200,13 @@ func main() {
 		FedClient: federation,
 		KeyRing:   keyRing,
 
-		AppserviceAPI:          asAPI,
+		AppserviceAPI:            asAPI,
-		EDUInternalAPI:         eduInputAPI,
+		FederationAPI:            fsAPI,
-		FederationAPI:          fsAPI,
+		RoomserverAPI:            rsAPI,
-		RoomserverAPI:          rsAPI,
+		UserAPI:                  userAPI,
-		UserAPI:                userAPI,
+		KeyAPI:                   keyAPI,
-		KeyAPI:                 keyAPI,
+		ExtPublicRoomsProvider:   roomProvider,
-		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation),
+		ExtUserDirectoryProvider: userProvider,
 	}
 	monolith.AddAllPublicRoutes(
 		base.ProcessContext,
@@ -247,13 +242,16 @@ func main() {
 			logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
 		}
 	})
+	httpRouter.HandleFunc("/pinecone", pRouter.ManholeHandler)
 	embed.Embed(httpRouter, *instancePort, "Pinecone Demo")
 
 	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
 	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
 	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 
-	pHTTP := pQUIC.HTTP()
+	pHTTP := pQUIC.Protocol("matrix").HTTP()
+	pHTTP.Mux().Handle(users.PublicURL, pMux)
 	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
 	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
 
@@ -274,21 +272,13 @@ func main() {
 	go func() {
 		pubkey := pRouter.PublicKey()
 		logrus.Info("Listening on ", hex.EncodeToString(pubkey[:]))
-		logrus.Fatal(httpServer.Serve(pQUIC))
+		logrus.Fatal(httpServer.Serve(pQUIC.Protocol("matrix")))
 	}()
 	go func() {
 		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
 		logrus.Info("Listening on ", httpBindAddr)
 		logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
 	}()
-	go func() {
-		logrus.Info("Sending wake-up message to known nodes")
-		req := &api.PerformBroadcastEDURequest{}
-		res := &api.PerformBroadcastEDUResponse{}
-		if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil {
-			logrus.WithError(err).Error("Failed to send wake-up message to known nodes")
-		}
-	}()
 
 	base.WaitForShutdown()
 }

cmd/dendrite-demo-pinecone/rooms/rooms.go

@@ -1,4 +1,4 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
+// Copyright 2022 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -50,9 +51,12 @@ func NewPineconeRoomProvider(
 }
 
 func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
-	list := []gomatrixserverlib.ServerName{}
+	list := map[gomatrixserverlib.ServerName]struct{}{}
+	for k := range defaults.DefaultServerNames {
+		list[k] = struct{}{}
+	}
 	for _, k := range p.r.Peers() {
-		list = append(list, gomatrixserverlib.ServerName(k.PublicKey))
+		list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
 	}
 	return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list)
 }
@@ -61,7 +65,7 @@ func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
 // Returns a list of public rooms.
 func bulkFetchPublicRoomsFromServers(
 	ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
-	homeservers []gomatrixserverlib.ServerName,
+	homeservers map[gomatrixserverlib.ServerName]struct{},
 ) (publicRooms []gomatrixserverlib.PublicRoom) {
 	limit := 200
 	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
@@ -74,7 +78,7 @@ func bulkFetchPublicRoomsFromServers(
 	wg.Add(len(homeservers))
 	// concurrently query for public rooms
 	reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5)
-	for _, hs := range homeservers {
+	for hs := range homeservers {
 		go func(homeserverDomain gomatrixserverlib.ServerName) {
 			defer wg.Done()
 			util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")

cmd/dendrite-demo-pinecone/users/users.go

@@ -0,0 +1,163 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package users
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
+	pineconeRouter "github.com/matrix-org/pinecone/router"
+	pineconeSessions "github.com/matrix-org/pinecone/sessions"
+)
+
+type PineconeUserProvider struct {
+	r         *pineconeRouter.Router
+	s         *pineconeSessions.Sessions
+	userAPI   userapi.UserProfileAPI
+	fedClient *gomatrixserverlib.FederationClient
+}
+
+const PublicURL = "/_matrix/p2p/profiles"
+
+func NewPineconeUserProvider(
+	r *pineconeRouter.Router,
+	s *pineconeSessions.Sessions,
+	userAPI userapi.UserProfileAPI,
+	fedClient *gomatrixserverlib.FederationClient,
+) *PineconeUserProvider {
+	p := &PineconeUserProvider{
+		r:         r,
+		s:         s,
+		userAPI:   userAPI,
+		fedClient: fedClient,
+	}
+	return p
+}
+
+func (p *PineconeUserProvider) FederatedUserProfiles(w http.ResponseWriter, r *http.Request) {
+	req := &userapi.QuerySearchProfilesRequest{Limit: 25}
+	res := &userapi.QuerySearchProfilesResponse{}
+	if err := clienthttputil.UnmarshalJSONRequest(r, &req); err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	if err := p.userAPI.QuerySearchProfiles(r.Context(), req, res); err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	j, err := json.Marshal(res)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.WriteHeader(200)
+	_, _ = w.Write(j)
+}
+
+func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error {
+	list := map[gomatrixserverlib.ServerName]struct{}{}
+	for k := range defaults.DefaultServerNames {
+		list[k] = struct{}{}
+	}
+	for _, k := range p.r.Peers() {
+		list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
+	}
+	res.Profiles = bulkFetchUserDirectoriesFromServers(context.Background(), req, p.fedClient, list)
+	return nil
+}
+
+// bulkFetchUserDirectoriesFromServers fetches users from the list of homeservers.
+// Returns a list of user profiles.
+func bulkFetchUserDirectoriesFromServers(
+	ctx context.Context, req *userapi.QuerySearchProfilesRequest,
+	fedClient *gomatrixserverlib.FederationClient,
+	homeservers map[gomatrixserverlib.ServerName]struct{},
+) (profiles []authtypes.Profile) {
+	jsonBody, err := json.Marshal(req)
+	if err != nil {
+		return nil
+	}
+
+	limit := 200
+	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
+	// goroutines send rooms to this channel
+	profileCh := make(chan authtypes.Profile, int(limit))
+	// signalling channel to tell goroutines to stop sending rooms and quit
+	done := make(chan bool)
+	// signalling to say when we can close the room channel
+	var wg sync.WaitGroup
+	wg.Add(len(homeservers))
+	// concurrently query for public rooms
+	reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5)
+	for hs := range homeservers {
+		go func(homeserverDomain gomatrixserverlib.ServerName) {
+			defer wg.Done()
+			util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for users")
+
+			jsonBodyReader := bytes.NewBuffer(jsonBody)
+			httpReq, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("matrix://%s%s", homeserverDomain, PublicURL), jsonBodyReader)
+			if err != nil {
+				util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn(
+					"bulkFetchUserDirectoriesFromServers: failed to create request",
+				)
+			}
+			res := &userapi.QuerySearchProfilesResponse{}
+			if err = fedClient.DoRequestAndParseResponse(reqctx, httpReq, res); err != nil {
+				util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn(
+					"bulkFetchUserDirectoriesFromServers: failed to query hs",
+				)
+				return
+			}
+			for _, profile := range res.Profiles {
+				profile.ServerName = string(homeserverDomain)
+				// atomically send a room or stop
+				select {
+				case profileCh <- profile:
+				case <-done:
+				case <-reqctx.Done():
+					util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending profiles")
+					return
+				}
+			}
+		}(hs)
+	}
+
+	select {
+	case <-time.After(5 * time.Second):
+	default:
+		wg.Wait()
+	}
+	reqcancel()
+	close(done)
+	close(profileCh)
+
+	for profile := range profileCh {
+		profiles = append(profiles, profile)
+	}
+
+	return profiles
+}

cmd/dendrite-demo-yggdrasil/main.go

@@ -32,8 +32,6 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms"
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal"
@@ -120,10 +118,6 @@ func main() {
 	userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
-	eduInputAPI := eduserver.NewInternalAPI(
-		base, cache.New(), userAPI,
-	)
-
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationapi.NewInternalAPI(
@@ -139,12 +133,11 @@ func main() {
 		FedClient: federation,
 		KeyRing:   keyRing,
 
-		AppserviceAPI:  asAPI,
+		AppserviceAPI: asAPI,
-		EDUInternalAPI: eduInputAPI,
+		FederationAPI: fsAPI,
-		FederationAPI:  fsAPI,
+		RoomserverAPI: rsAPI,
-		RoomserverAPI:  rsAPI,
+		UserAPI:       userAPI,
-		UserAPI:        userAPI,
+		KeyAPI:        keyAPI,
-		KeyAPI:         keyAPI,
 		ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider(
 			ygg, fsAPI, federation,
 		),

cmd/dendrite-monolith-server/main.go

@@ -19,8 +19,6 @@ import (
 	"os"
 
 	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
@@ -61,7 +59,6 @@ func main() {
 		// itself.
 		cfg.AppServiceAPI.InternalAPI.Connect = httpAPIAddr
 		cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.EDUServer.InternalAPI.Connect = httpAPIAddr
 		cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr
 		cfg.KeyServer.InternalAPI.Connect = httpAPIAddr
 		cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr
@@ -136,14 +133,6 @@ func main() {
 	rsImpl.SetUserAPI(userAPI)
 	keyImpl.SetUserAPI(userAPI)
 
-	eduInputAPI := eduserver.NewInternalAPI(
-		base, cache.New(), userAPI,
-	)
-	if base.UseHTTPAPIs {
-		eduserver.AddInternalRoutes(base.InternalAPIMux, eduInputAPI)
-		eduInputAPI = base.EDUServerClient()
-	}
-
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
 		AccountDB: accountDB,
@@ -151,12 +140,10 @@ func main() {
 		FedClient: federation,
 		KeyRing:   keyRing,
 
-		AppserviceAPI:  asAPI,
+		AppserviceAPI: asAPI, FederationAPI: fsAPI,
-		EDUInternalAPI: eduInputAPI,
+		RoomserverAPI: rsAPI,
-		FederationAPI:  fsAPI,
+		UserAPI:       userAPI,
-		RoomserverAPI:  rsAPI,
+		KeyAPI:        keyAPI,
-		UserAPI:        userAPI,
-		KeyAPI:         keyAPI,
 	}
 	monolith.AddAllPublicRoutes(
 		base.ProcessContext,

cmd/dendrite-polylith-multi/main.go

@@ -43,7 +43,6 @@ func main() {
 	components := map[string]entrypoint{
 		"appservice":    personalities.Appservice,
 		"clientapi":     personalities.ClientAPI,
-		"eduserver":     personalities.EDUServer,
 		"federationapi": personalities.FederationAPI,
 		"keyserver":     personalities.KeyServer,
 		"mediaapi":      personalities.MediaAPI,

cmd/dendrite-polylith-multi/personalities/clientapi.go

@@ -27,13 +27,12 @@ func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	asQuery := base.AppserviceHTTPClient()
 	rsAPI := base.RoomserverHTTPClient()
 	fsAPI := base.FederationAPIHTTPClient()
-	eduInputAPI := base.EDUServerClient()
 	userAPI := base.UserAPIClient()
 	keyAPI := base.KeyServerHTTPClient()
 
 	clientapi.AddPublicRoutes(
 		base.ProcessContext, base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI,
-		federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI,
+		federation, rsAPI, asQuery, transactions.New(), fsAPI, userAPI, userAPI,
 		keyAPI, nil, &cfg.MSCs,
 	)
 

cmd/dendrite-polylith-multi/personalities/eduserver.go

@@ -1,33 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func EDUServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
-	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.EDUServer.InternalAPI.Listen, // internal listener
-		basepkg.NoListener,                    // external listener
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -29,9 +29,9 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	keyRing := fsAPI.KeyRing()
 
 	federationapi.AddPublicRoutes(
-		base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux,
+		base.ProcessContext, base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux,
 		&base.Cfg.FederationAPI, userAPI, federation, keyRing,
-		rsAPI, fsAPI, base.EDUServerClient(), keyAPI,
+		rsAPI, fsAPI, keyAPI,
 		&base.Cfg.MSCs, nil,
 	)
 

cmd/dendritejs-pinecone/main.go

@@ -21,8 +21,6 @@ import (
 	"crypto/ed25519"
 	"encoding/hex"
 	"fmt"
-	"log"
-	"os"
 	"syscall/js"
 	"time"
 
@@ -31,8 +29,6 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver"
@@ -156,9 +152,8 @@ func startup() {
 	sk := generateKey()
 	pk := sk.Public().(ed25519.PublicKey)
 
-	logger := log.New(os.Stdout, "", 0)
+	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
-	pRouter := pineconeRouter.NewRouter(logger, sk, false)
+	pSessions := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
-	pSessions := pineconeSessions.NewSessions(logger, pRouter)
 
 	cfg := &config.Dendrite{}
 	cfg.Defaults(true)
@@ -193,7 +188,6 @@ func startup() {
 	userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
-	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI)
 	asQuery := appservice.NewInternalAPI(
 		base, userAPI, rsAPI,
 	)
@@ -208,12 +202,11 @@ func startup() {
 		FedClient: federation,
 		KeyRing:   keyRing,
 
-		AppserviceAPI:  asQuery,
+		AppserviceAPI: asQuery,
-		EDUInternalAPI: eduInputAPI,
+		FederationAPI: fedSenderAPI,
-		FederationAPI:  fedSenderAPI,
+		RoomserverAPI: rsAPI,
-		RoomserverAPI:  rsAPI,
+		UserAPI:       userAPI,
-		UserAPI:        userAPI,
+		KeyAPI:        keyAPI,
-		KeyAPI:         keyAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation),
 	}
@@ -232,7 +225,7 @@ func startup() {
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 
-	p2pRouter := pSessions.HTTP().Mux()
+	p2pRouter := pSessions.Protocol("matrix").HTTP().Mux()
 	p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux)
 	p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux)
 

cmd/dendritejs/main.go

@@ -24,8 +24,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/eduserver"
-	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver"
@@ -203,7 +201,6 @@ func main() {
 	}
 
 	rsAPI := roomserver.NewInternalAPI(base)
-	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI)
 	asQuery := appservice.NewInternalAPI(
 		base, userAPI, rsAPI,
 	)
@@ -222,7 +219,6 @@ func main() {
 		KeyRing:   &keyRing,
 
 		AppserviceAPI:       asQuery,
-		EDUInternalAPI:      eduInputAPI,
 		FederationSenderAPI: fedSenderAPI,
 		RoomserverAPI:       rsAPI,
 		UserAPI:             userAPI,

cmd/generate-config/main.go

@@ -91,6 +91,10 @@ func main() {
 		cfg.UserAPI.BCryptCost = bcrypt.MinCost
 		cfg.Global.JetStream.InMemory = true
 		cfg.ClientAPI.RegistrationSharedSecret = "complement"
+		cfg.Global.Presence = config.PresenceOptions{
+			EnableInbound:  true,
+			EnableOutbound: true,
+		}
 	}
 
 	j, err := yaml.Marshal(cfg)

dendrite-config.yaml

@@ -68,6 +68,13 @@ global:
   # to other servers and the federation API will not be exposed.
   disable_federation: false
 
+  # Configures the handling of presence events.
+  presence:
+    # Whether inbound presence events are allowed, e.g. receiving presence events from other servers
+    enable_inbound: false
+    # Whether outbound presence events are allowed, e.g. sending presence events to other servers
+    enable_outbound: false
+
   # Server notices allows server admins to send messages to all users.
   server_notices:
     enabled: false
@@ -187,12 +194,6 @@ client_api:
     threshold: 5
     cooloff_ms: 500
 
-# Configuration for the EDU server.
-edu_server:
-  internal_api:
-    listen: http://localhost:7778 # Only used in polylith deployments
-    connect: http://localhost:7778 # Only used in polylith deployments
-
 # Configuration for the Federation API.
 federation_api:
   internal_api:
@@ -206,12 +207,6 @@ federation_api:
     max_idle_conns: 2
     conn_max_lifetime: -1
 
-  # List of paths to X.509 certificates to be used by the external federation listeners.
-  # These certificates will be used to calculate the TLS fingerprints and other servers
-  # will expect the certificate to match these fingerprints. Certificates must be in PEM
-  # format.
-  federation_certificates: []
-
   # How many times we will try to resend a failed transaction to a specific server. The
   # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc.
   send_max_retries: 16

docs/CONTRIBUTING.md

@@ -94,4 +94,4 @@ For more general questions please use
 
 We ask that everyone who contributes to the project signs off their
 contributions, in accordance with the
-[DCO](https://github.com/matrix-org/matrix-doc/blob/main/CONTRIBUTING.rst#sign-off).
+[DCO](https://github.com/matrix-org/matrix-spec/blob/main/CONTRIBUTING.rst#sign-off).

docs/INSTALL.md

@@ -34,6 +34,10 @@ If you want to run a polylith deployment, you also need:
 
 * A standalone [NATS Server](https://github.com/nats-io/nats-server) deployment with JetStream enabled
 
+If you want to build it on Windows, you need `gcc` in the path:
+
+* [MinGW-w64](https://www.mingw-w64.org/)
+
 ## Building Dendrite
 
 Start by cloning the code:
@@ -45,9 +49,15 @@ cd dendrite
 
 Then build it:
 
-```bash
+* Linux or UNIX-like systems:
-./build.sh
+  ```bash
-```
+  ./build.sh
+  ```
+
+* Windows:
+  ```dos
+  build.cmd
+  ```
 
 ## Install NATS Server
 
@@ -263,14 +273,6 @@ This manages end-to-end encryption keys for users.
 ./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver
 ```
 
-#### EDU server
-
-This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to
-
-```bash
-./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver
-```
-
 #### User server
 
 This manages user accounts, device access tokens and user account data,

eduserver/api/input.go

@@ -1,103 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package api provides the types that are used to communicate with the typing server.
-package api
-
-import (
-	"context"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// InputTypingEvent is an event for notifying the typing server about typing updates.
-type InputTypingEvent struct {
-	// UserID of the user to update typing status.
-	UserID string `json:"user_id"`
-	// RoomID of the room the user is typing (or has stopped).
-	RoomID string `json:"room_id"`
-	// Typing is true if the user is typing, false if they have stopped.
-	Typing bool `json:"typing"`
-	// Timeout is the interval in milliseconds for which the user should be marked as typing.
-	TimeoutMS int64 `json:"timeout"`
-	// OriginServerTS when the server received the update.
-	OriginServerTS gomatrixserverlib.Timestamp `json:"origin_server_ts"`
-}
-
-type InputSendToDeviceEvent struct {
-	UserID   string `json:"user_id"`
-	DeviceID string `json:"device_id"`
-	gomatrixserverlib.SendToDeviceEvent
-}
-
-// InputTypingEventRequest is a request to EDUServerInputAPI
-type InputTypingEventRequest struct {
-	InputTypingEvent InputTypingEvent `json:"input_typing_event"`
-}
-
-// InputTypingEventResponse is a response to InputTypingEvents
-type InputTypingEventResponse struct{}
-
-// InputSendToDeviceEventRequest is a request to EDUServerInputAPI
-type InputSendToDeviceEventRequest struct {
-	InputSendToDeviceEvent InputSendToDeviceEvent `json:"input_send_to_device_event"`
-}
-
-// InputSendToDeviceEventResponse is a response to InputSendToDeviceEventRequest
-type InputSendToDeviceEventResponse struct{}
-
-type InputReceiptEvent struct {
-	UserID    string                      `json:"user_id"`
-	RoomID    string                      `json:"room_id"`
-	EventID   string                      `json:"event_id"`
-	Type      string                      `json:"type"`
-	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
-}
-
-// InputReceiptEventRequest is a request to EDUServerInputAPI
-type InputReceiptEventRequest struct {
-	InputReceiptEvent InputReceiptEvent `json:"input_receipt_event"`
-}
-
-// InputReceiptEventResponse is a response to InputReceiptEventRequest
-type InputReceiptEventResponse struct{}
-
-type InputCrossSigningKeyUpdateRequest struct {
-	CrossSigningKeyUpdate `json:"signing_keys"`
-}
-
-type InputCrossSigningKeyUpdateResponse struct{}
-
-// EDUServerInputAPI is used to write events to the typing server.
-type EDUServerInputAPI interface {
-	InputTypingEvent(
-		ctx context.Context,
-		request *InputTypingEventRequest,
-		response *InputTypingEventResponse,
-	) error
-
-	InputSendToDeviceEvent(
-		ctx context.Context,
-		request *InputSendToDeviceEventRequest,
-		response *InputSendToDeviceEventResponse,
-	) error
-
-	InputReceiptEvent(
-		ctx context.Context,
-		request *InputReceiptEventRequest,
-		response *InputReceiptEventResponse,
-	) error
-}

eduserver/api/output.go

@@ -1,57 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
-	"time"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// OutputTypingEvent is an entry in typing server output kafka log.
-// This contains the event with extra fields used to create 'm.typing' event
-// in clientapi & federation.
-type OutputTypingEvent struct {
-	// The Event for the typing edu event.
-	Event TypingEvent `json:"event"`
-	// ExpireTime is the interval after which the user should no longer be
-	// considered typing. Only available if Event.Typing is true.
-	ExpireTime *time.Time
-}
-
-// OutputSendToDeviceEvent is an entry in the send-to-device output kafka log.
-// This contains the full event content, along with the user ID and device ID
-// to which it is destined.
-type OutputSendToDeviceEvent struct {
-	UserID   string `json:"user_id"`
-	DeviceID string `json:"device_id"`
-	gomatrixserverlib.SendToDeviceEvent
-}
-
-// OutputReceiptEvent is an entry in the receipt output kafka log
-type OutputReceiptEvent struct {
-	UserID    string                      `json:"user_id"`
-	RoomID    string                      `json:"room_id"`
-	EventID   string                      `json:"event_id"`
-	Type      string                      `json:"type"`
-	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
-}
-
-// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log
-type OutputCrossSigningKeyUpdate struct {
-	CrossSigningKeyUpdate `json:"signing_keys"`
-}

eduserver/api/types.go

@@ -1,59 +0,0 @@
-// Copyright 2021 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import "github.com/matrix-org/gomatrixserverlib"
-
-const (
-	MSigningKeyUpdate = "m.signing_key_update"
-)
-
-type TypingEvent struct {
-	Type   string `json:"type"`
-	RoomID string `json:"room_id"`
-	UserID string `json:"user_id"`
-	Typing bool   `json:"typing"`
-}
-
-type ReceiptEvent struct {
-	UserID    string                      `json:"user_id"`
-	RoomID    string                      `json:"room_id"`
-	EventID   string                      `json:"event_id"`
-	Type      string                      `json:"type"`
-	Timestamp gomatrixserverlib.Timestamp `json:"timestamp"`
-}
-
-type FederationReceiptMRead struct {
-	User map[string]FederationReceiptData `json:"m.read"`
-}
-
-type FederationReceiptData struct {
-	Data     ReceiptTS `json:"data"`
-	EventIDs []string  `json:"event_ids"`
-}
-
-type ReceiptMRead struct {
-	User map[string]ReceiptTS `json:"m.read"`
-}
-
-type ReceiptTS struct {
-	TS gomatrixserverlib.Timestamp `json:"ts"`
-}
-
-type CrossSigningKeyUpdate struct {
-	MasterKey      *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"`
-	SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"`
-	UserID         string                             `json:"user_id"`
-}

eduserver/api/wrapper.go

@@ -1,88 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
-	"context"
-	"encoding/json"
-	"time"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// SendTyping sends a typing event to EDU server
-func SendTyping(
-	ctx context.Context, eduAPI EDUServerInputAPI, userID, roomID string,
-	typing bool, timeoutMS int64,
-) error {
-	requestData := InputTypingEvent{
-		UserID:         userID,
-		RoomID:         roomID,
-		Typing:         typing,
-		TimeoutMS:      timeoutMS,
-		OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
-	}
-
-	var response InputTypingEventResponse
-	err := eduAPI.InputTypingEvent(
-		ctx, &InputTypingEventRequest{InputTypingEvent: requestData}, &response,
-	)
-
-	return err
-}
-
-// SendToDevice sends a typing event to EDU server
-func SendToDevice(
-	ctx context.Context, eduAPI EDUServerInputAPI, sender, userID, deviceID, eventType string,
-	message interface{},
-) error {
-	js, err := json.Marshal(message)
-	if err != nil {
-		return err
-	}
-	requestData := InputSendToDeviceEvent{
-		UserID:   userID,
-		DeviceID: deviceID,
-		SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{
-			Sender:  sender,
-			Type:    eventType,
-			Content: js,
-		},
-	}
-	request := InputSendToDeviceEventRequest{
-		InputSendToDeviceEvent: requestData,
-	}
-	response := InputSendToDeviceEventResponse{}
-	return eduAPI.InputSendToDeviceEvent(ctx, &request, &response)
-}
-
-// SendReceipt sends a receipt event to EDU Server
-func SendReceipt(
-	ctx context.Context,
-	eduAPI EDUServerInputAPI, userID, roomID, eventID, receiptType string,
-	timestamp gomatrixserverlib.Timestamp,
-) error {
-	request := InputReceiptEventRequest{
-		InputReceiptEvent: InputReceiptEvent{
-			UserID:    userID,
-			RoomID:    roomID,
-			EventID:   eventID,
-			Type:      receiptType,
-			Timestamp: timestamp,
-		},
-	}
-	response := InputReceiptEventResponse{}
-	return eduAPI.InputReceiptEvent(ctx, &request, &response)
-}

eduserver/eduserver.go

@@ -1,56 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package eduserver
-
-import (
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/eduserver/api"
-	"github.com/matrix-org/dendrite/eduserver/cache"
-	"github.com/matrix-org/dendrite/eduserver/input"
-	"github.com/matrix-org/dendrite/eduserver/inthttp"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-)
-
-// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
-// on the given input API.
-func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI) {
-	inthttp.AddRoutes(inputAPI, internalMux)
-}
-
-// NewInternalAPI returns a concerete implementation of the internal API. Callers
-// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
-func NewInternalAPI(
-	base *base.BaseDendrite,
-	eduCache *cache.EDUCache,
-	userAPI userapi.UserInternalAPI,
-) api.EDUServerInputAPI {
-	cfg := &base.Cfg.EDUServer
-
-	js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
-
-	return &input.EDUServerInputAPI{
-		Cache:                        eduCache,
-		UserAPI:                      userAPI,
-		JetStream:                    js,
-		OutputTypingEventTopic:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
-		OutputSendToDeviceEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
-		OutputReceiptEventTopic:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
-		ServerName:                   cfg.Matrix.ServerName,
-	}
-}

eduserver/input/input.go

@@ -1,198 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package input
-
-import (
-	"context"
-	"encoding/json"
-	"time"
-
-	"github.com/matrix-org/dendrite/eduserver/api"
-	"github.com/matrix-org/dendrite/eduserver/cache"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	"github.com/sirupsen/logrus"
-)
-
-// EDUServerInputAPI implements api.EDUServerInputAPI
-type EDUServerInputAPI struct {
-	// Cache to store the current typing members in each room.
-	Cache *cache.EDUCache
-	// The kafka topic to output new typing events to.
-	OutputTypingEventTopic string
-	// The kafka topic to output new send to device events to.
-	OutputSendToDeviceEventTopic string
-	// The kafka topic to output new receipt events to
-	OutputReceiptEventTopic string
-	// kafka producer
-	JetStream nats.JetStreamContext
-	// Internal user query API
-	UserAPI userapi.UserInternalAPI
-	// our server name
-	ServerName gomatrixserverlib.ServerName
-}
-
-// InputTypingEvent implements api.EDUServerInputAPI
-func (t *EDUServerInputAPI) InputTypingEvent(
-	ctx context.Context,
-	request *api.InputTypingEventRequest,
-	response *api.InputTypingEventResponse,
-) error {
-	ite := &request.InputTypingEvent
-	if ite.Typing {
-		// user is typing, update our current state of users typing.
-		expireTime := ite.OriginServerTS.Time().Add(
-			time.Duration(ite.TimeoutMS) * time.Millisecond,
-		)
-		t.Cache.AddTypingUser(ite.UserID, ite.RoomID, &expireTime)
-	} else {
-		t.Cache.RemoveUser(ite.UserID, ite.RoomID)
-	}
-
-	return t.sendTypingEvent(ite)
-}
-
-// InputTypingEvent implements api.EDUServerInputAPI
-func (t *EDUServerInputAPI) InputSendToDeviceEvent(
-	ctx context.Context,
-	request *api.InputSendToDeviceEventRequest,
-	response *api.InputSendToDeviceEventResponse,
-) error {
-	ise := &request.InputSendToDeviceEvent
-	return t.sendToDeviceEvent(ise)
-}
-
-func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error {
-	ev := &api.TypingEvent{
-		Type:   gomatrixserverlib.MTyping,
-		RoomID: ite.RoomID,
-		UserID: ite.UserID,
-		Typing: ite.Typing,
-	}
-	ote := &api.OutputTypingEvent{
-		Event: *ev,
-	}
-
-	if ev.Typing {
-		expireTime := ite.OriginServerTS.Time().Add(
-			time.Duration(ite.TimeoutMS) * time.Millisecond,
-		)
-		ote.ExpireTime = &expireTime
-	}
-
-	eventJSON, err := json.Marshal(ote)
-	if err != nil {
-		return err
-	}
-	logrus.WithFields(logrus.Fields{
-		"room_id": ite.RoomID,
-		"user_id": ite.UserID,
-		"typing":  ite.Typing,
-	}).Tracef("Producing to topic '%s'", t.OutputTypingEventTopic)
-
-	_, err = t.JetStream.PublishMsg(&nats.Msg{
-		Subject: t.OutputTypingEventTopic,
-		Header:  nats.Header{},
-		Data:    eventJSON,
-	})
-	return err
-}
-
-func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) error {
-	devices := []string{}
-	_, domain, err := gomatrixserverlib.SplitID('@', ise.UserID)
-	if err != nil {
-		return err
-	}
-
-	// If the event is targeted locally then we want to expand the wildcard
-	// out into individual device IDs so that we can send them to each respective
-	// device. If the event isn't targeted locally then we can't expand the
-	// wildcard as we don't know about the remote devices, so instead we leave it
-	// as-is, so that the federation sender can send it on with the wildcard intact.
-	if domain == t.ServerName && ise.DeviceID == "*" {
-		var res userapi.QueryDevicesResponse
-		err = t.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{
-			UserID: ise.UserID,
-		}, &res)
-		if err != nil {
-			return err
-		}
-		for _, dev := range res.Devices {
-			devices = append(devices, dev.ID)
-		}
-	} else {
-		devices = append(devices, ise.DeviceID)
-	}
-
-	logrus.WithFields(logrus.Fields{
-		"user_id":     ise.UserID,
-		"num_devices": len(devices),
-		"type":        ise.Type,
-	}).Tracef("Producing to topic '%s'", t.OutputSendToDeviceEventTopic)
-	for _, device := range devices {
-		ote := &api.OutputSendToDeviceEvent{
-			UserID:            ise.UserID,
-			DeviceID:          device,
-			SendToDeviceEvent: ise.SendToDeviceEvent,
-		}
-
-		eventJSON, err := json.Marshal(ote)
-		if err != nil {
-			logrus.WithError(err).Error("sendToDevice failed json.Marshal")
-			return err
-		}
-
-		if _, err = t.JetStream.PublishMsg(&nats.Msg{
-			Subject: t.OutputSendToDeviceEventTopic,
-			Data:    eventJSON,
-		}); err != nil {
-			logrus.WithError(err).Error("sendToDevice failed t.Producer.SendMessage")
-			return err
-		}
-	}
-
-	return nil
-}
-
-// InputReceiptEvent implements api.EDUServerInputAPI
-// TODO: Intelligently batch requests sent by the same user (e.g wait a few milliseconds before emitting output events)
-func (t *EDUServerInputAPI) InputReceiptEvent(
-	ctx context.Context,
-	request *api.InputReceiptEventRequest,
-	response *api.InputReceiptEventResponse,
-) error {
-	logrus.WithFields(logrus.Fields{}).Tracef("Producing to topic '%s'", t.OutputReceiptEventTopic)
-	output := &api.OutputReceiptEvent{
-		UserID:    request.InputReceiptEvent.UserID,
-		RoomID:    request.InputReceiptEvent.RoomID,
-		EventID:   request.InputReceiptEvent.EventID,
-		Type:      request.InputReceiptEvent.Type,
-		Timestamp: request.InputReceiptEvent.Timestamp,
-	}
-	js, err := json.Marshal(output)
-	if err != nil {
-		return err
-	}
-
-	_, err = t.JetStream.PublishMsg(&nats.Msg{
-		Subject: t.OutputReceiptEventTopic,
-		Data:    js,
-	})
-	return err
-}

eduserver/inthttp/client.go

@@ -1,70 +0,0 @@
-package inthttp
-
-import (
-	"context"
-	"errors"
-	"net/http"
-
-	"github.com/matrix-org/dendrite/eduserver/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/opentracing/opentracing-go"
-)
-
-// HTTP paths for the internal HTTP APIs
-const (
-	EDUServerInputTypingEventPath       = "/eduserver/input"
-	EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice"
-	EDUServerInputReceiptEventPath      = "/eduserver/receipt"
-)
-
-// NewEDUServerClient creates a EDUServerInputAPI implemented by talking to a HTTP POST API.
-func NewEDUServerClient(eduServerURL string, httpClient *http.Client) (api.EDUServerInputAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewEDUServerClient: httpClient is <nil>")
-	}
-	return &httpEDUServerInputAPI{eduServerURL, httpClient}, nil
-}
-
-type httpEDUServerInputAPI struct {
-	eduServerURL string
-	httpClient   *http.Client
-}
-
-// InputTypingEvent implements EDUServerInputAPI
-func (h *httpEDUServerInputAPI) InputTypingEvent(
-	ctx context.Context,
-	request *api.InputTypingEventRequest,
-	response *api.InputTypingEventResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "InputTypingEvent")
-	defer span.Finish()
-
-	apiURL := h.eduServerURL + EDUServerInputTypingEventPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}
-
-// InputSendToDeviceEvent implements EDUServerInputAPI
-func (h *httpEDUServerInputAPI) InputSendToDeviceEvent(
-	ctx context.Context,
-	request *api.InputSendToDeviceEventRequest,
-	response *api.InputSendToDeviceEventResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "InputSendToDeviceEvent")
-	defer span.Finish()
-
-	apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}
-
-// InputSendToDeviceEvent implements EDUServerInputAPI
-func (h *httpEDUServerInputAPI) InputReceiptEvent(
-	ctx context.Context,
-	request *api.InputReceiptEventRequest,
-	response *api.InputReceiptEventResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "InputReceiptEventPath")
-	defer span.Finish()
-
-	apiURL := h.eduServerURL + EDUServerInputReceiptEventPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}

eduserver/inthttp/server.go

@@ -1,54 +0,0 @@
-package inthttp
-
-import (
-	"encoding/json"
-	"net/http"
-
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/eduserver/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/util"
-)
-
-// AddRoutes adds the EDUServerInputAPI handlers to the http.ServeMux.
-func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) {
-	internalAPIMux.Handle(EDUServerInputTypingEventPath,
-		httputil.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse {
-			var request api.InputTypingEventRequest
-			var response api.InputTypingEventResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := t.InputTypingEvent(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(EDUServerInputSendToDeviceEventPath,
-		httputil.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse {
-			var request api.InputSendToDeviceEventRequest
-			var response api.InputSendToDeviceEventResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := t.InputSendToDeviceEvent(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(EDUServerInputReceiptEventPath,
-		httputil.MakeInternalAPI("inputReceiptEvent", func(req *http.Request) util.JSONResponse {
-			var request api.InputReceiptEventRequest
-			var response api.InputReceiptEventResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := t.InputReceiptEvent(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-}

federationapi/consumers/eduserver.go

@@ -1,257 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package consumers
-
-import (
-	"context"
-	"encoding/json"
-
-	"github.com/matrix-org/dendrite/eduserver/api"
-	"github.com/matrix-org/dendrite/federationapi/queue"
-	"github.com/matrix-org/dendrite/federationapi/storage"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/nats-io/nats.go"
-	log "github.com/sirupsen/logrus"
-)
-
-// OutputEDUConsumer consumes events that originate in EDU server.
-type OutputEDUConsumer struct {
-	ctx               context.Context
-	jetstream         nats.JetStreamContext
-	durable           string
-	db                storage.Database
-	queues            *queue.OutgoingQueues
-	ServerName        gomatrixserverlib.ServerName
-	typingTopic       string
-	sendToDeviceTopic string
-	receiptTopic      string
-}
-
-// NewOutputEDUConsumer creates a new OutputEDUConsumer. Call Start() to begin consuming from EDU servers.
-func NewOutputEDUConsumer(
-	process *process.ProcessContext,
-	cfg *config.FederationAPI,
-	js nats.JetStreamContext,
-	queues *queue.OutgoingQueues,
-	store storage.Database,
-) *OutputEDUConsumer {
-	return &OutputEDUConsumer{
-		ctx:               process.Context(),
-		jetstream:         js,
-		queues:            queues,
-		db:                store,
-		ServerName:        cfg.Matrix.ServerName,
-		durable:           cfg.Matrix.JetStream.Durable("FederationAPIEDUServerConsumer"),
-		typingTopic:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
-		sendToDeviceTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
-		receiptTopic:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
-	}
-}
-
-// Start consuming from EDU servers
-func (t *OutputEDUConsumer) Start() error {
-	if err := jetstream.JetStreamConsumer(
-		t.ctx, t.jetstream, t.typingTopic, t.durable, t.onTypingEvent,
-		nats.DeliverAll(), nats.ManualAck(),
-	); err != nil {
-		return err
-	}
-	if err := jetstream.JetStreamConsumer(
-		t.ctx, t.jetstream, t.sendToDeviceTopic, t.durable, t.onSendToDeviceEvent,
-		nats.DeliverAll(), nats.ManualAck(),
-	); err != nil {
-		return err
-	}
-	if err := jetstream.JetStreamConsumer(
-		t.ctx, t.jetstream, t.receiptTopic, t.durable, t.onReceiptEvent,
-		nats.DeliverAll(), nats.ManualAck(),
-	); err != nil {
-		return err
-	}
-	return nil
-}
-
-// onSendToDeviceEvent is called in response to a message received on the
-// send-to-device events topic from the EDU server.
-func (t *OutputEDUConsumer) onSendToDeviceEvent(ctx context.Context, msg *nats.Msg) bool {
-	// Extract the send-to-device event from msg.
-	var ote api.OutputSendToDeviceEvent
-	if err := json.Unmarshal(msg.Data, &ote); err != nil {
-		log.WithError(err).Errorf("eduserver output log: message parse failed (expected send-to-device)")
-		return true
-	}
-
-	// only send send-to-device events which originated from us
-	_, originServerName, err := gomatrixserverlib.SplitID('@', ote.Sender)
-	if err != nil {
-		log.WithError(err).WithField("user_id", ote.Sender).Error("Failed to extract domain from send-to-device sender")
-		return true
-	}
-	if originServerName != t.ServerName {
-		log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere")
-		return true
-	}
-
-	_, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID)
-	if err != nil {
-		log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination")
-		return true
-	}
-
-	// Pack the EDU and marshal it
-	edu := &gomatrixserverlib.EDU{
-		Type:   gomatrixserverlib.MDirectToDevice,
-		Origin: string(t.ServerName),
-	}
-	tdm := gomatrixserverlib.ToDeviceMessage{
-		Sender:    ote.Sender,
-		Type:      ote.Type,
-		MessageID: util.RandomString(32),
-		Messages: map[string]map[string]json.RawMessage{
-			ote.UserID: {
-				ote.DeviceID: ote.Content,
-			},
-		},
-	}
-	if edu.Content, err = json.Marshal(tdm); err != nil {
-		log.WithError(err).Error("failed to marshal EDU JSON")
-		return true
-	}
-
-	log.Debugf("Sending send-to-device message into %q destination queue", destServerName)
-	if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil {
-		log.WithError(err).Error("failed to send EDU")
-		return false
-	}
-
-	return true
-}
-
-// onTypingEvent is called in response to a message received on the typing
-// events topic from the EDU server.
-func (t *OutputEDUConsumer) onTypingEvent(ctx context.Context, msg *nats.Msg) bool {
-	// Extract the typing event from msg.
-	var ote api.OutputTypingEvent
-	if err := json.Unmarshal(msg.Data, &ote); err != nil {
-		// Skip this msg but continue processing messages.
-		log.WithError(err).Errorf("eduserver output log: message parse failed (expected typing)")
-		_ = msg.Ack()
-		return true
-	}
-
-	// only send typing events which originated from us
-	_, typingServerName, err := gomatrixserverlib.SplitID('@', ote.Event.UserID)
-	if err != nil {
-		log.WithError(err).WithField("user_id", ote.Event.UserID).Error("Failed to extract domain from typing sender")
-		_ = msg.Ack()
-		return true
-	}
-	if typingServerName != t.ServerName {
-		return true
-	}
-
-	joined, err := t.db.GetJoinedHosts(ctx, ote.Event.RoomID)
-	if err != nil {
-		log.WithError(err).WithField("room_id", ote.Event.RoomID).Error("failed to get joined hosts for room")
-		return false
-	}
-
-	names := make([]gomatrixserverlib.ServerName, len(joined))
-	for i := range joined {
-		names[i] = joined[i].ServerName
-	}
-
-	edu := &gomatrixserverlib.EDU{Type: ote.Event.Type}
-	if edu.Content, err = json.Marshal(map[string]interface{}{
-		"room_id": ote.Event.RoomID,
-		"user_id": ote.Event.UserID,
-		"typing":  ote.Event.Typing,
-	}); err != nil {
-		log.WithError(err).Error("failed to marshal EDU JSON")
-		return true
-	}
-
-	if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
-		log.WithError(err).Error("failed to send EDU")
-		return false
-	}
-
-	return true
-}
-
-// onReceiptEvent is called in response to a message received on the receipt
-// events topic from the EDU server.
-func (t *OutputEDUConsumer) onReceiptEvent(ctx context.Context, msg *nats.Msg) bool {
-	// Extract the typing event from msg.
-	var receipt api.OutputReceiptEvent
-	if err := json.Unmarshal(msg.Data, &receipt); err != nil {
-		// Skip this msg but continue processing messages.
-		log.WithError(err).Errorf("eduserver output log: message parse failed (expected receipt)")
-		return true
-	}
-
-	// only send receipt events which originated from us
-	_, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID)
-	if err != nil {
-		log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender")
-		return true
-	}
-	if receiptServerName != t.ServerName {
-		return true
-	}
-
-	joined, err := t.db.GetJoinedHosts(ctx, receipt.RoomID)
-	if err != nil {
-		log.WithError(err).WithField("room_id", receipt.RoomID).Error("failed to get joined hosts for room")
-		return false
-	}
-
-	names := make([]gomatrixserverlib.ServerName, len(joined))
-	for i := range joined {
-		names[i] = joined[i].ServerName
-	}
-
-	content := map[string]api.FederationReceiptMRead{}
-	content[receipt.RoomID] = api.FederationReceiptMRead{
-		User: map[string]api.FederationReceiptData{
-			receipt.UserID: {
-				Data: api.ReceiptTS{
-					TS: receipt.Timestamp,
-				},
-				EventIDs: []string{receipt.EventID},
-			},
-		},
-	}
-
-	edu := &gomatrixserverlib.EDU{
-		Type:   gomatrixserverlib.MReceipt,
-		Origin: string(t.ServerName),
-	}
-	if edu.Content, err = json.Marshal(content); err != nil {
-		log.WithError(err).Error("failed to marshal EDU JSON")
-		return true
-	}
-
-	if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
-		log.WithError(err).Error("failed to send EDU")
-		return false
-	}
-
-	return true
-}

federationapi/consumers/keychange.go

@@ -18,9 +18,9 @@ import (
 	"context"
 	"encoding/json"
 
-	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/storage"
+	"github.com/matrix-org/dendrite/federationapi/types"
 	"github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -190,7 +190,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 
 	// Pack the EDU and marshal it
 	edu := &gomatrixserverlib.EDU{
-		Type:   eduserverAPI.MSigningKeyUpdate,
+		Type:   types.MSigningKeyUpdate,
 		Origin: string(t.serverName),
 	}
 	if edu.Content, err = json.Marshal(output); err != nil {

federationapi/consumers/presence.go

@@ -0,0 +1,143 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/storage"
+	fedTypes "github.com/matrix-org/dendrite/federationapi/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+)
+
+// OutputReceiptConsumer consumes events that originate in the clientapi.
+type OutputPresenceConsumer struct {
+	ctx                     context.Context
+	jetstream               nats.JetStreamContext
+	durable                 string
+	db                      storage.Database
+	queues                  *queue.OutgoingQueues
+	ServerName              gomatrixserverlib.ServerName
+	topic                   string
+	outboundPresenceEnabled bool
+}
+
+// NewOutputPresenceConsumer creates a new OutputPresenceConsumer. Call Start() to begin consuming events.
+func NewOutputPresenceConsumer(
+	process *process.ProcessContext,
+	cfg *config.FederationAPI,
+	js nats.JetStreamContext,
+	queues *queue.OutgoingQueues,
+	store storage.Database,
+) *OutputPresenceConsumer {
+	return &OutputPresenceConsumer{
+		ctx:                     process.Context(),
+		jetstream:               js,
+		queues:                  queues,
+		db:                      store,
+		ServerName:              cfg.Matrix.ServerName,
+		durable:                 cfg.Matrix.JetStream.Durable("FederationAPIPresenceConsumer"),
+		topic:                   cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		outboundPresenceEnabled: cfg.Matrix.Presence.EnableOutbound,
+	}
+}
+
+// Start consuming from the clientapi
+func (t *OutputPresenceConsumer) Start() error {
+	if !t.outboundPresenceEnabled {
+		return nil
+	}
+	return jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(),
+	)
+}
+
+// onMessage is called in response to a message received on the presence
+// events topic from the client api.
+func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+	// only send presence events which originated from us
+	userID := msg.Header.Get(jetstream.UserID)
+	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		log.WithError(err).WithField("user_id", userID).Error("failed to extract domain from receipt sender")
+		return true
+	}
+	if serverName != t.ServerName {
+		return true
+	}
+
+	presence := msg.Header.Get("presence")
+
+	ts, err := strconv.Atoi(msg.Header.Get("last_active_ts"))
+	if err != nil {
+		return true
+	}
+
+	joined, err := t.db.GetAllJoinedHosts(ctx)
+	if err != nil {
+		log.WithError(err).Error("failed to get joined hosts")
+		return true
+	}
+	if len(joined) == 0 {
+		return true
+	}
+
+	var statusMsg *string = nil
+	if data, ok := msg.Header["status_msg"]; ok && len(data) > 0 {
+		status := msg.Header.Get("status_msg")
+		statusMsg = &status
+	}
+
+	p := types.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(ts)}
+
+	content := fedTypes.Presence{
+		Push: []fedTypes.PresenceContent{
+			{
+				CurrentlyActive: p.CurrentlyActive(),
+				LastActiveAgo:   p.LastActiveAgo(),
+				Presence:        presence,
+				StatusMsg:       statusMsg,
+				UserID:          userID,
+			},
+		},
+	}
+
+	edu := &gomatrixserverlib.EDU{
+		Type:   gomatrixserverlib.MPresence,
+		Origin: string(t.ServerName),
+	}
+	if edu.Content, err = json.Marshal(content); err != nil {
+		log.WithError(err).Error("failed to marshal EDU JSON")
+		return true
+	}
+
+	log.Debugf("sending presence EDU to %d servers", len(joined))
+	if err = t.queues.SendEDU(edu, t.ServerName, joined); err != nil {
+		log.WithError(err).Error("failed to send EDU")
+		return false
+	}
+
+	return true
+}

federationapi/consumers/receipts.go

@@ -0,0 +1,141 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/storage"
+	fedTypes "github.com/matrix-org/dendrite/federationapi/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+)
+
+// OutputReceiptConsumer consumes events that originate in the clientapi.
+type OutputReceiptConsumer struct {
+	ctx        context.Context
+	jetstream  nats.JetStreamContext
+	durable    string
+	db         storage.Database
+	queues     *queue.OutgoingQueues
+	ServerName gomatrixserverlib.ServerName
+	topic      string
+}
+
+// NewOutputReceiptConsumer creates a new OutputReceiptConsumer. Call Start() to begin consuming typing events.
+func NewOutputReceiptConsumer(
+	process *process.ProcessContext,
+	cfg *config.FederationAPI,
+	js nats.JetStreamContext,
+	queues *queue.OutgoingQueues,
+	store storage.Database,
+) *OutputReceiptConsumer {
+	return &OutputReceiptConsumer{
+		ctx:        process.Context(),
+		jetstream:  js,
+		queues:     queues,
+		db:         store,
+		ServerName: cfg.Matrix.ServerName,
+		durable:    cfg.Matrix.JetStream.Durable("FederationAPIReceiptConsumer"),
+		topic:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+	}
+}
+
+// Start consuming from the clientapi
+func (t *OutputReceiptConsumer) Start() error {
+	return jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(),
+	)
+}
+
+// onMessage is called in response to a message received on the receipt
+// events topic from the client api.
+func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+	receipt := syncTypes.OutputReceiptEvent{
+		UserID:  msg.Header.Get(jetstream.UserID),
+		RoomID:  msg.Header.Get(jetstream.RoomID),
+		EventID: msg.Header.Get(jetstream.EventID),
+		Type:    msg.Header.Get("type"),
+	}
+
+	// only send receipt events which originated from us
+	_, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID)
+	if err != nil {
+		log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender")
+		return true
+	}
+	if receiptServerName != t.ServerName {
+		return true
+	}
+
+	timestamp, err := strconv.Atoi(msg.Header.Get("timestamp"))
+	if err != nil {
+		// If the message was invalid, log it and move on to the next message in the stream
+		log.WithError(err).Errorf("EDU output log: message parse failure")
+		sentry.CaptureException(err)
+		return true
+	}
+
+	receipt.Timestamp = gomatrixserverlib.Timestamp(timestamp)
+
+	joined, err := t.db.GetJoinedHosts(ctx, receipt.RoomID)
+	if err != nil {
+		log.WithError(err).WithField("room_id", receipt.RoomID).Error("failed to get joined hosts for room")
+		return false
+	}
+
+	names := make([]gomatrixserverlib.ServerName, len(joined))
+	for i := range joined {
+		names[i] = joined[i].ServerName
+	}
+
+	content := map[string]fedTypes.FederationReceiptMRead{}
+	content[receipt.RoomID] = fedTypes.FederationReceiptMRead{
+		User: map[string]fedTypes.FederationReceiptData{
+			receipt.UserID: {
+				Data: fedTypes.ReceiptTS{
+					TS: receipt.Timestamp,
+				},
+				EventIDs: []string{receipt.EventID},
+			},
+		},
+	}
+
+	edu := &gomatrixserverlib.EDU{
+		Type:   gomatrixserverlib.MReceipt,
+		Origin: string(t.ServerName),
+	}
+	if edu.Content, err = json.Marshal(content); err != nil {
+		log.WithError(err).Error("failed to marshal EDU JSON")
+		return true
+	}
+
+	if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
+		log.WithError(err).Error("failed to send EDU")
+		return false
+	}
+
+	return true
+}

federationapi/consumers/sendtodevice.go

@@ -0,0 +1,125 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/storage"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+)
+
+// OutputSendToDeviceConsumer consumes events that originate in the clientapi.
+type OutputSendToDeviceConsumer struct {
+	ctx        context.Context
+	jetstream  nats.JetStreamContext
+	durable    string
+	db         storage.Database
+	queues     *queue.OutgoingQueues
+	ServerName gomatrixserverlib.ServerName
+	topic      string
+}
+
+// NewOutputSendToDeviceConsumer creates a new OutputSendToDeviceConsumer. Call Start() to begin consuming send-to-device events.
+func NewOutputSendToDeviceConsumer(
+	process *process.ProcessContext,
+	cfg *config.FederationAPI,
+	js nats.JetStreamContext,
+	queues *queue.OutgoingQueues,
+	store storage.Database,
+) *OutputSendToDeviceConsumer {
+	return &OutputSendToDeviceConsumer{
+		ctx:        process.Context(),
+		jetstream:  js,
+		queues:     queues,
+		db:         store,
+		ServerName: cfg.Matrix.ServerName,
+		durable:    cfg.Matrix.JetStream.Durable("FederationAPIESendToDeviceConsumer"),
+		topic:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+	}
+}
+
+// Start consuming from the client api
+func (t *OutputSendToDeviceConsumer) Start() error {
+	return jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		nats.DeliverAll(), nats.ManualAck(),
+	)
+}
+
+// onMessage is called in response to a message received on the
+// send-to-device events topic from the client api.
+func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+	// only send send-to-device events which originated from us
+	sender := msg.Header.Get("sender")
+	_, originServerName, err := gomatrixserverlib.SplitID('@', sender)
+	if err != nil {
+		log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender")
+		return true
+	}
+	if originServerName != t.ServerName {
+		log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere")
+		return true
+	}
+	// Extract the send-to-device event from msg.
+	var ote syncTypes.OutputSendToDeviceEvent
+	if err = json.Unmarshal(msg.Data, &ote); err != nil {
+		log.WithError(err).Errorf("output log: message parse failed (expected send-to-device)")
+		return true
+	}
+
+	_, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID)
+	if err != nil {
+		log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination")
+		return true
+	}
+
+	// Pack the EDU and marshal it
+	edu := &gomatrixserverlib.EDU{
+		Type:   gomatrixserverlib.MDirectToDevice,
+		Origin: string(t.ServerName),
+	}
+	tdm := gomatrixserverlib.ToDeviceMessage{
+		Sender:    ote.Sender,
+		Type:      ote.Type,
+		MessageID: util.RandomString(32),
+		Messages: map[string]map[string]json.RawMessage{
+			ote.UserID: {
+				ote.DeviceID: ote.Content,
+			},
+		},
+	}
+	if edu.Content, err = json.Marshal(tdm); err != nil {
+		log.WithError(err).Error("failed to marshal EDU JSON")
+		return true
+	}
+
+	log.Debugf("Sending send-to-device message into %q destination queue", destServerName)
+	if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil {
+		log.WithError(err).Error("failed to send EDU")
+		return false
+	}
+
+	return true
+}

federationapi/consumers/typing.go

@@ -0,0 +1,119 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/storage"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+)
+
+// OutputTypingConsumer consumes events that originate in the clientapi.
+type OutputTypingConsumer struct {
+	ctx        context.Context
+	jetstream  nats.JetStreamContext
+	durable    string
+	db         storage.Database
+	queues     *queue.OutgoingQueues
+	ServerName gomatrixserverlib.ServerName
+	topic      string
+}
+
+// NewOutputTypingConsumer creates a new OutputTypingConsumer. Call Start() to begin consuming typing events.
+func NewOutputTypingConsumer(
+	process *process.ProcessContext,
+	cfg *config.FederationAPI,
+	js nats.JetStreamContext,
+	queues *queue.OutgoingQueues,
+	store storage.Database,
+) *OutputTypingConsumer {
+	return &OutputTypingConsumer{
+		ctx:        process.Context(),
+		jetstream:  js,
+		queues:     queues,
+		db:         store,
+		ServerName: cfg.Matrix.ServerName,
+		durable:    cfg.Matrix.JetStream.Durable("FederationAPITypingConsumer"),
+		topic:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
+	}
+}
+
+// Start consuming from the clientapi
+func (t *OutputTypingConsumer) Start() error {
+	return jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(),
+	)
+}
+
+// onMessage is called in response to a message received on the typing
+// events topic from the client api.
+func (t *OutputTypingConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+	// Extract the typing event from msg.
+	roomID := msg.Header.Get(jetstream.RoomID)
+	userID := msg.Header.Get(jetstream.UserID)
+	typing, err := strconv.ParseBool(msg.Header.Get("typing"))
+	if err != nil {
+		log.WithError(err).Errorf("EDU output log: typing parse failure")
+		return true
+	}
+
+	// only send typing events which originated from us
+	_, typingServerName, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		log.WithError(err).WithField("user_id", userID).Error("Failed to extract domain from typing sender")
+		_ = msg.Ack()
+		return true
+	}
+	if typingServerName != t.ServerName {
+		return true
+	}
+
+	joined, err := t.db.GetJoinedHosts(ctx, roomID)
+	if err != nil {
+		log.WithError(err).WithField("room_id", roomID).Error("failed to get joined hosts for room")
+		return false
+	}
+
+	names := make([]gomatrixserverlib.ServerName, len(joined))
+	for i := range joined {
+		names[i] = joined[i].ServerName
+	}
+
+	edu := &gomatrixserverlib.EDU{Type: "m.typing"}
+	if edu.Content, err = json.Marshal(map[string]interface{}{
+		"room_id": roomID,
+		"user_id": userID,
+		"typing":  typing,
+	}); err != nil {
+		log.WithError(err).Error("failed to marshal EDU JSON")
+		return true
+	}
+	if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
+		log.WithError(err).Error("failed to send EDU")
+		return false
+	}
+
+	return true
+}

federationapi/federationapi.go

@@ -16,12 +16,12 @@ package federationapi
 
 import (
 	"github.com/gorilla/mux"
-	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/consumers"
 	"github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/inthttp"
+	"github.com/matrix-org/dendrite/federationapi/producers"
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
@@ -31,6 +31,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
 
@@ -46,6 +47,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI) {
 
 // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.
 func AddPublicRoutes(
+	process *process.ProcessContext,
 	fedRouter, keyRouter, wellKnownRouter *mux.Router,
 	cfg *config.FederationAPI,
 	userAPI userapi.UserInternalAPI,
@@ -53,16 +55,27 @@ func AddPublicRoutes(
 	keyRing gomatrixserverlib.JSONVerifier,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	federationAPI federationAPI.FederationInternalAPI,
-	eduAPI eduserverAPI.EDUServerInputAPI,
 	keyAPI keyserverAPI.KeyInternalAPI,
 	mscCfg *config.MSCs,
 	servers federationAPI.ServersInRoomProvider,
 ) {
+
+	js, _ := jetstream.Prepare(process, &cfg.Matrix.JetStream)
+	producer := &producers.SyncAPIProducer{
+		JetStream:              js,
+		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		ServerName:             cfg.Matrix.ServerName,
+		UserAPI:                userAPI,
+	}
+
 	routing.Setup(
 		fedRouter, keyRouter, wellKnownRouter, cfg, rsAPI,
-		eduAPI, federationAPI, keyRing,
+		federationAPI, keyRing,
 		federation, userAPI, keyAPI, mscCfg,
-		servers,
+		servers, producer,
 	)
 }
 
@@ -112,19 +125,36 @@ func NewInternalAPI(
 	if err = rsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start room server consumer")
 	}
-
+	tsConsumer := consumers.NewOutputSendToDeviceConsumer(
-	tsConsumer := consumers.NewOutputEDUConsumer(
 		base.ProcessContext, cfg, js, queues, federationDB,
 	)
-	if err := tsConsumer.Start(); err != nil {
+	if err = tsConsumer.Start(); err != nil {
-		logrus.WithError(err).Panic("failed to start typing server consumer")
+		logrus.WithError(err).Panic("failed to start send-to-device consumer")
+	}
+	receiptConsumer := consumers.NewOutputReceiptConsumer(
+		base.ProcessContext, cfg, js, queues, federationDB,
+	)
+	if err = receiptConsumer.Start(); err != nil {
+		logrus.WithError(err).Panic("failed to start receipt consumer")
+	}
+	typingConsumer := consumers.NewOutputTypingConsumer(
+		base.ProcessContext, cfg, js, queues, federationDB,
+	)
+	if err = typingConsumer.Start(); err != nil {
+		logrus.WithError(err).Panic("failed to start typing consumer")
 	}
 	keyConsumer := consumers.NewKeyChangeConsumer(
 		base.ProcessContext, &base.Cfg.KeyServer, js, queues, federationDB, rsAPI,
 	)
-	if err := keyConsumer.Start(); err != nil {
+	if err = keyConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start key server consumer")
 	}
 
+	presenceConsumer := consumers.NewOutputPresenceConsumer(
+		base.ProcessContext, cfg, js, queues, federationDB,
+	)
+	if err = presenceConsumer.Start(); err != nil {
+		logrus.WithError(err).Panic("failed to start presence consumer")
+	}
 	return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, federation, stats, caches, queues, keyRing)
 }

federationapi/federationapi_test.go

@@ -30,7 +30,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	fsAPI := base.FederationAPIHTTPClient()
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, &cfg.MSCs, nil)
+	federationapi.AddPublicRoutes(base.ProcessContext, base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, &cfg.MSCs, nil)
 	baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true)
 	defer cancel()
 	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))

federationapi/internal/perform.go

@@ -392,17 +392,17 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 
 	// we have the peek state now so let's process regardless of whether upstream gives up
 	ctx = context.Background()
-
 	respState := respPeek.ToRespState()
-	authEvents := respState.AuthEvents.UntrustedEvents(respPeek.RoomVersion)
+
 	// authenticate the state returned (check its auth events etc)
 	// the equivalent of CheckSendJoinResponse()
+	authEvents, _, err := respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName))
+	if err != nil {
+		return fmt.Errorf("error checking state returned from peeking: %w", err)
+	}
 	if err = sanityCheckAuthChain(authEvents); err != nil {
 		return fmt.Errorf("sanityCheckAuthChain: %w", err)
 	}
-	if err = respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName)); err != nil {
-		return fmt.Errorf("error checking state returned from peeking: %w", err)
-	}
 
 	// If we've got this far, the remote server is peeking.
 	if renewing {

federationapi/producers/syncapi.go

@@ -0,0 +1,163 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package producers
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+	"time"
+
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+)
+
+// SyncAPIProducer produces events for the sync API server to consume
+type SyncAPIProducer struct {
+	TopicReceiptEvent      string
+	TopicSendToDeviceEvent string
+	TopicTypingEvent       string
+	TopicPresenceEvent     string
+	JetStream              nats.JetStreamContext
+	ServerName             gomatrixserverlib.ServerName
+	UserAPI                userapi.UserInternalAPI
+}
+
+func (p *SyncAPIProducer) SendReceipt(
+	ctx context.Context,
+	userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp,
+) error {
+	m := &nats.Msg{
+		Subject: p.TopicReceiptEvent,
+		Header:  nats.Header{},
+	}
+	m.Header.Set(jetstream.UserID, userID)
+	m.Header.Set(jetstream.RoomID, roomID)
+	m.Header.Set(jetstream.EventID, eventID)
+	m.Header.Set("type", receiptType)
+	m.Header.Set("timestamp", strconv.Itoa(int(timestamp)))
+
+	log.WithFields(log.Fields{}).Tracef("Producing to topic '%s'", p.TopicReceiptEvent)
+	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}
+
+func (p *SyncAPIProducer) SendToDevice(
+	ctx context.Context, sender, userID, deviceID, eventType string,
+	message interface{},
+) error {
+	devices := []string{}
+	_, domain, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return err
+	}
+
+	// If the event is targeted locally then we want to expand the wildcard
+	// out into individual device IDs so that we can send them to each respective
+	// device. If the event isn't targeted locally then we can't expand the
+	// wildcard as we don't know about the remote devices, so instead we leave it
+	// as-is, so that the federation sender can send it on with the wildcard intact.
+	if domain == p.ServerName && deviceID == "*" {
+		var res userapi.QueryDevicesResponse
+		err = p.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{
+			UserID: userID,
+		}, &res)
+		if err != nil {
+			return err
+		}
+		for _, dev := range res.Devices {
+			devices = append(devices, dev.ID)
+		}
+	} else {
+		devices = append(devices, deviceID)
+	}
+
+	js, err := json.Marshal(message)
+	if err != nil {
+		return err
+	}
+
+	log.WithFields(log.Fields{
+		"user_id":     userID,
+		"num_devices": len(devices),
+		"type":        eventType,
+	}).Tracef("Producing to topic '%s'", p.TopicSendToDeviceEvent)
+	for _, device := range devices {
+		ote := &types.OutputSendToDeviceEvent{
+			UserID:   userID,
+			DeviceID: device,
+			SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{
+				Sender:  sender,
+				Type:    eventType,
+				Content: js,
+			},
+		}
+
+		eventJSON, err := json.Marshal(ote)
+		if err != nil {
+			log.WithError(err).Error("sendToDevice failed json.Marshal")
+			return err
+		}
+		m := &nats.Msg{
+			Subject: p.TopicSendToDeviceEvent,
+			Data:    eventJSON,
+			Header:  nats.Header{},
+		}
+		m.Header.Set("sender", sender)
+		m.Header.Set(jetstream.UserID, userID)
+
+		if _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)); err != nil {
+			log.WithError(err).Error("sendToDevice failed t.Producer.SendMessage")
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *SyncAPIProducer) SendTyping(
+	ctx context.Context, userID, roomID string, typing bool, timeoutMS int64,
+) error {
+	m := &nats.Msg{
+		Subject: p.TopicTypingEvent,
+		Header:  nats.Header{},
+	}
+	m.Header.Set(jetstream.UserID, userID)
+	m.Header.Set(jetstream.RoomID, roomID)
+	m.Header.Set("typing", strconv.FormatBool(typing))
+	m.Header.Set("timeout_ms", strconv.Itoa(int(timeoutMS)))
+	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}
+
+func (p *SyncAPIProducer) SendPresence(
+	ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveAgo int64,
+) error {
+	m := nats.NewMsg(p.TopicPresenceEvent)
+	m.Header.Set(jetstream.UserID, userID)
+	m.Header.Set("presence", presence.String())
+	if statusMsg != nil {
+		m.Header.Set("status_msg", *statusMsg)
+	}
+	lastActiveTS := gomatrixserverlib.AsTimestamp(time.Now().Add(-(time.Duration(lastActiveAgo) * time.Millisecond)))
+
+	m.Header.Set("last_active_ts", strconv.Itoa(int(lastActiveTS)))
+	log.Debugf("Sending presence to syncAPI: %+v", m.Header)
+	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}

federationapi/queue/queue.go

@@ -104,28 +104,31 @@ func NewOutgoingQueues(
 	}
 	// Look up which servers we have pending items for and then rehydrate those queues.
 	if !disabled {
-		time.AfterFunc(time.Second*5, func() {
+		serverNames := map[gomatrixserverlib.ServerName]struct{}{}
-			serverNames := map[gomatrixserverlib.ServerName]struct{}{}
+		if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil {
-			if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil {
+			for _, serverName := range names {
-				for _, serverName := range names {
+				serverNames[serverName] = struct{}{}
-					serverNames[serverName] = struct{}{}
-				}
-			} else {
-				log.WithError(err).Error("Failed to get PDU server names for destination queue hydration")
 			}
-			if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil {
+		} else {
-				for _, serverName := range names {
+			log.WithError(err).Error("Failed to get PDU server names for destination queue hydration")
-					serverNames[serverName] = struct{}{}
+		}
-				}
+		if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil {
-			} else {
+			for _, serverName := range names {
-				log.WithError(err).Error("Failed to get EDU server names for destination queue hydration")
+				serverNames[serverName] = struct{}{}
 			}
-			for serverName := range serverNames {
+		} else {
-				if queue := queues.getQueue(serverName); queue != nil {
+			log.WithError(err).Error("Failed to get EDU server names for destination queue hydration")
-					queue.wakeQueueIfNeeded()
+		}
-				}
+		offset, step := time.Second*5, time.Second
+		if max := len(serverNames); max > 120 {
+			step = (time.Second * 120) / time.Duration(max)
+		}
+		for serverName := range serverNames {
+			if queue := queues.getQueue(serverName); queue != nil {
+				time.AfterFunc(offset, queue.wakeQueueIfNeeded)
+				offset += step
 			}
-		})
+		}
 	}
 	return queues
 }

federationapi/routing/routing.go

@@ -19,8 +19,8 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/producers"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
@@ -45,7 +45,6 @@ func Setup(
 	fedMux, keyMux, wkMux *mux.Router,
 	cfg *config.FederationAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	eduAPI eduserverAPI.EDUServerInputAPI,
 	fsAPI federationAPI.FederationInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
@@ -53,6 +52,7 @@ func Setup(
 	keyAPI keyserverAPI.KeyInternalAPI,
 	mscCfg *config.MSCs,
 	servers federationAPI.ServersInRoomProvider,
+	producer *producers.SyncAPIProducer,
 ) {
 	prometheus.MustRegister(
 		pduCountTotal, eduCountTotal,
@@ -121,7 +121,7 @@ func Setup(
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return Send(
 				httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]),
-				cfg, rsAPI, eduAPI, keyAPI, keys, federation, mu, servers,
+				cfg, rsAPI, keyAPI, keys, federation, mu, servers, producer,
 			)
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)

federationapi/routing/send.go

@@ -23,12 +23,14 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/producers"
+	"github.com/matrix-org/dendrite/federationapi/types"
 	"github.com/matrix-org/dendrite/internal"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
@@ -81,12 +83,12 @@ func Send(
 	txnID gomatrixserverlib.TransactionID,
 	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
-	eduAPI eduserverAPI.EDUServerInputAPI,
 	keyAPI keyapi.KeyInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
 	mu *internal.MutexByRoom,
 	servers federationAPI.ServersInRoomProvider,
+	producer *producers.SyncAPIProducer,
 ) util.JSONResponse {
 	// First we should check if this origin has already submitted this
 	// txn ID to us. If they have and the txnIDs map contains an entry,
@@ -120,13 +122,14 @@ func Send(
 	defer inFlightTxnsPerOrigin.Delete(index)
 
 	t := txnReq{
-		rsAPI:      rsAPI,
+		rsAPI:                  rsAPI,
-		eduAPI:     eduAPI,
+		keys:                   keys,
-		keys:       keys,
+		federation:             federation,
-		federation: federation,
+		servers:                servers,
-		servers:    servers,
+		keyAPI:                 keyAPI,
-		keyAPI:     keyAPI,
+		roomsMu:                mu,
-		roomsMu:    mu,
+		producer:               producer,
+		inboundPresenceEnabled: cfg.Matrix.Presence.EnableInbound,
 	}
 
 	var txnEvents struct {
@@ -178,13 +181,14 @@ func Send(
 
 type txnReq struct {
 	gomatrixserverlib.Transaction
-	rsAPI      api.RoomserverInternalAPI
+	rsAPI                  api.RoomserverInternalAPI
-	eduAPI     eduserverAPI.EDUServerInputAPI
+	keyAPI                 keyapi.KeyInternalAPI
-	keyAPI     keyapi.KeyInternalAPI
+	keys                   gomatrixserverlib.JSONVerifier
-	keys       gomatrixserverlib.JSONVerifier
+	federation             txnFederationClient
-	federation txnFederationClient
+	roomsMu                *internal.MutexByRoom
-	roomsMu    *internal.MutexByRoom
+	servers                federationAPI.ServersInRoomProvider
-	servers    federationAPI.ServersInRoomProvider
+	producer               *producers.SyncAPIProducer
+	inboundPresenceEnabled bool
 }
 
 // A subset of FederationClient functionality that txn requires. Useful for testing.
@@ -323,8 +327,8 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				util.GetLogger(ctx).Debugf("Dropping typing event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin)
 				continue
 			}
-			if err := eduserverAPI.SendTyping(ctx, t.eduAPI, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil {
+			if err := t.producer.SendTyping(ctx, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil {
-				util.GetLogger(ctx).WithError(err).Error("Failed to send typing event to edu server")
+				util.GetLogger(ctx).WithError(err).Error("Failed to send typing event to JetStream")
 			}
 		case gomatrixserverlib.MDirectToDevice:
 			// https://matrix.org/docs/spec/server_server/r0.1.3#m-direct-to-device-schema
@@ -336,12 +340,12 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			for userID, byUser := range directPayload.Messages {
 				for deviceID, message := range byUser {
 					// TODO: check that the user and the device actually exist here
-					if err := eduserverAPI.SendToDevice(ctx, t.eduAPI, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil {
+					if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil {
 						util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
 							"sender":    directPayload.Sender,
 							"user_id":   userID,
 							"device_id": deviceID,
-						}).Error("Failed to send send-to-device event to edu server")
+						}).Error("Failed to send send-to-device event to JetStream")
 					}
 				}
 			}
@@ -349,7 +353,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			t.processDeviceListUpdate(ctx, e)
 		case gomatrixserverlib.MReceipt:
 			// https://matrix.org/docs/spec/server_server/r0.1.4#receipts
-			payload := map[string]eduserverAPI.FederationReceiptMRead{}
+			payload := map[string]types.FederationReceiptMRead{}
 
 			if err := json.Unmarshal(e.Content, &payload); err != nil {
 				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal receipt event")
@@ -373,23 +377,47 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 							"user_id": userID,
 							"room_id": roomID,
 							"events":  mread.EventIDs,
-						}).Error("Failed to send receipt event to edu server")
+						}).Error("Failed to send receipt event to JetStream")
 						continue
 					}
 				}
 			}
-		case eduserverAPI.MSigningKeyUpdate:
+		case types.MSigningKeyUpdate:
 			if err := t.processSigningKeyUpdate(ctx, e); err != nil {
 				logrus.WithError(err).Errorf("Failed to process signing key update")
 			}
+		case gomatrixserverlib.MPresence:
+			if t.inboundPresenceEnabled {
+				if err := t.processPresence(ctx, e); err != nil {
+					logrus.WithError(err).Errorf("Failed to process presence update")
+				}
+			}
 		default:
 			util.GetLogger(ctx).WithField("type", e.Type).Debug("Unhandled EDU")
 		}
 	}
 }
 
+// processPresence handles m.receipt events
+func (t *txnReq) processPresence(ctx context.Context, e gomatrixserverlib.EDU) error {
+	payload := types.Presence{}
+	if err := json.Unmarshal(e.Content, &payload); err != nil {
+		return err
+	}
+	for _, content := range payload.Push {
+		presence, ok := syncTypes.PresenceFromString(content.Presence)
+		if !ok {
+			continue
+		}
+		if err := t.producer.SendPresence(ctx, content.UserID, presence, content.StatusMsg, content.LastActiveAgo); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverlib.EDU) error {
-	var updatePayload eduserverAPI.CrossSigningKeyUpdate
+	var updatePayload keyapi.CrossSigningKeyUpdate
 	if err := json.Unmarshal(e.Content, &updatePayload); err != nil {
 		util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
 			"user_id": updatePayload.UserID,
@@ -416,7 +444,7 @@ func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverli
 	return nil
 }
 
-// processReceiptEvent sends receipt events to the edu server
+// processReceiptEvent sends receipt events to JetStream
 func (t *txnReq) processReceiptEvent(ctx context.Context,
 	userID, roomID, receiptType string,
 	timestamp gomatrixserverlib.Timestamp,
@@ -424,17 +452,7 @@ func (t *txnReq) processReceiptEvent(ctx context.Context,
 ) error {
 	// store every event
 	for _, eventID := range eventIDs {
-		req := eduserverAPI.InputReceiptEventRequest{
+		if err := t.producer.SendReceipt(ctx, userID, roomID, eventID, receiptType, timestamp); err != nil {
-			InputReceiptEvent: eduserverAPI.InputReceiptEvent{
-				UserID:    userID,
-				RoomID:    roomID,
-				EventID:   eventID,
-				Type:      receiptType,
-				Timestamp: timestamp,
-			},
-		}
-		resp := eduserverAPI.InputReceiptEventResponse{}
-		if err := t.eduAPI.InputReceiptEvent(ctx, &req, &resp); err != nil {
 			return fmt.Errorf("unable to set receipt event: %w", err)
 		}
 	}

federationapi/routing/send_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 	"time"
 
-	eduAPI "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -53,44 +52,6 @@ func init() {
 	}
 }
 
-type testEDUProducer struct {
-	// this producer keeps track of calls to InputTypingEvent
-	invocations []eduAPI.InputTypingEventRequest
-}
-
-func (p *testEDUProducer) InputTypingEvent(
-	ctx context.Context,
-	request *eduAPI.InputTypingEventRequest,
-	response *eduAPI.InputTypingEventResponse,
-) error {
-	p.invocations = append(p.invocations, *request)
-	return nil
-}
-
-func (p *testEDUProducer) InputSendToDeviceEvent(
-	ctx context.Context,
-	request *eduAPI.InputSendToDeviceEventRequest,
-	response *eduAPI.InputSendToDeviceEventResponse,
-) error {
-	return nil
-}
-
-func (o *testEDUProducer) InputReceiptEvent(
-	ctx context.Context,
-	request *eduAPI.InputReceiptEventRequest,
-	response *eduAPI.InputReceiptEventResponse,
-) error {
-	return nil
-}
-
-func (o *testEDUProducer) InputCrossSigningKeyUpdate(
-	ctx context.Context,
-	request *eduAPI.InputCrossSigningKeyUpdateRequest,
-	response *eduAPI.InputCrossSigningKeyUpdateResponse,
-) error {
-	return nil
-}
-
 type testRoomserverAPI struct {
 	api.RoomserverInternalAPITrace
 	inputRoomEvents           []api.InputRoomEvent
@@ -225,7 +186,6 @@ func (c *txnFedClient) LookupMissingEvents(ctx context.Context, s gomatrixserver
 func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq {
 	t := &txnReq{
 		rsAPI:      rsAPI,
-		eduAPI:     &testEDUProducer{},
 		keys:       &test.NopJSONVerifier{},
 		federation: fedClient,
 		roomsMu:    internal.NewMutexByRoom(),

federationapi/types/types.go

@@ -18,6 +18,8 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
+const MSigningKeyUpdate = "m.signing_key_update" // TODO: move to gomatrixserverlib
+
 // A JoinedHost is a server that is joined to a matrix room.
 type JoinedHost struct {
 	// The MemberEventID of a m.room.member join event.
@@ -51,3 +53,28 @@ type InboundPeek struct {
 	RenewedTimestamp  int64
 	RenewalInterval   int64
 }
+
+type FederationReceiptMRead struct {
+	User map[string]FederationReceiptData `json:"m.read"`
+}
+
+type FederationReceiptData struct {
+	Data     ReceiptTS `json:"data"`
+	EventIDs []string  `json:"event_ids"`
+}
+
+type ReceiptTS struct {
+	TS gomatrixserverlib.Timestamp `json:"ts"`
+}
+
+type Presence struct {
+	Push []PresenceContent `json:"push"`
+}
+
+type PresenceContent struct {
+	CurrentlyActive bool    `json:"currently_active,omitempty"`
+	LastActiveAgo   int64   `json:"last_active_ago"`
+	Presence        string  `json:"presence"`
+	StatusMsg       *string `json:"status_msg,omitempty"`
+	UserID          string  `json:"user_id"`
+}

go.mod

@@ -38,8 +38,8 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5
-	github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278
+	github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
 	github.com/morikuni/aec v1.0.0 // indirect
@@ -48,8 +48,6 @@ require (
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31
-	github.com/onsi/ginkgo v1.16.4 // indirect
-	github.com/onsi/gomega v1.13.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
@@ -64,9 +62,8 @@ require (
 	go.uber.org/atomic v1.9.0
 	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
 	golang.org/x/image v0.0.0-20211028202545-6944b10bf410
-	golang.org/x/mobile v0.0.0-20220112015953-858099ff7816
+	golang.org/x/mobile v0.0.0-20220325161704-447654d348e3
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
-	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	gopkg.in/h2non/bimg.v1 v1.1.5
 	gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -1,6 +1,8 @@
 bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
@@ -31,7 +33,12 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
+dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
+dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0 h1:QUqcb7BOcBU2p7Nax7pESOb8hrZYtI0Ts6j4v4mvcQo=
@@ -117,6 +124,7 @@ github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQy
 github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y=
 github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ=
 github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
@@ -135,6 +143,7 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY
 github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
@@ -154,6 +163,7 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
@@ -166,6 +176,8 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
+github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
+github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -276,6 +288,7 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
@@ -345,10 +358,12 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -370,6 +385,7 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
 github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
@@ -403,6 +419,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
@@ -433,6 +450,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -440,6 +458,8 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -478,6 +498,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -499,6 +520,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
@@ -517,6 +540,7 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
@@ -541,9 +565,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
-github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0=
-github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4=
 github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
 github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
@@ -618,6 +641,7 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj
 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
 github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
+github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -700,6 +724,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -919,15 +944,26 @@ github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI
 github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
 github.com/libp2p/go-yamux/v2 v2.0.0 h1:vSGhAy5u6iHBq11ZDcyHH4Blcf9xlBhT4WQDoOE90LU=
 github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U=
+github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A=
+github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
 github.com/lunixbochs/vtclean v0.0.0-20160125035106-4fbf7632a2c6/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
+github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
 github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
+github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
+github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
+github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
+github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc=
+github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
+github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y=
+github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
 github.com/masterzen/azure-sdk-for-go v3.2.0-beta.0.20161014135628-ee4f0065d00c+incompatible/go.mod h1:mf8fjOu33zCqxUjuiU3I8S1lJMyEAlH+0F2+M5xl3hE=
 github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc=
 github.com/masterzen/winrm v0.0.0-20161014151040-7a535cd943fc/go.mod h1:CfZSN7zwz5gJiFhZJz49Uzk7mEBHIceWmbFmYx7Hf7E=
@@ -941,10 +977,10 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 h1:IINbE/0jSYGb7M31StazufyIQdYWSivRlhuns3JYPOM=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5 h1:Fkennny7+Z/5pygrhjFMZbz1j++P2hhhLoT7NO3p8DQ=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
-github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278 h1:lRrvMMv7x1FIVW1mcBdU89lvbgAXKz6RyYR0VQTAr3E=
+github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d h1:1+T4eOPRsf6cr0lMPW4oO2k8TTHm4mqIh65kpEID5Rk=
-github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
+github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@@ -977,6 +1013,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
+github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
 github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
@@ -1087,11 +1124,12 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
+github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
+github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e h1:5tEHLzvDeS6IeqO2o9FFhsE3V2erYj8FlMt2J91wzsk=
 github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc=
 github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c h1:G2qsv7D0rY94HAu8pXmElMluuMHQ85waxIDQBhIzV2Q=
 github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
-github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
@@ -1115,6 +1153,7 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
@@ -1159,6 +1198,7 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -1178,6 +1218,7 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY
 github.com/pressly/goose v2.7.0+incompatible h1:PWejVEv07LCerQEzMMeAtjuyCKbyprZ/LBa6K5P0OCQ=
 github.com/pressly/goose v2.7.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
 github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -1193,6 +1234,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -1202,6 +1244,7 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9
 github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -1230,7 +1273,29 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
 github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
+github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
+github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
+github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
+github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
+github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
+github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
+github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
+github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
+github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
+github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
+github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
+github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
+github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
+github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
+github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
+github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
+github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
 github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -1246,6 +1311,8 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
+github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
 github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
 github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU=
 github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
@@ -1287,6 +1354,7 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
@@ -1321,6 +1389,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
+github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
@@ -1359,6 +1429,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
 github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
@@ -1368,6 +1439,7 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
 go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
+go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
@@ -1393,16 +1465,20 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
 go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
 go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM=
 go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
+go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
+golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -1445,6 +1521,7 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMx
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1458,8 +1535,9 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mobile v0.0.0-20220112015953-858099ff7816 h1:jhDgkcu3yQ4tasBZ+1YwDmK7eFmuVf1w1k+NGGGxfmE=
 golang.org/x/mobile v0.0.0-20220112015953-858099ff7816/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
+golang.org/x/mobile v0.0.0-20220325161704-447654d348e3 h1:ZDL7hDvJEQEcHVkoZawKmRUgbqn1pOIzb8EinBh5csU=
+golang.org/x/mobile v0.0.0-20220325161704-447654d348e3/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@@ -1473,12 +1551,15 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -1527,11 +1608,14 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1547,6 +1631,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1556,6 +1641,7 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1604,6 +1690,7 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1629,6 +1716,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1639,8 +1727,8 @@ golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
+golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 h1:D1v9ucDTYBtbz5vNuBbAhIMAGhQhJ6Ym5ah3maMVNX4=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1664,7 +1752,9 @@ golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13W
 golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1720,6 +1810,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 h1:YuekqPskqwCCPM79F1X5Dhv4ezTCj+Ki1oNwiafxkA0=
 golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
@@ -1737,6 +1828,9 @@ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1754,6 +1848,8 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
@@ -1762,6 +1858,9 @@ google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
+google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -1795,6 +1894,9 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8=
 google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
+google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1888,6 +1990,8 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -1935,3 +2039,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

internal/caching/cache_typing.go

@@ -14,7 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cache
+package caching
 
 import (
 	"sync"
@@ -53,8 +53,8 @@ func (t *EDUCache) newRoomData() *roomData {
 	}
 }
 
-// New returns a new EDUCache initialised for use.
+// NewTypingCache returns a new EDUCache initialised for use.
-func New() *EDUCache {
+func NewTypingCache() *EDUCache {
 	return &EDUCache{data: make(map[string]*roomData)}
 }
 

internal/caching/cache_typing_test.go

@@ -14,7 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cache
+package caching
 
 import (
 	"testing"
@@ -24,9 +24,9 @@ import (
 )
 
 func TestEDUCache(t *testing.T) {
-	tCache := New()
+	tCache := NewTypingCache()
 	if tCache == nil {
-		t.Fatal("New failed")
+		t.Fatal("NewTypingCache failed")
 	}
 
 	t.Run("AddTypingUser", func(t *testing.T) {

internal/eventutil/types.go

@@ -17,6 +17,8 @@ package eventutil
 import (
 	"errors"
 	"strconv"
+
+	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 // ErrProfileNoExists is returned when trying to lookup a user's profile that
@@ -26,9 +28,10 @@ var ErrProfileNoExists = errors.New("no known profile for given user ID")
 // AccountData represents account data sent from the client API server to the
 // sync API server
 type AccountData struct {
-	RoomID     string          `json:"room_id"`
+	RoomID       string              `json:"room_id"`
-	Type       string          `json:"type"`
+	Type         string              `json:"type"`
-	ReadMarker *ReadMarkerJSON `json:"read_marker,omitempty"` // optional
+	ReadMarker   *ReadMarkerJSON     `json:"read_marker,omitempty"`   // optional
+	IgnoredUsers *types.IgnoredUsers `json:"ignored_users,omitempty"` // optional
 }
 
 type ReadMarkerJSON struct {

internal/test/config.go

@@ -78,8 +78,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress())
 	cfg.Global.PrivateKeyPath = config.Path(serverKeyPath)
 
-	cfg.FederationAPI.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)}
-
 	cfg.MediaAPI.BasePath = config.Path(mediaBasePath)
 
 	cfg.Global.JetStream.Addresses = []string{kafkaURI}
@@ -97,7 +95,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database)
 
 	cfg.AppServiceAPI.InternalAPI.Listen = assignAddress()
-	cfg.EDUServer.InternalAPI.Listen = assignAddress()
 	cfg.FederationAPI.InternalAPI.Listen = assignAddress()
 	cfg.KeyServer.InternalAPI.Listen = assignAddress()
 	cfg.MediaAPI.InternalAPI.Listen = assignAddress()
@@ -106,7 +103,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
 	cfg.UserAPI.InternalAPI.Listen = assignAddress()
 
 	cfg.AppServiceAPI.InternalAPI.Connect = cfg.AppServiceAPI.InternalAPI.Listen
-	cfg.EDUServer.InternalAPI.Connect = cfg.EDUServer.InternalAPI.Listen
 	cfg.FederationAPI.InternalAPI.Connect = cfg.FederationAPI.InternalAPI.Listen
 	cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen
 	cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen

internal/version.go

@@ -16,8 +16,8 @@ var build string
 
 const (
 	VersionMajor = 0
-	VersionMinor = 7
+	VersionMinor = 8
-	VersionPatch = 0
+	VersionPatch = 1
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/api/api.go

@@ -21,7 +21,6 @@ import (
 	"strings"
 	"time"
 
-	eduapi "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/keyserver/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -66,14 +65,25 @@ const (
 
 // DeviceMessage represents the message produced into Kafka by the key server.
 type DeviceMessage struct {
-	Type                                DeviceMessageType `json:"Type,omitempty"`
+	Type                         DeviceMessageType `json:"Type,omitempty"`
-	*DeviceKeys                         `json:"DeviceKeys,omitempty"`
+	*DeviceKeys                  `json:"DeviceKeys,omitempty"`
-	*eduapi.OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"`
+	*OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"`
 	// A monotonically increasing number which represents device changes for this user.
 	StreamID       int64
 	DeviceChangeID int64
 }
 
+// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log
+type OutputCrossSigningKeyUpdate struct {
+	CrossSigningKeyUpdate `json:"signing_keys"`
+}
+
+type CrossSigningKeyUpdate struct {
+	MasterKey      *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"`
+	SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"`
+	UserID         string                             `json:"user_id"`
+}
+
 // DeviceKeysEqual returns true if the device keys updates contain the
 // same display name and key JSON. This will return false if either of
 // the updates is not a device keys update, or if the user ID/device ID

keyserver/internal/cross_signing.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"strings"
 
-	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/keyserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -246,7 +245,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 	}
 
 	// Finally, generate a notification that we updated the keys.
-	update := eduserverAPI.CrossSigningKeyUpdate{
+	update := api.CrossSigningKeyUpdate{
 		UserID: req.UserID,
 	}
 	if mk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster]; ok {
@@ -337,7 +336,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req
 	for userID := range req.Signatures {
 		masterKey := queryRes.MasterKeys[userID]
 		selfSigningKey := queryRes.SelfSigningKeys[userID]
-		update := eduserverAPI.CrossSigningKeyUpdate{
+		update := api.CrossSigningKeyUpdate{
 			UserID:         userID,
 			MasterKey:      &masterKey,
 			SelfSigningKey: &selfSigningKey,

keyserver/internal/device_list_update.go

@@ -157,8 +157,15 @@ func (u *DeviceListUpdater) Start() error {
 	if err != nil {
 		return err
 	}
+	offset, step := time.Second*10, time.Second
+	if max := len(staleLists); max > 120 {
+		step = (time.Second * 120) / time.Duration(max)
+	}
 	for _, userID := range staleLists {
-		u.notifyWorkers(userID)
+		time.AfterFunc(offset, func() {
+			u.notifyWorkers(userID)
+		})
+		offset += step
 	}
 	return nil
 }

keyserver/producers/keychange.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/json"
 
-	eduapi "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/keyserver/storage"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -70,10 +69,10 @@ func (p *KeyChange) ProduceKeyChanges(keys []api.DeviceMessage) error {
 	return nil
 }
 
-func (p *KeyChange) ProduceSigningKeyUpdate(key eduapi.CrossSigningKeyUpdate) error {
+func (p *KeyChange) ProduceSigningKeyUpdate(key api.CrossSigningKeyUpdate) error {
 	output := &api.DeviceMessage{
 		Type: api.TypeCrossSigningUpdate,
-		OutputCrossSigningKeyUpdate: &eduapi.OutputCrossSigningKeyUpdate{
+		OutputCrossSigningKeyUpdate: &api.OutputCrossSigningKeyUpdate{
 			CrossSigningKeyUpdate: key,
 		},
 	}

mediaapi/routing/upload.go

@@ -169,7 +169,7 @@ func (r *uploadRequest) doUpload(
 	}
 
 	// Check if temp file size exceeds max file size configuration
-	if bytesWritten > types.FileSizeBytes(*cfg.MaxFileSizeBytes) {
+	if *cfg.MaxFileSizeBytes > 0 && bytesWritten > types.FileSizeBytes(*cfg.MaxFileSizeBytes) {
 		fileutils.RemoveDir(tmpDir, r.Logger) // delete temp file
 		return requestEntityTooLargeJSONResponse(*cfg.MaxFileSizeBytes)
 	}

mediaapi/routing/upload_test.go

@@ -36,6 +36,7 @@ func Test_uploadRequest_doUpload(t *testing.T) {
 	}
 
 	maxSize := config.FileSizeBytes(8)
+	unlimitedSize := config.FileSizeBytes(0)
 	logger := log.New().WithField("mediaapi", "test")
 	testdataPath := filepath.Join(wd, "./testdata")
 
@@ -117,6 +118,27 @@ func Test_uploadRequest_doUpload(t *testing.T) {
 			},
 			want: requestEntityTooLargeJSONResponse(maxSize),
 		},
+		{
+			name: "upload ok with unlimited filesize",
+			args: args{
+				ctx:       context.Background(),
+				reqReader: strings.NewReader("test test test"),
+				cfg: &config.MediaAPI{
+					MaxFileSizeBytes:  &unlimitedSize,
+					BasePath:          config.Path(testdataPath),
+					AbsBasePath:       config.Path(testdataPath),
+					DynamicThumbnails: false,
+				},
+				db: db,
+			},
+			fields: fields{
+				Logger: logger,
+				MediaMetadata: &types.MediaMetadata{
+					MediaID:    "1339",
+					UploadName: "test fail",
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

roomserver/api/api.go

@@ -170,6 +170,9 @@ type RoomserverInternalAPI interface {
 	// PerformForget forgets a rooms history for a specific user
 	PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error
 
+	// PerformRoomUpgrade upgrades a room to a newer version
+	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse)
+
 	// Asks for the default room version as preferred by the server.
 	QueryRoomVersionCapabilities(
 		ctx context.Context,

roomserver/api/api_trace.go

@@ -67,6 +67,15 @@ func (t *RoomserverInternalAPITrace) PerformUnpeek(
 	util.GetLogger(ctx).Infof("PerformUnpeek req=%+v res=%+v", js(req), js(res))
 }
 
+func (t *RoomserverInternalAPITrace) PerformRoomUpgrade(
+	ctx context.Context,
+	req *PerformRoomUpgradeRequest,
+	res *PerformRoomUpgradeResponse,
+) {
+	t.Impl.PerformRoomUpgrade(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformRoomUpgrade req=%+v res=%+v", js(req), js(res))
+}
+
 func (t *RoomserverInternalAPITrace) PerformJoin(
 	ctx context.Context,
 	req *PerformJoinRequest,

roomserver/api/perform.go

@@ -203,3 +203,14 @@ type PerformForgetRequest struct {
 }
 
 type PerformForgetResponse struct{}
+
+type PerformRoomUpgradeRequest struct {
+	RoomID      string                        `json:"room_id"`
+	UserID      string                        `json:"user_id"`
+	RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
+}
+
+type PerformRoomUpgradeResponse struct {
+	NewRoomID string
+	Error     *PerformError
+}

roomserver/api/query.go

@@ -128,6 +128,8 @@ type QueryMembershipForUserResponse struct {
 type QueryMembershipsForRoomRequest struct {
 	// If true, only returns the membership events of "join" membership
 	JoinedOnly bool `json:"joined_only"`
+	// If true, only returns the membership events of local users
+	LocalOnly bool `json:"local_only"`
 	// ID of the room to fetch memberships from
 	RoomID string `json:"room_id"`
 	// Optional - ID of the user sending the request, for checking if the

roomserver/internal/api.go

@@ -34,6 +34,7 @@ type RoomserverInternalAPI struct {
 	*perform.Publisher
 	*perform.Backfiller
 	*perform.Forgetter
+	*perform.Upgrader
 	ProcessContext         *process.ProcessContext
 	DB                     storage.Database
 	Cfg                    *config.RoomServer
@@ -159,6 +160,10 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalA
 	r.Forgetter = &perform.Forgetter{
 		DB: r.DB,
 	}
+	r.Upgrader = &perform.Upgrader{
+		Cfg:    r.Cfg,
+		URSAPI: r,
+	}
 
 	if err := r.Inputer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start roomserver input API")

roomserver/internal/input/input_missing.go

@@ -613,12 +613,13 @@ func (t *missingStateReq) lookupMissingStateViaState(
 		return nil, err
 	}
 	// Check that the returned state is valid.
-	if err := state.Check(ctx, roomVersion, t.keys, nil); err != nil {
+	authEvents, stateEvents, err := state.Check(ctx, roomVersion, t.keys, nil)
+	if err != nil {
 		return nil, err
 	}
 	parsedState := &parsedRespState{
-		AuthEvents:  make([]*gomatrixserverlib.Event, len(state.AuthEvents)),
+		AuthEvents:  authEvents,
-		StateEvents: make([]*gomatrixserverlib.Event, len(state.StateEvents)),
+		StateEvents: stateEvents,
 	}
 	// Cache the results of this state lookup and deduplicate anything we already
 	// have in the cache, freeing up memory.

roomserver/internal/input/input_test.go

@@ -2,7 +2,6 @@ package input_test
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"testing"
 	"time"
@@ -12,30 +11,22 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
 )
 
-func psqlConnectionString() config.DataSource {
+var js nats.JetStreamContext
-	user := os.Getenv("POSTGRES_USER")
+var jc *nats.Conn
-	if user == "" {
+
-		user = "dendrite"
+func TestMain(m *testing.M) {
-	}
+	var pc *process.ProcessContext
-	dbName := os.Getenv("POSTGRES_DB")
+	pc, js, jc = jetstream.PrepareForTests()
-	if dbName == "" {
+	code := m.Run()
-		dbName = "dendrite"
+	pc.ShutdownDendrite()
-	}
+	pc.WaitForComponentsToFinish()
-	connStr := fmt.Sprintf(
+	os.Exit(code)
-		"user=%s dbname=%s sslmode=disable", user, dbName,
-	)
-	password := os.Getenv("POSTGRES_PASSWORD")
-	if password != "" {
-		connStr += fmt.Sprintf(" password=%s", password)
-	}
-	host := os.Getenv("POSTGRES_HOST")
-	if host != "" {
-		connStr += fmt.Sprintf(" host=%s", host)
-	}
-	return config.DataSource(connStr)
 }
 
 func TestSingleTransactionOnInput(t *testing.T) {
@@ -63,7 +54,7 @@ func TestSingleTransactionOnInput(t *testing.T) {
 	}
 	db, err := storage.Open(
 		&config.DatabaseOptions{
-			ConnectionString:   psqlConnectionString(),
+			ConnectionString:   "",
 			MaxOpenConnections: 1,
 			MaxIdleConnections: 1,
 		},
@@ -74,7 +65,9 @@ func TestSingleTransactionOnInput(t *testing.T) {
 		t.SkipNow()
 	}
 	inputter := &input.Inputer{
-		DB: db,
+		DB:         db,
+		JetStream:  js,
+		NATSClient: jc,
 	}
 	res := &api.InputRoomEventsResponse{}
 	inputter.InputRoomEvents(

roomserver/internal/perform/perform_upgrade.go

@@ -0,0 +1,709 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package perform
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+)
+
+type Upgrader struct {
+	Cfg    *config.RoomServer
+	URSAPI api.RoomserverInternalAPI
+}
+
+// fledglingEvent is a helper representation of an event used when creating many events in succession.
+type fledglingEvent struct {
+	Type     string      `json:"type"`
+	StateKey string      `json:"state_key"`
+	Content  interface{} `json:"content"`
+}
+
+// PerformRoomUpgrade upgrades a room from one version to another
+func (r *Upgrader) PerformRoomUpgrade(
+	ctx context.Context,
+	req *api.PerformRoomUpgradeRequest,
+	res *api.PerformRoomUpgradeResponse,
+) {
+	res.NewRoomID, res.Error = r.performRoomUpgrade(ctx, req)
+	if res.Error != nil {
+		res.NewRoomID = ""
+		logrus.WithContext(ctx).WithError(res.Error).Error("Room upgrade failed")
+	}
+}
+
+func (r *Upgrader) performRoomUpgrade(
+	ctx context.Context,
+	req *api.PerformRoomUpgradeRequest,
+) (string, *api.PerformError) {
+	roomID := req.RoomID
+	userID := req.UserID
+	evTime := time.Now()
+
+	// Return an immediate error if the room does not exist
+	if err := r.validateRoomExists(ctx, roomID); err != nil {
+		return "", &api.PerformError{
+			Code: api.PerformErrorNoRoom,
+			Msg:  "Error validating that the room exists",
+		}
+	}
+
+	// 1. Check if the user is authorized to actually perform the upgrade (can send m.room.tombstone)
+	if !r.userIsAuthorized(ctx, userID, roomID) {
+		return "", &api.PerformError{
+			Code: api.PerformErrorNotAllowed,
+			Msg:  "You don't have permission to upgrade the room, power level too low.",
+		}
+	}
+
+	// TODO (#267): Check room ID doesn't clash with an existing one, and we
+	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
+	newRoomID := fmt.Sprintf("!%s:%s", util.RandomString(16), r.Cfg.Matrix.ServerName)
+
+	// Get the existing room state for the old room.
+	oldRoomReq := &api.QueryLatestEventsAndStateRequest{
+		RoomID: roomID,
+	}
+	oldRoomRes := &api.QueryLatestEventsAndStateResponse{}
+	if err := r.URSAPI.QueryLatestEventsAndState(ctx, oldRoomReq, oldRoomRes); err != nil {
+		return "", &api.PerformError{
+			Msg: fmt.Sprintf("Failed to get latest state: %s", err),
+		}
+	}
+
+	// Make the tombstone event
+	tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, userID, roomID, newRoomID)
+	if pErr != nil {
+		return "", pErr
+	}
+
+	// Generate the initial events we need to send into the new room. This includes copied state events and bans
+	// as well as the power level events needed to set up the room
+	eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, userID, roomID, string(req.RoomVersion), tombstoneEvent)
+	if pErr != nil {
+		return "", pErr
+	}
+
+	// 5. Send the tombstone event to the old room (must do this before we set the new canonical_alias)
+	if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil {
+		return "", pErr
+	}
+
+	// Send the setup events to the new room
+	if pErr = r.sendInitialEvents(ctx, evTime, userID, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil {
+		return "", pErr
+	}
+
+	// If the old room was public, make sure the new one is too
+	if pErr = r.publishIfOldRoomWasPublic(ctx, roomID, newRoomID); pErr != nil {
+		return "", pErr
+	}
+
+	// If the old room had a canonical alias event, it should be deleted in the old room
+	if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, userID, roomID); pErr != nil {
+		return "", pErr
+	}
+
+	// 4. Move local aliases to the new room
+	if pErr = moveLocalAliases(ctx, roomID, newRoomID, userID, r.URSAPI); pErr != nil {
+		return "", pErr
+	}
+
+	// 6. Restrict power levels in the old room
+	if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, userID, roomID); pErr != nil {
+		return "", pErr
+	}
+
+	return newRoomID, nil
+}
+
+func (r *Upgrader) getRoomPowerLevels(ctx context.Context, roomID string) (*gomatrixserverlib.PowerLevelContent, *api.PerformError) {
+	oldPowerLevelsEvent := api.GetStateEvent(ctx, r.URSAPI, roomID, gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomPowerLevels,
+		StateKey:  "",
+	})
+	powerLevelContent, err := oldPowerLevelsEvent.PowerLevels()
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error()
+		return nil, &api.PerformError{
+			Msg: "powerLevel event was not actually a power level event",
+		}
+	}
+	return powerLevelContent, nil
+}
+
+func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.Time, userID, roomID string) *api.PerformError {
+	restrictedPowerLevelContent, pErr := r.getRoomPowerLevels(ctx, roomID)
+	if pErr != nil {
+		return pErr
+	}
+
+	// From: https://spec.matrix.org/v1.2/client-server-api/#server-behaviour-16
+	// If possible, the power levels in the old room should also be modified to
+	// prevent sending of events and inviting new users. For example, setting
+	// events_default and invite to the greater of 50 and users_default + 1.
+	restrictedDefaultPowerLevel := int64(50)
+	if restrictedPowerLevelContent.UsersDefault+1 > restrictedDefaultPowerLevel {
+		restrictedDefaultPowerLevel = restrictedPowerLevelContent.UsersDefault + 1
+	}
+	restrictedPowerLevelContent.EventsDefault = restrictedDefaultPowerLevel
+	restrictedPowerLevelContent.Invite = restrictedDefaultPowerLevel
+
+	restrictedPowerLevelsHeadered, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{
+		Type:     gomatrixserverlib.MRoomPowerLevels,
+		StateKey: "",
+		Content:  restrictedPowerLevelContent,
+	})
+	if resErr != nil {
+		if resErr.Code == api.PerformErrorNotAllowed {
+			util.GetLogger(ctx).WithField(logrus.ErrorKey, resErr).Warn("UpgradeRoom: Could not restrict power levels in old room")
+		} else {
+			return resErr
+		}
+	} else {
+		if resErr = r.sendHeaderedEvent(ctx, restrictedPowerLevelsHeadered); resErr != nil {
+			return resErr
+		}
+	}
+	return nil
+}
+
+func moveLocalAliases(ctx context.Context,
+	roomID, newRoomID, userID string,
+	URSAPI api.RoomserverInternalAPI) *api.PerformError {
+	var err error
+
+	aliasReq := api.GetAliasesForRoomIDRequest{RoomID: roomID}
+	aliasRes := api.GetAliasesForRoomIDResponse{}
+	if err = URSAPI.GetAliasesForRoomID(ctx, &aliasReq, &aliasRes); err != nil {
+		return &api.PerformError{
+			Msg: "Could not get aliases for old room",
+		}
+	}
+
+	for _, alias := range aliasRes.Aliases {
+		removeAliasReq := api.RemoveRoomAliasRequest{UserID: userID, Alias: alias}
+		removeAliasRes := api.RemoveRoomAliasResponse{}
+		if err = URSAPI.RemoveRoomAlias(ctx, &removeAliasReq, &removeAliasRes); err != nil {
+			return &api.PerformError{
+				Msg: "api.RemoveRoomAlias failed",
+			}
+		}
+
+		setAliasReq := api.SetRoomAliasRequest{UserID: userID, Alias: alias, RoomID: newRoomID}
+		setAliasRes := api.SetRoomAliasResponse{}
+		if err = URSAPI.SetRoomAlias(ctx, &setAliasReq, &setAliasRes); err != nil {
+			return &api.PerformError{
+				Msg: "api.SetRoomAlias failed",
+			}
+		}
+	}
+	return nil
+}
+
+func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, evTime time.Time, userID, roomID string) *api.PerformError {
+	for _, event := range oldRoom.StateEvents {
+		if event.Type() != gomatrixserverlib.MRoomCanonicalAlias || !event.StateKeyEquals("") {
+			continue
+		}
+		var aliasContent struct {
+			Alias      string   `json:"alias"`
+			AltAliases []string `json:"alt_aliases"`
+		}
+		if err := json.Unmarshal(event.Content(), &aliasContent); err != nil {
+			return &api.PerformError{
+				Msg: fmt.Sprintf("Failed to unmarshal canonical aliases: %s", err),
+			}
+		}
+		if aliasContent.Alias == "" && len(aliasContent.AltAliases) == 0 {
+			// There are no canonical aliases to clear, therefore do nothing.
+			return nil
+		}
+	}
+
+	emptyCanonicalAliasEvent, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{
+		Type:    gomatrixserverlib.MRoomCanonicalAlias,
+		Content: map[string]interface{}{},
+	})
+	if resErr != nil {
+		if resErr.Code == api.PerformErrorNotAllowed {
+			util.GetLogger(ctx).WithField(logrus.ErrorKey, resErr).Warn("UpgradeRoom: Could not set empty canonical alias event in old room")
+		} else {
+			return resErr
+		}
+	} else {
+		if resErr = r.sendHeaderedEvent(ctx, emptyCanonicalAliasEvent); resErr != nil {
+			return resErr
+		}
+	}
+	return nil
+}
+
+func (r *Upgrader) publishIfOldRoomWasPublic(ctx context.Context, roomID, newRoomID string) *api.PerformError {
+	// check if the old room was published
+	var pubQueryRes api.QueryPublishedRoomsResponse
+	err := r.URSAPI.QueryPublishedRooms(ctx, &api.QueryPublishedRoomsRequest{
+		RoomID: roomID,
+	}, &pubQueryRes)
+	if err != nil {
+		return &api.PerformError{
+			Msg: "QueryPublishedRooms failed",
+		}
+	}
+
+	// if the old room is published (was public), publish the new room
+	if len(pubQueryRes.RoomIDs) == 1 {
+		publishNewRoomAndUnpublishOldRoom(ctx, r.URSAPI, roomID, newRoomID)
+	}
+	return nil
+}
+
+func publishNewRoomAndUnpublishOldRoom(
+	ctx context.Context,
+	URSAPI api.RoomserverInternalAPI,
+	oldRoomID, newRoomID string,
+) {
+	// expose this room in the published room list
+	var pubNewRoomRes api.PerformPublishResponse
+	URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{
+		RoomID:     newRoomID,
+		Visibility: "public",
+	}, &pubNewRoomRes)
+	if pubNewRoomRes.Error != nil {
+		// treat as non-fatal since the room is already made by this point
+		util.GetLogger(ctx).WithError(pubNewRoomRes.Error).Error("failed to visibility:public")
+	}
+
+	var unpubOldRoomRes api.PerformPublishResponse
+	// remove the old room from the published room list
+	URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{
+		RoomID:     oldRoomID,
+		Visibility: "private",
+	}, &unpubOldRoomRes)
+	if unpubOldRoomRes.Error != nil {
+		// treat as non-fatal since the room is already made by this point
+		util.GetLogger(ctx).WithError(unpubOldRoomRes.Error).Error("failed to visibility:private")
+	}
+}
+
+func (r *Upgrader) validateRoomExists(ctx context.Context, roomID string) error {
+	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
+	verRes := api.QueryRoomVersionForRoomResponse{}
+	if err := r.URSAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
+		return &api.PerformError{
+			Code: api.PerformErrorNoRoom,
+			Msg:  "Room does not exist",
+		}
+	}
+	return nil
+}
+
+func (r *Upgrader) userIsAuthorized(ctx context.Context, userID, roomID string,
+) bool {
+	plEvent := api.GetStateEvent(ctx, r.URSAPI, roomID, gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomPowerLevels,
+		StateKey:  "",
+	})
+	if plEvent == nil {
+		return false
+	}
+	pl, err := plEvent.PowerLevels()
+	if err != nil {
+		return false
+	}
+	// Check for power level required to send tombstone event (marks the current room as obsolete),
+	// if not found, use the StateDefault power level
+	return pl.UserLevel(userID) >= pl.EventLevel("m.room.tombstone", true)
+}
+
+// nolint:gocyclo
+func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, userID, roomID, newVersion string, tombstoneEvent *gomatrixserverlib.HeaderedEvent) ([]fledglingEvent, *api.PerformError) {
+	state := make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent, len(oldRoom.StateEvents))
+	for _, event := range oldRoom.StateEvents {
+		if event.StateKey() == nil {
+			// This shouldn't ever happen, but better to be safe than sorry.
+			continue
+		}
+		if event.Type() == gomatrixserverlib.MRoomMember && !event.StateKeyEquals(userID) {
+			// With the exception of bans and invites which we do want to copy, we
+			// should ignore membership events that aren't our own, as event auth will
+			// prevent us from being able to create membership events on behalf of other
+			// users anyway unless they are invites or bans.
+			membership, err := event.Membership()
+			if err != nil {
+				continue
+			}
+			switch membership {
+			case gomatrixserverlib.Ban:
+			case gomatrixserverlib.Invite:
+			default:
+				continue
+			}
+		}
+		state[gomatrixserverlib.StateKeyTuple{EventType: event.Type(), StateKey: *event.StateKey()}] = event
+	}
+
+	// The following events are ones that we are going to override manually
+	// in the following section.
+	override := map[gomatrixserverlib.StateKeyTuple]struct{}{
+		{EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}:      {},
+		{EventType: gomatrixserverlib.MRoomMember, StateKey: userID}:  {},
+		{EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}: {},
+		{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}:   {},
+	}
+
+	// The overridden events are essential events that must be present in the
+	// old room state. Check that they are there.
+	for tuple := range override {
+		if _, ok := state[tuple]; !ok {
+			return nil, &api.PerformError{
+				Msg: fmt.Sprintf("Essential event of type %q state key %q is missing", tuple.EventType, tuple.StateKey),
+			}
+		}
+	}
+
+	oldCreateEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}]
+	oldMembershipEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomMember, StateKey: userID}]
+	oldPowerLevelsEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}]
+	oldJoinRulesEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}]
+
+	// Create the new room create event. Using a map here instead of CreateContent
+	// means that we preserve any other interesting fields that might be present
+	// in the create event (such as for the room types MSC).
+	newCreateContent := map[string]interface{}{}
+	_ = json.Unmarshal(oldCreateEvent.Content(), &newCreateContent)
+	newCreateContent["creator"] = userID
+	newCreateContent["room_version"] = newVersion
+	newCreateContent["predecessor"] = gomatrixserverlib.PreviousRoom{
+		EventID: tombstoneEvent.EventID(),
+		RoomID:  roomID,
+	}
+	newCreateEvent := fledglingEvent{
+		Type:     gomatrixserverlib.MRoomCreate,
+		StateKey: "",
+		Content:  newCreateContent,
+	}
+
+	// Now create the new membership event. Same rules apply as above, so
+	// that we preserve fields we don't otherwise know about. We'll always
+	// set the membership to join though, because that is necessary to auth
+	// the events after it.
+	newMembershipContent := map[string]interface{}{}
+	_ = json.Unmarshal(oldMembershipEvent.Content(), &newMembershipContent)
+	newMembershipContent["membership"] = gomatrixserverlib.Join
+	newMembershipEvent := fledglingEvent{
+		Type:     gomatrixserverlib.MRoomMember,
+		StateKey: userID,
+		Content:  newMembershipContent,
+	}
+
+	// We might need to temporarily give ourselves a higher power level
+	// than we had in the old room in order to be able to send all of
+	// the relevant state events. This function will return whether we
+	// had to override the power level events or not — if we did, we
+	// need to send the original power levels again later on.
+	powerLevelContent, err := oldPowerLevelsEvent.PowerLevels()
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error()
+		return nil, &api.PerformError{
+			Msg: "Power level event content was invalid",
+		}
+	}
+	tempPowerLevelsEvent, powerLevelsOverridden := createTemporaryPowerLevels(powerLevelContent, userID)
+
+	// Now do the join rules event, same as the create and membership
+	// events. We'll set a sane default of "invite" so that if the
+	// existing join rules contains garbage, the room can still be
+	// upgraded.
+	newJoinRulesContent := map[string]interface{}{
+		"join_rule": gomatrixserverlib.Invite, // sane default
+	}
+	_ = json.Unmarshal(oldJoinRulesEvent.Content(), &newJoinRulesContent)
+	newJoinRulesEvent := fledglingEvent{
+		Type:     gomatrixserverlib.MRoomJoinRules,
+		StateKey: "",
+		Content:  newJoinRulesContent,
+	}
+
+	eventsToMake := make([]fledglingEvent, 0, len(state))
+	eventsToMake = append(
+		eventsToMake, newCreateEvent, newMembershipEvent,
+		tempPowerLevelsEvent, newJoinRulesEvent,
+	)
+
+	// For some reason Sytest expects there to be a guest access event.
+	// Create one if it doesn't exist.
+	if _, ok := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomGuestAccess, StateKey: ""}]; !ok {
+		eventsToMake = append(eventsToMake, fledglingEvent{
+			Type: gomatrixserverlib.MRoomGuestAccess,
+			Content: map[string]string{
+				"guest_access": "forbidden",
+			},
+		})
+	}
+
+	// Duplicate all of the old state events into the new room.
+	for tuple, event := range state {
+		if _, ok := override[tuple]; ok {
+			// Don't duplicate events we have overridden already. They
+			// are already in `eventsToMake`.
+			continue
+		}
+		newEvent := fledglingEvent{
+			Type:     tuple.EventType,
+			StateKey: tuple.StateKey,
+		}
+		if err = json.Unmarshal(event.Content(), &newEvent.Content); err != nil {
+			logrus.WithError(err).Error("Failed to unmarshal old event")
+			continue
+		}
+		eventsToMake = append(eventsToMake, newEvent)
+	}
+
+	// If we sent a temporary power level event into the room before,
+	// override that now by restoring the original power levels.
+	if powerLevelsOverridden {
+		eventsToMake = append(eventsToMake, fledglingEvent{
+			Type:    gomatrixserverlib.MRoomPowerLevels,
+			Content: powerLevelContent,
+		})
+	}
+	return eventsToMake, nil
+}
+
+func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, userID, newRoomID, newVersion string, eventsToMake []fledglingEvent) *api.PerformError {
+	var err error
+	var builtEvents []*gomatrixserverlib.HeaderedEvent
+	authEvents := gomatrixserverlib.NewAuthEvents(nil)
+	for i, e := range eventsToMake {
+		depth := i + 1 // depth starts at 1
+
+		builder := gomatrixserverlib.EventBuilder{
+			Sender:   userID,
+			RoomID:   newRoomID,
+			Type:     e.Type,
+			StateKey: &e.StateKey,
+			Depth:    int64(depth),
+		}
+		err = builder.SetContent(e.Content)
+		if err != nil {
+			return &api.PerformError{
+				Msg: "builder.SetContent failed",
+			}
+		}
+		if i > 0 {
+			builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
+		}
+		var event *gomatrixserverlib.Event
+		event, err = r.buildEvent(&builder, &authEvents, evTime, gomatrixserverlib.RoomVersion(newVersion))
+		if err != nil {
+			return &api.PerformError{
+				Msg: "buildEvent failed",
+			}
+		}
+
+		if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil {
+			return &api.PerformError{
+				Msg: "gomatrixserverlib.Allowed failed",
+			}
+		}
+
+		// Add the event to the list of auth events
+		builtEvents = append(builtEvents, event.Headered(gomatrixserverlib.RoomVersion(newVersion)))
+		err = authEvents.AddEvent(event)
+		if err != nil {
+			return &api.PerformError{
+				Msg: "authEvents.AddEvent failed",
+			}
+		}
+	}
+
+	inputs := make([]api.InputRoomEvent, 0, len(builtEvents))
+	for _, event := range builtEvents {
+		inputs = append(inputs, api.InputRoomEvent{
+			Kind:         api.KindNew,
+			Event:        event,
+			Origin:       r.Cfg.Matrix.ServerName,
+			SendAsServer: api.DoNotSendToOtherServers,
+		})
+	}
+	if err = api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil {
+		return &api.PerformError{
+			Msg: "api.SendInputRoomEvents failed",
+		}
+	}
+	return nil
+}
+
+func (r *Upgrader) makeTombstoneEvent(
+	ctx context.Context,
+	evTime time.Time,
+	userID, roomID, newRoomID string,
+) (*gomatrixserverlib.HeaderedEvent, *api.PerformError) {
+	content := map[string]interface{}{
+		"body":             "This room has been replaced",
+		"replacement_room": newRoomID,
+	}
+	event := fledglingEvent{
+		Type:    "m.room.tombstone",
+		Content: content,
+	}
+	return r.makeHeaderedEvent(ctx, evTime, userID, roomID, event)
+}
+
+func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, userID, roomID string, event fledglingEvent) (*gomatrixserverlib.HeaderedEvent, *api.PerformError) {
+	builder := gomatrixserverlib.EventBuilder{
+		Sender:   userID,
+		RoomID:   roomID,
+		Type:     event.Type,
+		StateKey: &event.StateKey,
+	}
+	err := builder.SetContent(event.Content)
+	if err != nil {
+		return nil, &api.PerformError{
+			Msg: "builder.SetContent failed",
+		}
+	}
+	var queryRes api.QueryLatestEventsAndStateResponse
+	headeredEvent, err := eventutil.QueryAndBuildEvent(ctx, &builder, r.Cfg.Matrix, evTime, r.URSAPI, &queryRes)
+	if err == eventutil.ErrRoomNoExists {
+		return nil, &api.PerformError{
+			Code: api.PerformErrorNoRoom,
+			Msg:  "Room does not exist",
+		}
+	} else if e, ok := err.(gomatrixserverlib.BadJSONError); ok {
+		return nil, &api.PerformError{
+			Msg: e.Error(),
+		}
+	} else if e, ok := err.(gomatrixserverlib.EventValidationError); ok {
+		if e.Code == gomatrixserverlib.EventValidationTooLarge {
+			return nil, &api.PerformError{
+				Msg: e.Error(),
+			}
+		}
+		return nil, &api.PerformError{
+			Msg: e.Error(),
+		}
+	} else if err != nil {
+		return nil, &api.PerformError{
+			Msg: "eventutil.BuildEvent failed",
+		}
+	}
+	// check to see if this user can perform this operation
+	stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents))
+	for i := range queryRes.StateEvents {
+		stateEvents[i] = queryRes.StateEvents[i].Event
+	}
+	provider := gomatrixserverlib.NewAuthEvents(stateEvents)
+	if err = gomatrixserverlib.Allowed(headeredEvent.Event, &provider); err != nil {
+		return nil, &api.PerformError{
+			Code: api.PerformErrorNotAllowed,
+			Msg:  err.Error(), // TODO: Is this error string comprehensible to the client?
+		}
+	}
+
+	return headeredEvent, nil
+}
+
+func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelContent, userID string) (fledglingEvent, bool) {
+	// Work out what power level we need in order to be able to send events
+	// of all types into the room.
+	neededPowerLevel := powerLevelContent.StateDefault
+	for _, powerLevel := range powerLevelContent.Events {
+		if powerLevel > neededPowerLevel {
+			neededPowerLevel = powerLevel
+		}
+	}
+
+	// Make a copy of the existing power level content.
+	tempPowerLevelContent := *powerLevelContent
+	powerLevelsOverridden := false
+
+	// At this point, the "Users", "Events" and "Notifications" keys are all
+	// pointing to the map of the original PL content, so we will specifically
+	// override the users map with a new one and duplicate the values deeply,
+	// so that we can modify them without modifying the original.
+	tempPowerLevelContent.Users = make(map[string]int64, len(powerLevelContent.Users))
+	for key, value := range powerLevelContent.Users {
+		tempPowerLevelContent.Users[key] = value
+	}
+
+	// If the user who is upgrading the room doesn't already have sufficient
+	// power, then elevate their power levels.
+	if tempPowerLevelContent.UserLevel(userID) < neededPowerLevel {
+		tempPowerLevelContent.Users[userID] = neededPowerLevel
+		powerLevelsOverridden = true
+	}
+
+	// Then return the temporary power levels event.
+	return fledglingEvent{
+		Type:    gomatrixserverlib.MRoomPowerLevels,
+		Content: tempPowerLevelContent,
+	}, powerLevelsOverridden
+}
+
+func (r *Upgrader) sendHeaderedEvent(
+	ctx context.Context,
+	headeredEvent *gomatrixserverlib.HeaderedEvent,
+) *api.PerformError {
+	var inputs []api.InputRoomEvent
+	inputs = append(inputs, api.InputRoomEvent{
+		Kind:         api.KindNew,
+		Event:        headeredEvent,
+		Origin:       r.Cfg.Matrix.ServerName,
+		SendAsServer: api.DoNotSendToOtherServers,
+	})
+	if err := api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil {
+		return &api.PerformError{
+			Msg: "api.SendInputRoomEvents failed",
+		}
+	}
+
+	return nil
+}
+
+func (r *Upgrader) buildEvent(
+	builder *gomatrixserverlib.EventBuilder,
+	provider gomatrixserverlib.AuthEventProvider,
+	evTime time.Time,
+	roomVersion gomatrixserverlib.RoomVersion,
+) (*gomatrixserverlib.Event, error) {
+	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
+	if err != nil {
+		return nil, err
+	}
+	refs, err := eventsNeeded.AuthEventReferences(provider)
+	if err != nil {
+		return nil, err
+	}
+	builder.AuthEvents = refs
+	event, err := builder.Build(
+		evTime, r.Cfg.Matrix.ServerName, r.Cfg.Matrix.KeyID,
+		r.Cfg.Matrix.PrivateKey, roomVersion,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err)
+	}
+	return event, nil
+}

roomserver/internal/query/query.go

@@ -220,7 +220,7 @@ func (r *Queryer) QueryMembershipsForRoom(
 	if request.Sender == "" {
 		var events []types.Event
 		var eventNIDs []types.EventNID
-		eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, request.JoinedOnly, false)
+		eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, request.JoinedOnly, request.LocalOnly)
 		if err != nil {
 			return fmt.Errorf("r.DB.GetMembershipEventNIDsForRoom: %w", err)
 		}

roomserver/inthttp/client.go

@@ -32,6 +32,7 @@ const (
 	RoomserverPerformInvitePath      = "/roomserver/performInvite"
 	RoomserverPerformPeekPath        = "/roomserver/performPeek"
 	RoomserverPerformUnpeekPath      = "/roomserver/performUnpeek"
+	RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade"
 	RoomserverPerformJoinPath        = "/roomserver/performJoin"
 	RoomserverPerformLeavePath       = "/roomserver/performLeave"
 	RoomserverPerformBackfillPath    = "/roomserver/performBackfill"
@@ -252,6 +253,23 @@ func (h *httpRoomserverInternalAPI) PerformUnpeek(
 	}
 }
 
+func (h *httpRoomserverInternalAPI) PerformRoomUpgrade(
+	ctx context.Context,
+	request *api.PerformRoomUpgradeRequest,
+	response *api.PerformRoomUpgradeResponse,
+) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformRoomUpgrade")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverPerformRoomUpgradePath
+	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	if err != nil {
+		response.Error = &api.PerformError{
+			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
+		}
+	}
+}
+
 func (h *httpRoomserverInternalAPI) PerformLeave(
 	ctx context.Context,
 	request *api.PerformLeaveRequest,

roomserver/inthttp/server.go

@@ -96,6 +96,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverPerformRoomUpgradePath,
+		httputil.MakeInternalAPI("performRoomUpgrade", func(req *http.Request) util.JSONResponse {
+			var request api.PerformRoomUpgradeRequest
+			var response api.PerformRoomUpgradeResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			r.PerformRoomUpgrade(req.Context(), &request, &response)
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	internalAPIMux.Handle(RoomserverPerformPublishPath,
 		httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse {
 			var request api.PerformPublishRequest

setup/base/base.go

@@ -45,8 +45,6 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	asinthttp "github.com/matrix-org/dendrite/appservice/inthttp"
-	eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
-	eduinthttp "github.com/matrix-org/dendrite/eduserver/inthttp"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	federationIntHTTP "github.com/matrix-org/dendrite/federationapi/inthttp"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
@@ -247,15 +245,6 @@ func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI {
 	return userAPI
 }
 
-// EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP
-func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI {
-	e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.apiHttpClient)
-	if err != nil {
-		logrus.WithError(err).Panic("EDUServerClient failed", b.apiHttpClient)
-	}
-	return e
-}
-
 // FederationAPIHTTPClient returns FederationInternalAPI for hitting
 // the federation API server over HTTP
 func (b *BaseDendrite) FederationAPIHTTPClient() federationAPI.FederationInternalAPI {
@@ -289,6 +278,7 @@ func (b *BaseDendrite) CreateAccountsDB() userdb.Database {
 		b.Cfg.UserAPI.BCryptCost,
 		b.Cfg.UserAPI.OpenIDTokenLifetimeMS,
 		userapi.DefaultLoginTokenLifetime,
+		b.Cfg.Global.ServerNotices.LocalPart,
 	)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to accounts db")

setup/config/config.go

@@ -56,7 +56,6 @@ type Dendrite struct {
 	Global        Global        `yaml:"global"`
 	AppServiceAPI AppServiceAPI `yaml:"app_service_api"`
 	ClientAPI     ClientAPI     `yaml:"client_api"`
-	EDUServer     EDUServer     `yaml:"edu_server"`
 	FederationAPI FederationAPI `yaml:"federation_api"`
 	KeyServer     KeyServer     `yaml:"key_server"`
 	MediaAPI      MediaAPI      `yaml:"media_api"`
@@ -296,7 +295,6 @@ func (c *Dendrite) Defaults(generate bool) {
 
 	c.Global.Defaults(generate)
 	c.ClientAPI.Defaults(generate)
-	c.EDUServer.Defaults(generate)
 	c.FederationAPI.Defaults(generate)
 	c.KeyServer.Defaults(generate)
 	c.MediaAPI.Defaults(generate)
@@ -314,8 +312,7 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) {
 		Verify(configErrs *ConfigErrors, isMonolith bool)
 	}
 	for _, c := range []verifiable{
-		&c.Global, &c.ClientAPI,
+		&c.Global, &c.ClientAPI, &c.FederationAPI,
-		&c.EDUServer, &c.FederationAPI,
 		&c.KeyServer, &c.MediaAPI, &c.RoomServer,
 		&c.SyncAPI, &c.UserAPI,
 		&c.AppServiceAPI, &c.MSCs,
@@ -327,7 +324,6 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) {
 func (c *Dendrite) Wiring() {
 	c.Global.JetStream.Matrix = &c.Global
 	c.ClientAPI.Matrix = &c.Global
-	c.EDUServer.Matrix = &c.Global
 	c.FederationAPI.Matrix = &c.Global
 	c.KeyServer.Matrix = &c.Global
 	c.MediaAPI.Matrix = &c.Global
@@ -519,15 +515,6 @@ func (config *Dendrite) UserAPIURL() string {
 	return string(config.UserAPI.InternalAPI.Connect)
 }
 
-// EDUServerURL returns an HTTP URL for where the EDU server is listening.
-func (config *Dendrite) EDUServerURL() string {
-	// Hard code the EDU server to talk HTTP for now.
-	// If we support HTTPS we need to think of a practical way to do certificate validation.
-	// People setting up servers shouldn't need to get a certificate valid for the public
-	// internet for an internal API.
-	return string(config.EDUServer.InternalAPI.Connect)
-}
-
 // KeyServerURL returns an HTTP URL for where the key server is listening.
 func (config *Dendrite) KeyServerURL() string {
 	// Hard code the key server to talk HTTP for now.

setup/config/config_eduserver.go

@@ -1,17 +0,0 @@
-package config
-
-type EDUServer struct {
-	Matrix *Global `yaml:"-"`
-
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
-}
-
-func (c *EDUServer) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7778"
-	c.InternalAPI.Connect = "http://localhost:7778"
-}
-
-func (c *EDUServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "edu_server.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "edu_server.internal_api.connect", string(c.InternalAPI.Connect))
-}

setup/config/config_federationapi.go

@@ -12,13 +12,6 @@ type FederationAPI struct {
 	// send transactions to remote servers.
 	Database DatabaseOptions `yaml:"database"`
 
-	// List of paths to X509 certificates used by the external federation listeners.
-	// These are used to calculate the TLS fingerprints to publish for this server.
-	// Other matrix servers talking to this server will expect the x509 certificate
-	// to match one of these certificates.
-	// The certificates should be in PEM format.
-	FederationCertificatePaths []Path `yaml:"federation_certificates"`
-
 	// Federation failure threshold. How many consecutive failures that we should
 	// tolerate when sending federation requests to a specific server. The backoff
 	// is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc.
@@ -57,8 +50,6 @@ func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 		checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
 	}
 	checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
-	// TODO: not applicable always, e.g. in demos
-	//checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths)))
 }
 
 // The config for setting a proxy to use for server->server requests