Merge branch 'master' into add-nats-support

build/gobind-yggdrasil/monolith.go

@@ -45,10 +45,6 @@ func (m *DendriteMonolith) PeerCount() int {
 	return m.YggdrasilNode.PeerCount()
 }
 
-func (m *DendriteMonolith) SessionCount() int {
-	return m.YggdrasilNode.SessionCount()
-}
-
 func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
 	m.YggdrasilNode.SetMulticastEnabled(enabled)
 }
@@ -78,7 +74,7 @@ func (m *DendriteMonolith) Start() {
 		panic(err)
 	}
 
-	ygg, err := yggconn.Setup("dendrite", m.StorageDirectory)
+	ygg, err := yggconn.Setup("dendrite", m.StorageDirectory, "")
 	if err != nil {
 		panic(err)
 	}
@@ -87,7 +83,7 @@ func (m *DendriteMonolith) Start() {
 	cfg := &config.Dendrite{}
 	cfg.Defaults()
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Global.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("file:%s/", m.StorageDirectory))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-account.db", m.StorageDirectory))
@@ -133,18 +129,6 @@ func (m *DendriteMonolith) Start() {
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
 
-	ygg.SetSessionFunc(func(address string) {
-		req := &api.PerformServersAliveRequest{
-			Servers: []gomatrixserverlib.ServerName{
-				gomatrixserverlib.ServerName(address),
-			},
-		}
-		res := &api.PerformServersAliveResponse{}
-		if err := fsAPI.PerformServersAlive(context.TODO(), req, res); err != nil {
-			logrus.WithError(err).Error("Failed to send wake-up message to newly connected node")
-		}
-	})
-
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
 	rsAPI.SetFederationSenderAPI(fsAPI)

clientapi/auth/user_interactive.go

@@ -78,7 +78,7 @@ func (r *Login) Username() string {
 	if r.Identifier.Type == "m.id.user" {
 		return r.Identifier.User
 	}
-	// deprecated but without it Riot iOS won't log in
+	// deprecated but without it Element iOS won't log in
 	return r.User
 }
 

clientapi/routing/membership.go

@@ -47,6 +47,37 @@ func SendBan(
 	if reqErr != nil {
 		return *reqErr
 	}
+
+	errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+
+	plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomPowerLevels,
+		StateKey:  "",
+	})
+	if plEvent == nil {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Forbidden("You don't have permission to ban this user, no power_levels event in this room."),
+		}
+	}
+	pl, err := plEvent.PowerLevels()
+	if err != nil {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Forbidden("You don't have permission to ban this user, the power_levels event for this room is malformed so auth checks cannot be performed."),
+		}
+	}
+	allowedToBan := pl.UserLevel(device.UserID) >= pl.Ban
+	if !allowedToBan {
+		return util.JSONResponse{
+			Code: 403,
+			JSON: jsonerror.Forbidden("You don't have permission to ban this user, power level too low."),
+		}
+	}
+
 	return sendMembership(req.Context(), accountDB, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
 }
 

cmd/dendrite-demo-pinecone/embed/embed_elementweb.go

@@ -1,4 +1,4 @@
-// +build riotweb
+// +build elementweb
 
 package embed
 
@@ -12,8 +12,8 @@ import (
 	"github.com/tidwall/sjson"
 )
 
-// From within the Riot Web directory:
+// From within the Element Web directory:
-// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_riotweb.go -private -pkg embed .
+// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_elementweb.go -private -pkg embed .
 
 var cssFile = regexp.MustCompile("\\.css$")
 var jsFile = regexp.MustCompile("\\.js$")
@@ -68,7 +68,7 @@ func Embed(rootMux *mux.Router, listenPort int, serverName string) {
 		}
 		js, _ := sjson.SetBytes(buf, "default_server_config.m\\.homeserver.base_url", url)
 		js, _ = sjson.SetBytes(js, "default_server_config.m\\.homeserver.server_name", serverName)
-		js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Riot %s", serverName))
+		js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Element %s", serverName))
 		js, _ = sjson.SetBytes(js, "disable_guests", true)
 		js, _ = sjson.SetBytes(js, "disable_3pid_login", true)
 		js, _ = sjson.DeleteBytes(js, "welcomeUserId")
@@ -76,7 +76,7 @@ func Embed(rootMux *mux.Router, listenPort int, serverName string) {
 	})
 
 	fmt.Println("*-------------------------------*")
-	fmt.Println("| This build includes Riot Web! |")
+	fmt.Println("| This build includes Element Web! |")
 	fmt.Println("*-------------------------------*")
 	fmt.Println("Point your browser to:", url)
 	fmt.Println()

cmd/dendrite-demo-pinecone/embed/embed_other.go

@@ -1,4 +1,4 @@
-// +build !riotweb
+// +build !elementweb
 
 package embed
 

cmd/dendrite-demo-yggdrasil/convert/25519.go

@@ -1,53 +0,0 @@
-// Copyright 2019 Google LLC
-//
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file or at
-// https://developers.google.com/open-source/licenses/bsd
-//
-// Original code from https://github.com/FiloSottile/age/blob/bbab440e198a4d67ba78591176c7853e62d29e04/internal/age/ssh.go
-
-package convert
-
-import (
-	"crypto/ed25519"
-	"crypto/sha512"
-	"math/big"
-
-	"golang.org/x/crypto/curve25519"
-)
-
-var curve25519P, _ = new(big.Int).SetString("57896044618658097711785492504343953926634992332820282019728792003956564819949", 10)
-
-func Ed25519PrivateKeyToCurve25519(pk ed25519.PrivateKey) []byte {
-	h := sha512.New()
-	_, _ = h.Write(pk.Seed())
-	out := h.Sum(nil)
-	return out[:curve25519.ScalarSize]
-}
-
-func Ed25519PublicKeyToCurve25519(pk ed25519.PublicKey) []byte {
-	// ed25519.PublicKey is a little endian representation of the y-coordinate,
-	// with the most significant bit set based on the sign of the x-coordinate.
-	bigEndianY := make([]byte, ed25519.PublicKeySize)
-	for i, b := range pk {
-		bigEndianY[ed25519.PublicKeySize-i-1] = b
-	}
-	bigEndianY[0] &= 0b0111_1111
-
-	// The Montgomery u-coordinate is derived through the bilinear map
-	//     u = (1 + y) / (1 - y)
-	// See https://blog.filippo.io/using-ed25519-keys-for-encryption.
-	y := new(big.Int).SetBytes(bigEndianY)
-	denom := big.NewInt(1)
-	denom.ModInverse(denom.Sub(denom, y), curve25519P) // 1 / (1 - y)
-	u := y.Mul(y.Add(y, big.NewInt(1)), denom)
-	u.Mod(u, curve25519P)
-
-	out := make([]byte, curve25519.PointSize)
-	uBytes := u.Bytes()
-	for i, b := range uBytes {
-		out[len(uBytes)-i-1] = b
-	}
-
-	return out
-}

cmd/dendrite-demo-yggdrasil/convert/25519_test.go

@@ -1,51 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package convert
-
-import (
-	"bytes"
-	"crypto/ed25519"
-	"encoding/hex"
-	"testing"
-
-	"golang.org/x/crypto/curve25519"
-)
-
-func TestKeyConversion(t *testing.T) {
-	edPub, edPriv, err := ed25519.GenerateKey(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("Signing public:", hex.EncodeToString(edPub))
-	t.Log("Signing private:", hex.EncodeToString(edPriv))
-
-	cuPriv := Ed25519PrivateKeyToCurve25519(edPriv)
-	t.Log("Encryption private:", hex.EncodeToString(cuPriv))
-
-	cuPub := Ed25519PublicKeyToCurve25519(edPub)
-	t.Log("Converted encryption public:", hex.EncodeToString(cuPub))
-
-	var realPub, realPriv [32]byte
-	copy(realPriv[:32], cuPriv[:32])
-	curve25519.ScalarBaseMult(&realPub, &realPriv)
-	t.Log("Scalar-multed encryption public:", hex.EncodeToString(realPub[:]))
-
-	if !bytes.Equal(realPriv[:], cuPriv[:]) {
-		t.Fatal("Private keys should be equal (this means the test is broken)")
-	}
-	if !bytes.Equal(realPub[:], cuPub[:]) {
-		t.Fatal("Public keys should be equal")
-	}
-}

cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go

@@ -1,4 +1,4 @@
-// +build riotweb
+// +build elementweb
 
 package embed
 
@@ -12,8 +12,8 @@ import (
 	"github.com/tidwall/sjson"
 )
 
-// From within the Riot Web directory:
+// From within the Element Web directory:
-// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_riotweb.go -private -pkg embed .
+// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_elementweb.go -private -pkg embed .
 
 var cssFile = regexp.MustCompile("\\.css$")
 var jsFile = regexp.MustCompile("\\.js$")
@@ -68,16 +68,16 @@ func Embed(rootMux *mux.Router, listenPort int, serverName string) {
 		}
 		js, _ := sjson.SetBytes(buf, "default_server_config.m\\.homeserver.base_url", url)
 		js, _ = sjson.SetBytes(js, "default_server_config.m\\.homeserver.server_name", serverName)
-		js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Riot %s", serverName))
+		js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Element %s", serverName))
 		js, _ = sjson.SetBytes(js, "disable_guests", true)
 		js, _ = sjson.SetBytes(js, "disable_3pid_login", true)
 		js, _ = sjson.DeleteBytes(js, "welcomeUserId")
 		_, _ = w.Write(js)
 	})
 
-	fmt.Println("*-------------------------------*")
+	fmt.Println("*----------------------------------*")
-	fmt.Println("| This build includes Riot Web! |")
+	fmt.Println("| This build includes Element Web! |")
-	fmt.Println("*-------------------------------*")
+	fmt.Println("*----------------------------------*")
 	fmt.Println("Point your browser to:", url)
 	fmt.Println()
 }

cmd/dendrite-demo-yggdrasil/embed/embed_other.go

@@ -1,4 +1,4 @@
-// +build !riotweb
+// +build !elementweb
 
 package embed
 

cmd/dendrite-demo-yggdrasil/main.go

@@ -56,21 +56,23 @@ func main() {
 	flag.Parse()
 	internal.SetupPprof()
 
-	ygg, err := yggconn.Setup(*instanceName, ".")
+	ygg, err := yggconn.Setup(*instanceName, ".", *instancePeer)
 	if err != nil {
 		panic(err)
 	}
+	/*
 		ygg.SetMulticastEnabled(true)
 		if instancePeer != nil && *instancePeer != "" {
 			if err = ygg.SetStaticPeer(*instancePeer); err != nil {
 				logrus.WithError(err).Error("Failed to set static peer")
 			}
 		}
+	*/
 
 	cfg := &config.Dendrite{}
 	cfg.Defaults()
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Global.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
@@ -116,18 +118,6 @@ func main() {
 		base, federation, rsAPI, keyRing, true,
 	)
 
-	ygg.SetSessionFunc(func(address string) {
-		req := &api.PerformServersAliveRequest{
-			Servers: []gomatrixserverlib.ServerName{
-				gomatrixserverlib.ServerName(address),
-			},
-		}
-		res := &api.PerformServersAliveResponse{}
-		if err := fsAPI.PerformServersAlive(context.TODO(), req, res); err != nil {
-			logrus.WithError(err).Error("Failed to send wake-up message to newly connected node")
-		}
-	})
-
 	rsComponent.SetFederationSenderAPI(fsAPI)
 
 	monolith := setup.Monolith{

cmd/dendrite-demo-yggdrasil/yggconn/client.go

@@ -51,7 +51,6 @@ func (n *Node) CreateFederationClient(
 				ResponseHeaderTimeout: 10 * time.Second,
 				IdleConnTimeout:       30 * time.Second,
 				DialContext:           n.DialerContext,
-				TLSClientConfig:       n.tlsConfig,
 			},
 		},
 	)

cmd/dendrite-demo-yggdrasil/yggconn/node.go

@@ -17,7 +17,6 @@ package yggconn
 import (
 	"context"
 	"crypto/ed25519"
-	"crypto/tls"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -26,60 +25,48 @@ import (
 	"net"
 	"os"
 	"strings"
-	"sync"
-	"time"
 
 	"github.com/lucas-clemente/quic-go"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
 	"github.com/matrix-org/gomatrixserverlib"
-	"go.uber.org/atomic"
+	"github.com/neilalexander/utp"
 
+	ironwoodtypes "github.com/Arceliar/ironwood/types"
 	yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config"
+	yggdrasilcore "github.com/yggdrasil-network/yggdrasil-go/src/core"
+	yggdrasildefaults "github.com/yggdrasil-network/yggdrasil-go/src/defaults"
 	yggdrasilmulticast "github.com/yggdrasil-network/yggdrasil-go/src/multicast"
-	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
 
 	gologme "github.com/gologme/log"
 )
 
 type Node struct {
-	core         *yggdrasil.Core
+	core      *yggdrasilcore.Core
 	config    *yggdrasilconfig.NodeConfig
-	state        *yggdrasilconfig.NodeState
 	multicast *yggdrasilmulticast.Multicast
 	log       *gologme.Logger
 	listener  quic.Listener
-	tlsConfig    *tls.Config
+	utpSocket *utp.Socket
-	quicConfig   *quic.Config
+	incoming  chan net.Conn
-	sessions     sync.Map // string -> *session
-	sessionCount atomic.Uint32
-	sessionFunc  func(address string)
-	coords       sync.Map // string -> yggdrasil.Coords
-	incoming     chan QUICStream
-	NewSession   func(remote gomatrixserverlib.ServerName)
 }
 
-func (n *Node) Dialer(_, address string) (net.Conn, error) {
+func (n *Node) DialerContext(ctx context.Context, _, address string) (net.Conn, error) {
 	tokens := strings.Split(address, ":")
 	raw, err := hex.DecodeString(tokens[0])
 	if err != nil {
 		return nil, fmt.Errorf("hex.DecodeString: %w", err)
 	}
-	converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw))
+	pk := make(ironwoodtypes.Addr, ed25519.PublicKeySize)
-	convhex := hex.EncodeToString(converted)
+	copy(pk, raw[:])
-	return n.Dial("curve25519", convhex)
+	return n.utpSocket.DialAddrContext(ctx, pk)
 }
 
-func (n *Node) DialerContext(ctx context.Context, network, address string) (net.Conn, error) {
+func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) {
-	return n.Dialer(network, address)
-}
-
-func Setup(instanceName, storageDirectory string) (*Node, error) {
 	n := &Node{
-		core:      &yggdrasil.Core{},
+		core:      &yggdrasilcore.Core{},
-		config:    yggdrasilconfig.GenerateConfig(),
+		config:    yggdrasildefaults.GenerateConfig(),
 		multicast: &yggdrasilmulticast.Multicast{},
 		log:       gologme.New(os.Stdout, "YGG ", log.Flags()),
-		incoming:  make(chan QUICStream),
+		incoming:  make(chan net.Conn),
 	}
 
 	yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
@@ -93,24 +80,11 @@ func Setup(instanceName, storageDirectory string) (*Node, error) {
 		}
 	}
 
-	n.core.SetCoordChangeCallback(func(old, new yggdrasil.Coords) {
-		fmt.Println("COORDINATE CHANGE!")
-		fmt.Println("Old:", old)
-		fmt.Println("New:", new)
-		n.sessions.Range(func(k, v interface{}) bool {
-			if s, ok := v.(*session); ok {
-				fmt.Println("Killing session", k)
-				s.kill()
-			}
-			return true
-		})
-	})
-
 	n.config.Peers = []string{}
+	if peerURI != "" {
+		n.config.Peers = append(n.config.Peers, peerURI)
+	}
 	n.config.AdminListen = "none"
-	n.config.MulticastInterfaces = []string{}
-	n.config.EncryptionPrivateKey = hex.EncodeToString(n.EncryptionPrivateKey())
-	n.config.EncryptionPublicKey = hex.EncodeToString(n.EncryptionPublicKey())
 
 	j, err := json.MarshalIndent(n.config, "", "  ")
 	if err != nil {
@@ -123,34 +97,22 @@ func Setup(instanceName, storageDirectory string) (*Node, error) {
 	n.log.EnableLevel("error")
 	n.log.EnableLevel("warn")
 	n.log.EnableLevel("info")
-	n.state, err = n.core.Start(n.config, n.log)
+	if err = n.core.Start(n.config, n.log); err != nil {
+		panic(err)
+	}
+	n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core)
 	if err != nil {
 		panic(err)
 	}
-	if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil {
+	if err = n.multicast.Init(n.core, n.config, n.log, nil); err != nil {
 		panic(err)
 	}
 	if err = n.multicast.Start(); err != nil {
 		panic(err)
 	}
 
-	n.tlsConfig = n.generateTLSConfig()
+	n.log.Println("Public key:", n.core.PublicKey())
-	n.quicConfig = &quic.Config{
+	go n.listenFromYgg()
-		MaxIncomingStreams:    0,
-		MaxIncomingUniStreams: 0,
-		KeepAlive:             true,
-		MaxIdleTimeout:        time.Minute * 30,
-		HandshakeTimeout:      time.Second * 15,
-	}
-	copy(n.quicConfig.StatelessResetKey, n.EncryptionPublicKey())
-
-	n.log.Println("Public curve25519:", n.core.EncryptionPublicKey())
-	n.log.Println("Public ed25519:", n.core.SigningPublicKey())
-
-	go func() {
-		time.Sleep(time.Second)
-		n.listenFromYgg()
-	}()
 
 	return n, nil
 }
@@ -163,64 +125,33 @@ func (n *Node) Stop() {
 }
 
 func (n *Node) DerivedServerName() string {
-	return hex.EncodeToString(n.SigningPublicKey())
+	return hex.EncodeToString(n.PublicKey())
 }
 
-func (n *Node) DerivedSessionName() string {
+func (n *Node) PrivateKey() ed25519.PrivateKey {
-	return hex.EncodeToString(n.EncryptionPublicKey())
+	sk := make(ed25519.PrivateKey, ed25519.PrivateKeySize)
+	sb, err := hex.DecodeString(n.config.PrivateKey)
+	if err == nil {
+		copy(sk, sb[:])
+	} else {
+		panic(err)
+	}
+	return sk
 }
 
-func (n *Node) EncryptionPublicKey() []byte {
+func (n *Node) PublicKey() ed25519.PublicKey {
-	edkey := n.SigningPublicKey()
+	return n.core.PublicKey()
-	return convert.Ed25519PublicKeyToCurve25519(edkey)
-}
-
-func (n *Node) EncryptionPrivateKey() []byte {
-	edkey := n.SigningPrivateKey()
-	return convert.Ed25519PrivateKeyToCurve25519(edkey)
-}
-
-func (n *Node) SigningPublicKey() ed25519.PublicKey {
-	pubBytes, _ := hex.DecodeString(n.config.SigningPublicKey)
-	return ed25519.PublicKey(pubBytes)
-}
-
-func (n *Node) SigningPrivateKey() ed25519.PrivateKey {
-	privBytes, _ := hex.DecodeString(n.config.SigningPrivateKey)
-	return ed25519.PrivateKey(privBytes)
-}
-
-func (n *Node) SetSessionFunc(f func(address string)) {
-	n.sessionFunc = f
 }
 
 func (n *Node) PeerCount() int {
-	return len(n.core.GetPeers()) - 1
+	return len(n.core.GetPeers())
-}
-
-func (n *Node) SessionCount() int {
-	return int(n.sessionCount.Load())
 }
 
 func (n *Node) KnownNodes() []gomatrixserverlib.ServerName {
-	nodemap := map[string]struct{}{
+	nodemap := map[string]struct{}{}
-		//"b5ae50589e50991dd9dd7d59c5c5f7a4521e8da5b603b7f57076272abc58b374": {},
+	for _, peer := range n.core.GetPeers() {
+		nodemap[hex.EncodeToString(peer.Key)] = struct{}{}
 	}
-	for _, peer := range n.core.GetSwitchPeers() {
-		nodemap[hex.EncodeToString(peer.SigPublicKey[:])] = struct{}{}
-	}
-	n.sessions.Range(func(_, v interface{}) bool {
-		session, ok := v.(quic.Session)
-		if !ok {
-			return true
-		}
-		if len(session.ConnectionState().PeerCertificates) != 1 {
-			return true
-		}
-		subjectName := session.ConnectionState().PeerCertificates[0].Subject.CommonName
-		nodemap[subjectName] = struct{}{}
-		return true
-	})
 	var nodes []gomatrixserverlib.ServerName
 	for node := range nodemap {
 		nodes = append(nodes, gomatrixserverlib.ServerName(node))
@@ -229,53 +160,22 @@ func (n *Node) KnownNodes() []gomatrixserverlib.ServerName {
 }
 
 func (n *Node) SetMulticastEnabled(enabled bool) {
-	if enabled {
+	// TODO: There's no dynamic reconfiguration in Yggdrasil v0.4
-		n.config.MulticastInterfaces = []string{".*"}
+	// so we need a solution for this.
-	} else {
-		n.config.MulticastInterfaces = []string{}
-	}
-	n.multicast.UpdateConfig(n.config)
-	if !enabled {
-		n.DisconnectMulticastPeers()
-	}
 }
 
 func (n *Node) DisconnectMulticastPeers() {
-	for _, sp := range n.core.GetSwitchPeers() {
+	// TODO: There's no dynamic reconfiguration in Yggdrasil v0.4
-		if !strings.HasPrefix(sp.Endpoint, "fe80") {
+	// so we need a solution for this.
-			continue
-		}
-		if err := n.core.DisconnectPeer(sp.Port); err != nil {
-			n.log.Printf("Failed to disconnect port %d: %s", sp.Port, err)
-		}
-	}
 }
 
 func (n *Node) DisconnectNonMulticastPeers() {
-	for _, sp := range n.core.GetSwitchPeers() {
+	// TODO: There's no dynamic reconfiguration in Yggdrasil v0.4
-		if strings.HasPrefix(sp.Endpoint, "fe80") {
+	// so we need a solution for this.
-			continue
-		}
-		if err := n.core.DisconnectPeer(sp.Port); err != nil {
-			n.log.Printf("Failed to disconnect port %d: %s", sp.Port, err)
-		}
-	}
 }
 
 func (n *Node) SetStaticPeer(uri string) error {
-	n.config.Peers = []string{}
+	// TODO: There's no dynamic reconfiguration in Yggdrasil v0.4
-	n.core.UpdateConfig(n.config)
+	// so we need a solution for this.
-	n.DisconnectNonMulticastPeers()
-	if uri != "" {
-		n.log.Infoln("Adding static peer", uri)
-		if err := n.core.AddPeer(uri, ""); err != nil {
-			n.log.Warnln("Adding static peer failed:", err)
-			return err
-		}
-		if err := n.core.CallPeer(uri, ""); err != nil {
-			n.log.Warnln("Calling static peer failed:", err)
-			return err
-		}
-	}
 	return nil
 }

cmd/dendrite-demo-yggdrasil/yggconn/session.go

@@ -16,94 +16,17 @@ package yggconn
 
 import (
 	"context"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/hex"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	"math/big"
 	"net"
-	"time"
-
-	"github.com/lucas-clemente/quic-go"
-	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-	"github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil"
 )
 
-type session struct {
-	node    *Node
-	session quic.Session
-	address string
-	context context.Context
-	cancel  context.CancelFunc
-}
-
-func (n *Node) newSession(sess quic.Session, address string) *session {
-	ctx, cancel := context.WithCancel(context.TODO())
-	return &session{
-		node:    n,
-		session: sess,
-		address: address,
-		context: ctx,
-		cancel:  cancel,
-	}
-}
-
-func (s *session) kill() {
-	s.cancel()
-}
-
 func (n *Node) listenFromYgg() {
-	var err error
-	n.listener, err = quic.Listen(
-		n.core,       // yggdrasil.PacketConn
-		n.tlsConfig,  // TLS config
-		n.quicConfig, // QUIC config
-	)
-	if err != nil {
-		panic(err)
-	}
-
 	for {
-		n.log.Infoln("Waiting to accept QUIC sessions")
+		conn, err := n.utpSocket.Accept()
-		session, err := n.listener.Accept(context.TODO())
 		if err != nil {
-			n.log.Println("n.listener.Accept:", err)
+			n.log.Println("n.utpSocket.Accept:", err)
 			return
 		}
-		if len(session.ConnectionState().PeerCertificates) != 1 {
+		n.incoming <- conn
-			_ = session.CloseWithError(0, "expected a peer certificate")
-			continue
-		}
-		address := session.ConnectionState().PeerCertificates[0].DNSNames[0]
-		n.log.Infoln("Accepted connection from", address)
-		go n.newSession(session, address).listenFromQUIC()
-		go n.sessionFunc(address)
-	}
-}
-
-func (s *session) listenFromQUIC() {
-	if existing, ok := s.node.sessions.Load(s.address); ok {
-		if existingSession, ok := existing.(*session); ok {
-			fmt.Println("Killing existing session to replace", s.address)
-			existingSession.kill()
-		}
-	}
-	s.node.sessionCount.Inc()
-	s.node.sessions.Store(s.address, s)
-	defer s.node.sessions.Delete(s.address)
-	defer s.node.sessionCount.Dec()
-	for {
-		st, err := s.session.AcceptStream(s.context)
-		if err != nil {
-			s.node.log.Println("session.AcceptStream:", err)
-			return
-		}
-		s.node.incoming <- QUICStream{st, s.session}
 	}
 }
 
@@ -129,155 +52,5 @@ func (n *Node) Dial(network, address string) (net.Conn, error) {
 
 // Implements http.Transport.DialContext
 func (n *Node) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
-	s, ok1 := n.sessions.Load(address)
+	return n.utpSocket.DialContext(ctx, network, address)
-	session, ok2 := s.(*session)
-	if !ok1 || !ok2 {
-		// First of all, check if we think we know the coords of this
-		// node. If we do then we'll try to dial to it directly. This
-		// will either succeed or fail.
-		if v, ok := n.coords.Load(address); ok {
-			coords, ok := v.(yggdrasil.Coords)
-			if !ok {
-				n.coords.Delete(address)
-				return nil, errors.New("should have found yggdrasil.Coords but didn't")
-			}
-			n.log.Infof("Coords %s for %q cached, trying to dial", coords.String(), address)
-			var err error
-			// We think we know the coords. Try to dial the node.
-			if session, err = n.tryDial(address, coords); err != nil {
-				// We thought we knew the coords but it didn't result
-				// in a successful dial. Nuke them from the cache.
-				n.coords.Delete(address)
-				n.log.Infof("Cached coords %s for %q failed", coords.String(), address)
-			}
-		}
-
-		// We either don't know the coords for the node, or we failed
-		// to dial it before, in which case try to resolve the coords.
-		if _, ok := n.coords.Load(address); !ok {
-			var coords yggdrasil.Coords
-			var err error
-
-			// First look and see if the node is something that we already
-			// know about from our direct switch peers.
-			for _, peer := range n.core.GetSwitchPeers() {
-				if peer.PublicKey.String() == address {
-					coords = peer.Coords
-					n.log.Infof("%q is a direct peer, coords are %s", address, coords.String())
-					n.coords.Store(address, coords)
-					break
-				}
-			}
-
-			// If it isn' a node that we know directly then try to search
-			// the network.
-			if coords == nil {
-				n.log.Infof("Searching for coords for %q", address)
-				dest, derr := hex.DecodeString(address)
-				if derr != nil {
-					return nil, derr
-				}
-				if len(dest) != crypto.BoxPubKeyLen {
-					return nil, errors.New("invalid key length supplied")
-				}
-				var pubKey crypto.BoxPubKey
-				copy(pubKey[:], dest)
-				nodeID := crypto.GetNodeID(&pubKey)
-				nodeMask := &crypto.NodeID{}
-				for i := range nodeMask {
-					nodeMask[i] = 0xFF
-				}
-
-				fmt.Println("Resolving coords")
-				coords, err = n.core.Resolve(nodeID, nodeMask)
-				if err != nil {
-					return nil, fmt.Errorf("n.core.Resolve: %w", err)
-				}
-				fmt.Println("Found coords:", coords)
-				n.coords.Store(address, coords)
-			}
-
-			// We now know the coords in theory. Let's try dialling the
-			// node again.
-			if session, err = n.tryDial(address, coords); err != nil {
-				return nil, fmt.Errorf("n.tryDial: %w", err)
-			}
-		}
-	}
-
-	if session == nil {
-		return nil, fmt.Errorf("should have found session but didn't")
-	}
-
-	st, err := session.session.OpenStream()
-	if err != nil {
-		n.log.Println("session.OpenStream:", err)
-		_ = session.session.CloseWithError(0, "expected to be able to open session")
-		return nil, err
-	}
-	return QUICStream{st, session.session}, nil
-}
-
-func (n *Node) tryDial(address string, coords yggdrasil.Coords) (*session, error) {
-	quicSession, err := quic.Dial(
-		n.core,       // yggdrasil.PacketConn
-		coords,       // dial address
-		address,      // dial SNI
-		n.tlsConfig,  // TLS config
-		n.quicConfig, // QUIC config
-	)
-	if err != nil {
-		return nil, err
-	}
-	if len(quicSession.ConnectionState().PeerCertificates) != 1 {
-		_ = quicSession.CloseWithError(0, "expected a peer certificate")
-		return nil, errors.New("didn't receive a peer certificate")
-	}
-	if len(quicSession.ConnectionState().PeerCertificates[0].DNSNames) != 1 {
-		_ = quicSession.CloseWithError(0, "expected a DNS name")
-		return nil, errors.New("didn't receive a DNS name")
-	}
-	if gotAddress := quicSession.ConnectionState().PeerCertificates[0].DNSNames[0]; address != gotAddress {
-		_ = quicSession.CloseWithError(0, "you aren't the host I was hoping for")
-		return nil, fmt.Errorf("expected %q but dialled %q", address, gotAddress)
-	}
-	session := n.newSession(quicSession, address)
-	go session.listenFromQUIC()
-	go n.sessionFunc(address)
-	return session, nil
-}
-
-func (n *Node) generateTLSConfig() *tls.Config {
-	key, err := rsa.GenerateKey(rand.Reader, 1024)
-	if err != nil {
-		panic(err)
-	}
-	template := x509.Certificate{
-		Subject: pkix.Name{
-			CommonName: n.DerivedServerName(),
-		},
-		SerialNumber: big.NewInt(1),
-		NotAfter:     time.Now().Add(time.Hour * 24 * 365),
-		DNSNames:     []string{n.DerivedSessionName()},
-	}
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
-	if err != nil {
-		panic(err)
-	}
-	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
-
-	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
-	if err != nil {
-		panic(err)
-	}
-	return &tls.Config{
-		Certificates:       []tls.Certificate{tlsCert},
-		NextProtos:         []string{"quic-matrix-ygg"},
-		InsecureSkipVerify: true,
-		ClientAuth:         tls.RequireAnyClientCert,
-		GetClientCertificate: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-			return &tlsCert, nil
-		},
-	}
 }

cmd/dendrite-demo-yggdrasil/yggconn/stream.go

@@ -1,20 +0,0 @@
-package yggconn
-
-import (
-	"net"
-
-	"github.com/lucas-clemente/quic-go"
-)
-
-type QUICStream struct {
-	quic.Stream
-	session quic.Session
-}
-
-func (s QUICStream) LocalAddr() net.Addr {
-	return s.session.LocalAddr()
-}
-
-func (s QUICStream) RemoteAddr() net.Addr {
-	return s.session.RemoteAddr()
-}

dendrite-config.yaml

@@ -112,7 +112,7 @@ global:
     # Maximum number of entries to hold in the DNS cache, and
     # for how long those items should be considered valid in seconds.
     cache_size: 256
-    cache_lifetime: 300
+    cache_lifetime: "5m" # 5minutes; see https://pkg.go.dev/time@master#ParseDuration for more
 
 # Configuration for the Appservice API.
 app_service_api:

federationapi/routing/join.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // MakeJoin implements the /make_join API
@@ -228,6 +229,21 @@ func SendJoin(
 		}
 	}
 
+	// Check that this is in fact a join event
+	membership, err := event.Membership()
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing content.membership key"),
+		}
+	}
+	if membership != gomatrixserverlib.Join {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("membership must be 'join'"),
+		}
+	}
+
 	// Check that the event is signed by the server sending the request.
 	redacted := event.Redact()
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
@@ -296,16 +312,26 @@ func SendJoin(
 	// We are responsible for notifying other servers that the user has joined
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
 	if !alreadyJoined {
-		if err = api.SendEvents(
+		var response api.InputRoomEventsResponse
-			httpReq.Context(), rsAPI,
+		rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
-			api.KindNew,
+			InputRoomEvents: []api.InputRoomEvent{
-			[]*gomatrixserverlib.HeaderedEvent{
+				{
-				event.Headered(stateAndAuthChainResponse.RoomVersion),
+					Kind:          api.KindNew,
+					Event:         event.Headered(stateAndAuthChainResponse.RoomVersion),
+					AuthEventIDs:  event.AuthEventIDs(),
+					SendAsServer:  string(cfg.Matrix.ServerName),
+					TransactionID: nil,
 				},
-			cfg.Matrix.ServerName,
+			},
-			nil,
+		}, &response)
-		); err != nil {
+		if response.ErrMsg != "" {
-			util.GetLogger(httpReq.Context()).WithError(err).Error("SendEvents failed")
+			util.GetLogger(httpReq.Context()).WithField(logrus.ErrorKey, response.ErrMsg).Error("SendEvents failed")
+			if response.NotAllowed {
+				return util.JSONResponse{
+					Code: http.StatusBadRequest,
+					JSON: jsonerror.Forbidden(response.ErrMsg),
+				}
+			}
 			return jsonerror.InternalServerError()
 		}
 	}

federationapi/routing/keys.go

@@ -188,25 +188,30 @@ func NotaryKeys(
 	}
 	response.ServerKeys = []json.RawMessage{}
 
-	for serverName := range req.ServerKeys {
+	for serverName, kidToCriteria := range req.ServerKeys {
-		var keys *gomatrixserverlib.ServerKeys
+		var keyList []gomatrixserverlib.ServerKeys
 		if serverName == cfg.Matrix.ServerName {
 			if k, err := localKeys(cfg, time.Now().Add(cfg.Matrix.KeyValidityPeriod)); err == nil {
-				keys = k
+				keyList = append(keyList, *k)
 			} else {
 				return util.ErrorResponse(err)
 			}
 		} else {
-			if k, err := fsAPI.GetServerKeys(httpReq.Context(), serverName); err == nil {
+			var resp federationSenderAPI.QueryServerKeysResponse
-				keys = &k
+			err := fsAPI.QueryServerKeys(httpReq.Context(), &federationSenderAPI.QueryServerKeysRequest{
-			} else {
+				ServerName:      serverName,
+				KeyIDToCriteria: kidToCriteria,
+			}, &resp)
+			if err != nil {
 				return util.ErrorResponse(err)
 			}
+			keyList = append(keyList, resp.ServerKeys...)
 		}
-		if keys == nil {
+		if len(keyList) == 0 {
 			continue
 		}
 
+		for _, keys := range keyList {
 			j, err := json.Marshal(keys)
 			if err != nil {
 				logrus.WithError(err).Errorf("Failed to marshal %q response", serverName)
@@ -223,6 +228,7 @@ func NotaryKeys(
 
 			response.ServerKeys = append(response.ServerKeys, js)
 		}
+	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,

federationapi/routing/leave.go

@@ -22,6 +22,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // MakeLeave implements the /make_leave API
@@ -174,6 +175,13 @@ func SendLeave(
 		}
 	}
 
+	if event.StateKey() == nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue("missing state_key"),
+		}
+	}
+
 	// Check if the user has already left. If so, no-op!
 	queryReq := &api.QueryLatestEventsAndStateRequest{
 		RoomID: roomID,
@@ -240,7 +248,10 @@ func SendLeave(
 	mem, err := event.Membership()
 	if err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("event.Membership failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing content.membership key"),
+		}
 	}
 	if mem != gomatrixserverlib.Leave {
 		return util.JSONResponse{
@@ -252,16 +263,27 @@ func SendLeave(
 	// Send the events to the room server.
 	// We are responsible for notifying other servers that the user has left
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
-	if err = api.SendEvents(
+	var response api.InputRoomEventsResponse
-		httpReq.Context(), rsAPI,
+	rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
-		api.KindNew,
+		InputRoomEvents: []api.InputRoomEvent{
-		[]*gomatrixserverlib.HeaderedEvent{
+			{
-			event.Headered(verRes.RoomVersion),
+				Kind:          api.KindNew,
+				Event:         event.Headered(verRes.RoomVersion),
+				AuthEventIDs:  event.AuthEventIDs(),
+				SendAsServer:  string(cfg.Matrix.ServerName),
+				TransactionID: nil,
 			},
-		cfg.Matrix.ServerName,
+		},
-		nil,
+	}, &response)
-	); err != nil {
+
-		util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed")
+	if response.ErrMsg != "" {
+		util.GetLogger(httpReq.Context()).WithField(logrus.ErrorKey, response.ErrMsg).WithField("not_allowed", response.NotAllowed).Error("producer.SendEvents failed")
+		if response.NotAllowed {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.Forbidden(response.ErrMsg),
+			}
+		}
 		return jsonerror.InternalServerError()
 	}
 

federationsender/api/api.go

@@ -20,7 +20,6 @@ type FederationClient interface {
 	ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
 	QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
 	GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
-	GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error)
 	MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
 	MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, r gomatrixserverlib.MSC2946SpacesRequest) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
 	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
@@ -41,6 +40,8 @@ func (e *FederationClientError) Error() string {
 type FederationSenderInternalAPI interface {
 	FederationClient
 
+	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
+
 	// PerformDirectoryLookup looks up a remote room ID from a room alias.
 	PerformDirectoryLookup(
 		ctx context.Context,
@@ -94,6 +95,25 @@ type FederationSenderInternalAPI interface {
 	) error
 }
 
+type QueryServerKeysRequest struct {
+	ServerName      gomatrixserverlib.ServerName
+	KeyIDToCriteria map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyNotaryQueryCriteria
+}
+
+func (q *QueryServerKeysRequest) KeyIDs() []gomatrixserverlib.KeyID {
+	kids := make([]gomatrixserverlib.KeyID, len(q.KeyIDToCriteria))
+	i := 0
+	for keyID := range q.KeyIDToCriteria {
+		kids[i] = keyID
+		i++
+	}
+	return kids
+}
+
+type QueryServerKeysResponse struct {
+	ServerKeys []gomatrixserverlib.ServerKeys
+}
+
 type PerformDirectoryLookupRequest struct {
 	RoomAlias  string                       `json:"room_alias"`
 	ServerName gomatrixserverlib.ServerName `json:"server_name"`

federationsender/internal/api.go

@@ -202,20 +202,6 @@ func (a *FederationSenderInternalAPI) GetEvent(
 	return ires.(gomatrixserverlib.Transaction), nil
 }
 
-func (a *FederationSenderInternalAPI) GetServerKeys(
-	ctx context.Context, s gomatrixserverlib.ServerName,
-) (gomatrixserverlib.ServerKeys, error) {
-	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
-	defer cancel()
-	ires, err := a.doRequest(s, func() (interface{}, error) {
-		return a.federation.GetServerKeys(ctx, s)
-	})
-	if err != nil {
-		return gomatrixserverlib.ServerKeys{}, err
-	}
-	return ires.(gomatrixserverlib.ServerKeys), nil
-}
-
 func (a *FederationSenderInternalAPI) LookupServerKeys(
 	ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
 ) ([]gomatrixserverlib.ServerKeys, error) {

federationsender/internal/query.go

@@ -2,8 +2,12 @@ package internal
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	"github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
 )
 
 // QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI
@@ -20,3 +24,74 @@ func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom(
 
 	return
 }
+
+func (a *FederationSenderInternalAPI) fetchServerKeysDirectly(ctx context.Context, serverName gomatrixserverlib.ServerName) (*gomatrixserverlib.ServerKeys, error) {
+	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+	ires, err := a.doRequest(serverName, func() (interface{}, error) {
+		return a.federation.GetServerKeys(ctx, serverName)
+	})
+	if err != nil {
+		return nil, err
+	}
+	sks := ires.(gomatrixserverlib.ServerKeys)
+	return &sks, nil
+}
+
+func (a *FederationSenderInternalAPI) fetchServerKeysFromCache(
+	ctx context.Context, req *api.QueryServerKeysRequest,
+) ([]gomatrixserverlib.ServerKeys, error) {
+	var results []gomatrixserverlib.ServerKeys
+	for keyID, criteria := range req.KeyIDToCriteria {
+		serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{keyID})
+		if len(serverKeysResponses) == 0 {
+			return nil, fmt.Errorf("failed to find server key response for key ID %s", keyID)
+		}
+		// we should only get 1 result as we only gave 1 key ID
+		sk := serverKeysResponses[0]
+		util.GetLogger(ctx).Infof("fetchServerKeysFromCache: minvalid:%v  keys: %+v", criteria.MinimumValidUntilTS, sk)
+		if criteria.MinimumValidUntilTS != 0 {
+			// check if it's still valid. if they have the same value that's also valid
+			if sk.ValidUntilTS < criteria.MinimumValidUntilTS {
+				return nil, fmt.Errorf(
+					"found server response for key ID %s but it is no longer valid, min: %v valid_until: %v",
+					keyID, criteria.MinimumValidUntilTS, sk.ValidUntilTS,
+				)
+			}
+		}
+		results = append(results, sk)
+	}
+	return results, nil
+}
+
+func (a *FederationSenderInternalAPI) QueryServerKeys(
+	ctx context.Context, req *api.QueryServerKeysRequest, res *api.QueryServerKeysResponse,
+) error {
+	// attempt to satisfy the entire request from the cache first
+	results, err := a.fetchServerKeysFromCache(ctx, req)
+	if err == nil {
+		// satisfied entirely from cache, return it
+		res.ServerKeys = results
+		return nil
+	}
+	util.GetLogger(ctx).WithField("server", req.ServerName).WithError(err).Warn("notary: failed to satisfy keys request entirely from cache, hitting direct")
+
+	serverKeys, err := a.fetchServerKeysDirectly(ctx, req.ServerName)
+	if err != nil {
+		// try to load as much as we can from the cache in a best effort basis
+		util.GetLogger(ctx).WithField("server", req.ServerName).WithError(err).Warn("notary: failed to ask server for keys, returning best effort keys")
+		serverKeysResponses, dbErr := a.db.GetNotaryKeys(ctx, req.ServerName, req.KeyIDs())
+		if dbErr != nil {
+			return fmt.Errorf("notary: server returned %s, and db returned %s", err, dbErr)
+		}
+		res.ServerKeys = serverKeysResponses
+		return nil
+	}
+	// cache it!
+	if err = a.db.UpdateNotaryKeys(context.Background(), req.ServerName, *serverKeys); err != nil {
+		// non-fatal, still return the response
+		util.GetLogger(ctx).WithError(err).Warn("failed to UpdateNotaryKeys")
+	}
+	res.ServerKeys = []gomatrixserverlib.ServerKeys{*serverKeys}
+	return nil
+}

federationsender/inthttp/client.go

@@ -15,6 +15,7 @@ import (
 // HTTP paths for the internal HTTP API
 const (
 	FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom"
+	FederationSenderQueryServerKeysPath                  = "/federationsender/queryServerKeys"
 
 	FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup"
 	FederationSenderPerformJoinRequestPath            = "/federationsender/performJoinRequest"
@@ -31,7 +32,6 @@ const (
 	FederationSenderLookupStatePath        = "/federationsender/client/lookupState"
 	FederationSenderLookupStateIDsPath     = "/federationsender/client/lookupStateIDs"
 	FederationSenderGetEventPath           = "/federationsender/client/getEvent"
-	FederationSenderGetServerKeysPath      = "/federationsender/client/getServerKeys"
 	FederationSenderLookupServerKeysPath   = "/federationsender/client/lookupServerKeys"
 	FederationSenderEventRelationshipsPath = "/federationsender/client/msc2836eventRelationships"
 	FederationSenderSpacesSummaryPath      = "/federationsender/client/msc2946spacesSummary"
@@ -377,31 +377,14 @@ func (h *httpFederationSenderInternalAPI) GetEvent(
 	return *response.Res, nil
 }
 
-type getServerKeys struct {
+func (h *httpFederationSenderInternalAPI) QueryServerKeys(
-	S          gomatrixserverlib.ServerName
+	ctx context.Context, req *api.QueryServerKeysRequest, res *api.QueryServerKeysResponse,
-	ServerKeys gomatrixserverlib.ServerKeys
+) error {
-	Err        *api.FederationClientError
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServerKeys")
-}
-
-func (h *httpFederationSenderInternalAPI) GetServerKeys(
-	ctx context.Context, s gomatrixserverlib.ServerName,
-) (gomatrixserverlib.ServerKeys, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetServerKeys")
 	defer span.Finish()
 
-	request := getServerKeys{
+	apiURL := h.federationSenderURL + FederationSenderQueryServerKeysPath
-		S: s,
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
-	}
-	var response getServerKeys
-	apiURL := h.federationSenderURL + FederationSenderGetServerKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.ServerKeys{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.ServerKeys{}, response.Err
-	}
-	return response.ServerKeys, nil
 }
 
 type lookupServerKeys struct {

federationsender/inthttp/server.go

@@ -264,25 +264,17 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route
 		}),
 	)
 	internalAPIMux.Handle(
-		FederationSenderGetServerKeysPath,
+		FederationSenderQueryServerKeysPath,
-		httputil.MakeInternalAPI("GetServerKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("QueryServerKeys", func(req *http.Request) util.JSONResponse {
-			var request getServerKeys
+			var request api.QueryServerKeysRequest
+			var response api.QueryServerKeysResponse
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
 				return util.MessageResponse(http.StatusBadRequest, err.Error())
 			}
-			res, err := intAPI.GetServerKeys(req.Context(), request.S)
+			if err := intAPI.QueryServerKeys(req.Context(), &request, &response); err != nil {
-			if err != nil {
+				return util.ErrorResponse(err)
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
 			}
-				}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-			}
-			request.ServerKeys = res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
 		}),
 	)
 	internalAPIMux.Handle(

federationsender/storage/interface.go

@@ -66,4 +66,10 @@ type Database interface {
 	RenewInboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error
 	GetInboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string) (*types.InboundPeek, error)
 	GetInboundPeeks(ctx context.Context, roomID string) ([]types.InboundPeek, error)
+
+	// Update the notary with the given server keys from the given server name.
+	UpdateNotaryKeys(ctx context.Context, serverName gomatrixserverlib.ServerName, serverKeys gomatrixserverlib.ServerKeys) error
+	// Query the notary for the server keys for the given server. If `optKeyIDs` is not empty, multiple server keys may be returned (between 1 - len(optKeyIDs))
+	// such that the combination of all server keys will include all the `optKeyIDs`.
+	GetNotaryKeys(ctx context.Context, serverName gomatrixserverlib.ServerName, optKeyIDs []gomatrixserverlib.KeyID) ([]gomatrixserverlib.ServerKeys, error)
 }

federationsender/storage/postgres/notary_server_keys_json_table.go

@@ -0,0 +1,64 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/federationsender/storage/tables"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const notaryServerKeysJSONSchema = `
+CREATE SEQUENCE IF NOT EXISTS federationsender_notary_server_keys_json_pkey;
+CREATE TABLE IF NOT EXISTS federationsender_notary_server_keys_json (
+    notary_id BIGINT PRIMARY KEY NOT NULL DEFAULT nextval('federationsender_notary_server_keys_json_pkey'),
+	response_json TEXT NOT NULL,
+	server_name TEXT NOT NULL,
+	valid_until BIGINT NOT NULL
+);
+`
+
+const insertServerKeysJSONSQL = "" +
+	"INSERT INTO federationsender_notary_server_keys_json (response_json, server_name, valid_until) VALUES ($1, $2, $3)" +
+	" RETURNING notary_id"
+
+type notaryServerKeysStatements struct {
+	db                       *sql.DB
+	insertServerKeysJSONStmt *sql.Stmt
+}
+
+func NewPostgresNotaryServerKeysTable(db *sql.DB) (s *notaryServerKeysStatements, err error) {
+	s = &notaryServerKeysStatements{
+		db: db,
+	}
+	_, err = db.Exec(notaryServerKeysJSONSchema)
+	if err != nil {
+		return
+	}
+
+	if s.insertServerKeysJSONStmt, err = db.Prepare(insertServerKeysJSONSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *notaryServerKeysStatements) InsertJSONResponse(
+	ctx context.Context, txn *sql.Tx, keyQueryResponseJSON gomatrixserverlib.ServerKeys, serverName gomatrixserverlib.ServerName, validUntil gomatrixserverlib.Timestamp,
+) (tables.NotaryID, error) {
+	var notaryID tables.NotaryID
+	return notaryID, txn.Stmt(s.insertServerKeysJSONStmt).QueryRowContext(ctx, string(keyQueryResponseJSON.Raw), serverName, validUntil).Scan(&notaryID)
+}

federationsender/storage/postgres/notary_server_keys_metadata_table.go

@@ -0,0 +1,167 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/federationsender/storage/tables"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const notaryServerKeysMetadataSchema = `
+CREATE TABLE IF NOT EXISTS federationsender_notary_server_keys_metadata (
+    notary_id BIGINT NOT NULL,
+	server_name TEXT NOT NULL,
+	key_id TEXT NOT NULL,
+	UNIQUE (server_name, key_id)
+);
+`
+
+const upsertServerKeysSQL = "" +
+	"INSERT INTO federationsender_notary_server_keys_metadata (notary_id, server_name, key_id) VALUES ($1, $2, $3)" +
+	" ON CONFLICT (server_name, key_id) DO UPDATE SET notary_id = $1"
+
+// for a given (server_name, key_id), find the existing notary ID and valid until. Used to check if we will replace it
+// JOINs with the json table
+const selectNotaryKeyMetadataSQL = `
+	SELECT federationsender_notary_server_keys_metadata.notary_id, valid_until FROM federationsender_notary_server_keys_json
+	JOIN federationsender_notary_server_keys_metadata ON
+	federationsender_notary_server_keys_metadata.notary_id = federationsender_notary_server_keys_json.notary_id
+	WHERE federationsender_notary_server_keys_metadata.server_name = $1 AND federationsender_notary_server_keys_metadata.key_id = $2
+`
+
+// select the response which has the highest valid_until value
+// JOINs with the json table
+const selectNotaryKeyResponsesSQL = `
+	SELECT response_json FROM federationsender_notary_server_keys_json
+	WHERE server_name = $1 AND valid_until = (
+		SELECT MAX(valid_until) FROM federationsender_notary_server_keys_json WHERE server_name = $1
+	)
+`
+
+// select the responses which have the given key IDs
+// JOINs with the json table
+const selectNotaryKeyResponsesWithKeyIDsSQL = `
+	SELECT response_json FROM federationsender_notary_server_keys_json
+	JOIN federationsender_notary_server_keys_metadata ON
+	federationsender_notary_server_keys_metadata.notary_id = federationsender_notary_server_keys_json.notary_id
+	WHERE federationsender_notary_server_keys_json.server_name = $1 AND federationsender_notary_server_keys_metadata.key_id = ANY ($2)
+	GROUP BY federationsender_notary_server_keys_json.notary_id
+`
+
+// JOINs with the metadata table
+const deleteUnusedServerKeysJSONSQL = `
+	DELETE FROM federationsender_notary_server_keys_json WHERE federationsender_notary_server_keys_json.notary_id NOT IN (
+		SELECT DISTINCT notary_id FROM federationsender_notary_server_keys_metadata
+	)
+`
+
+type notaryServerKeysMetadataStatements struct {
+	db                                     *sql.DB
+	upsertServerKeysStmt                   *sql.Stmt
+	selectNotaryKeyResponsesStmt           *sql.Stmt
+	selectNotaryKeyResponsesWithKeyIDsStmt *sql.Stmt
+	selectNotaryKeyMetadataStmt            *sql.Stmt
+	deleteUnusedServerKeysJSONStmt         *sql.Stmt
+}
+
+func NewPostgresNotaryServerKeysMetadataTable(db *sql.DB) (s *notaryServerKeysMetadataStatements, err error) {
+	s = &notaryServerKeysMetadataStatements{
+		db: db,
+	}
+	_, err = db.Exec(notaryServerKeysMetadataSchema)
+	if err != nil {
+		return
+	}
+
+	if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil {
+		return
+	}
+	if s.selectNotaryKeyResponsesStmt, err = db.Prepare(selectNotaryKeyResponsesSQL); err != nil {
+		return
+	}
+	if s.selectNotaryKeyResponsesWithKeyIDsStmt, err = db.Prepare(selectNotaryKeyResponsesWithKeyIDsSQL); err != nil {
+		return
+	}
+	if s.selectNotaryKeyMetadataStmt, err = db.Prepare(selectNotaryKeyMetadataSQL); err != nil {
+		return
+	}
+	if s.deleteUnusedServerKeysJSONStmt, err = db.Prepare(deleteUnusedServerKeysJSONSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *notaryServerKeysMetadataStatements) UpsertKey(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, keyID gomatrixserverlib.KeyID, newNotaryID tables.NotaryID, newValidUntil gomatrixserverlib.Timestamp,
+) (tables.NotaryID, error) {
+	notaryID := newNotaryID
+	// see if the existing notary ID a) exists, b) has a longer valid_until
+	var existingNotaryID tables.NotaryID
+	var existingValidUntil gomatrixserverlib.Timestamp
+	if err := txn.Stmt(s.selectNotaryKeyMetadataStmt).QueryRowContext(ctx, serverName, keyID).Scan(&existingNotaryID, &existingValidUntil); err != nil {
+		if err != sql.ErrNoRows {
+			return 0, err
+		}
+	}
+	if existingValidUntil.Time().After(newValidUntil.Time()) {
+		// the existing valid_until is valid longer, so use that.
+		return existingNotaryID, nil
+	}
+	// overwrite the notary_id for this (server_name, key_id) tuple
+	_, err := txn.Stmt(s.upsertServerKeysStmt).ExecContext(ctx, notaryID, serverName, keyID)
+	return notaryID, err
+}
+
+func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, keyIDs []gomatrixserverlib.KeyID) ([]gomatrixserverlib.ServerKeys, error) {
+	var rows *sql.Rows
+	var err error
+	if len(keyIDs) == 0 {
+		rows, err = txn.Stmt(s.selectNotaryKeyResponsesStmt).QueryContext(ctx, string(serverName))
+	} else {
+		keyIDstr := make([]string, len(keyIDs))
+		for i := range keyIDs {
+			keyIDstr[i] = string(keyIDs[i])
+		}
+		rows, err = txn.Stmt(s.selectNotaryKeyResponsesWithKeyIDsStmt).QueryContext(ctx, string(serverName), pq.StringArray(keyIDstr))
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectNotaryKeyResponsesStmt close failed")
+	var results []gomatrixserverlib.ServerKeys
+	for rows.Next() {
+		var sk gomatrixserverlib.ServerKeys
+		var raw string
+		if err = rows.Scan(&raw); err != nil {
+			return nil, err
+		}
+		if err = json.Unmarshal([]byte(raw), &sk); err != nil {
+			return nil, err
+		}
+		results = append(results, sk)
+	}
+	return results, nil
+}
+
+func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error {
+	_, err := txn.Stmt(s.deleteUnusedServerKeysJSONStmt).ExecContext(ctx)
+	return err
+}

federationsender/storage/postgres/storage.go

@@ -17,6 +17,7 @@ package postgres
 
 import (
 	"database/sql"
+	"fmt"
 
 	"github.com/matrix-org/dendrite/federationsender/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/federationsender/storage/shared"
@@ -69,6 +70,14 @@ func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationS
 	if err != nil {
 		return nil, err
 	}
+	notaryJSON, err := NewPostgresNotaryServerKeysTable(d.db)
+	if err != nil {
+		return nil, fmt.Errorf("NewPostgresNotaryServerKeysTable: %s", err)
+	}
+	notaryMetadata, err := NewPostgresNotaryServerKeysMetadataTable(d.db)
+	if err != nil {
+		return nil, fmt.Errorf("NewPostgresNotaryServerKeysMetadataTable: %s", err)
+	}
 	m := sqlutil.NewMigrations()
 	deltas.LoadRemoveRoomsTable(m)
 	if err = m.RunDeltas(d.db, dbProperties); err != nil {
@@ -85,6 +94,8 @@ func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationS
 		FederationSenderBlacklist:     blacklist,
 		FederationSenderInboundPeeks:  inboundPeeks,
 		FederationSenderOutboundPeeks: outboundPeeks,
+		NotaryServerKeysJSON:          notaryJSON,
+		NotaryServerKeysMetadata:      notaryMetadata,
 	}
 	if err = d.PartitionOffsetStatements.Prepare(d.db, d.writer, "federationsender"); err != nil {
 		return nil, err

federationsender/storage/shared/storage.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"time"
 
 	"github.com/matrix-org/dendrite/federationsender/storage/tables"
 	"github.com/matrix-org/dendrite/federationsender/types"
@@ -37,6 +38,8 @@ type Database struct {
 	FederationSenderBlacklist     tables.FederationSenderBlacklist
 	FederationSenderOutboundPeeks tables.FederationSenderOutboundPeeks
 	FederationSenderInboundPeeks  tables.FederationSenderInboundPeeks
+	NotaryServerKeysJSON          tables.FederationSenderNotaryServerKeysJSON
+	NotaryServerKeysMetadata      tables.FederationSenderNotaryServerKeysMetadata
 }
 
 // An Receipt contains the NIDs of a call to GetNextTransactionPDUs/EDUs.
@@ -197,3 +200,47 @@ func (d *Database) GetInboundPeek(ctx context.Context, serverName gomatrixserver
 func (d *Database) GetInboundPeeks(ctx context.Context, roomID string) ([]types.InboundPeek, error) {
 	return d.FederationSenderInboundPeeks.SelectInboundPeeks(ctx, nil, roomID)
 }
+
+func (d *Database) UpdateNotaryKeys(ctx context.Context, serverName gomatrixserverlib.ServerName, serverKeys gomatrixserverlib.ServerKeys) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		validUntil := serverKeys.ValidUntilTS
+		// Servers MUST use the lesser of this field and 7 days into the future when determining if a key is valid.
+		// This is to avoid a situation where an attacker publishes a key which is valid for a significant amount of
+		// time without a way for the homeserver owner to revoke it.
+		// https://spec.matrix.org/unstable/server-server-api/#querying-keys-through-another-server
+		weekIntoFuture := time.Now().Add(7 * 24 * time.Hour)
+		if weekIntoFuture.Before(validUntil.Time()) {
+			validUntil = gomatrixserverlib.AsTimestamp(weekIntoFuture)
+		}
+		notaryID, err := d.NotaryServerKeysJSON.InsertJSONResponse(ctx, txn, serverKeys, serverName, validUntil)
+		if err != nil {
+			return err
+		}
+		// update the metadata for the keys
+		for keyID := range serverKeys.OldVerifyKeys {
+			_, err = d.NotaryServerKeysMetadata.UpsertKey(ctx, txn, serverName, keyID, notaryID, validUntil)
+			if err != nil {
+				return err
+			}
+		}
+		for keyID := range serverKeys.VerifyKeys {
+			_, err = d.NotaryServerKeysMetadata.UpsertKey(ctx, txn, serverName, keyID, notaryID, validUntil)
+			if err != nil {
+				return err
+			}
+		}
+
+		// clean up old responses
+		return d.NotaryServerKeysMetadata.DeleteOldJSONResponses(ctx, txn)
+	})
+}
+
+func (d *Database) GetNotaryKeys(
+	ctx context.Context, serverName gomatrixserverlib.ServerName, optKeyIDs []gomatrixserverlib.KeyID,
+) (sks []gomatrixserverlib.ServerKeys, err error) {
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		sks, err = d.NotaryServerKeysMetadata.SelectKeys(ctx, txn, serverName, optKeyIDs)
+		return err
+	})
+	return sks, err
+}

federationsender/storage/sqlite3/notary_server_keys_json_table.go

@@ -0,0 +1,63 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/federationsender/storage/tables"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const notaryServerKeysJSONSchema = `
+CREATE TABLE IF NOT EXISTS federationsender_notary_server_keys_json (
+    notary_id INTEGER PRIMARY KEY AUTOINCREMENT,
+	response_json TEXT NOT NULL,
+	server_name TEXT NOT NULL,
+	valid_until BIGINT NOT NULL
+);
+`
+
+const insertServerKeysJSONSQL = "" +
+	"INSERT INTO federationsender_notary_server_keys_json (response_json, server_name, valid_until) VALUES ($1, $2, $3)" +
+	" RETURNING notary_id"
+
+type notaryServerKeysStatements struct {
+	db                       *sql.DB
+	insertServerKeysJSONStmt *sql.Stmt
+}
+
+func NewSQLiteNotaryServerKeysTable(db *sql.DB) (s *notaryServerKeysStatements, err error) {
+	s = &notaryServerKeysStatements{
+		db: db,
+	}
+	_, err = db.Exec(notaryServerKeysJSONSchema)
+	if err != nil {
+		return
+	}
+
+	if s.insertServerKeysJSONStmt, err = db.Prepare(insertServerKeysJSONSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *notaryServerKeysStatements) InsertJSONResponse(
+	ctx context.Context, txn *sql.Tx, keyQueryResponseJSON gomatrixserverlib.ServerKeys, serverName gomatrixserverlib.ServerName, validUntil gomatrixserverlib.Timestamp,
+) (tables.NotaryID, error) {
+	var notaryID tables.NotaryID
+	return notaryID, txn.Stmt(s.insertServerKeysJSONStmt).QueryRowContext(ctx, string(keyQueryResponseJSON.Raw), serverName, validUntil).Scan(&notaryID)
+}

federationsender/storage/sqlite3/notary_server_keys_metadata_table.go

@@ -0,0 +1,169 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/matrix-org/dendrite/federationsender/storage/tables"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const notaryServerKeysMetadataSchema = `
+CREATE TABLE IF NOT EXISTS federationsender_notary_server_keys_metadata (
+    notary_id BIGINT NOT NULL,
+	server_name TEXT NOT NULL,
+	key_id TEXT NOT NULL,
+	UNIQUE (server_name, key_id)
+);
+`
+
+const upsertServerKeysSQL = "" +
+	"INSERT INTO federationsender_notary_server_keys_metadata (notary_id, server_name, key_id) VALUES ($1, $2, $3)" +
+	" ON CONFLICT (server_name, key_id) DO UPDATE SET notary_id = $1"
+
+// for a given (server_name, key_id), find the existing notary ID and valid until. Used to check if we will replace it
+// JOINs with the json table
+const selectNotaryKeyMetadataSQL = `
+	SELECT federationsender_notary_server_keys_metadata.notary_id, valid_until FROM federationsender_notary_server_keys_json
+	JOIN federationsender_notary_server_keys_metadata ON
+	federationsender_notary_server_keys_metadata.notary_id = federationsender_notary_server_keys_json.notary_id
+	WHERE federationsender_notary_server_keys_metadata.server_name = $1 AND federationsender_notary_server_keys_metadata.key_id = $2
+`
+
+// select the response which has the highest valid_until value
+// JOINs with the json table
+const selectNotaryKeyResponsesSQL = `
+	SELECT response_json FROM federationsender_notary_server_keys_json
+	WHERE server_name = $1 AND valid_until = (
+		SELECT MAX(valid_until) FROM federationsender_notary_server_keys_json WHERE server_name = $1
+	)
+`
+
+// select the responses which have the given key IDs
+// JOINs with the json table
+const selectNotaryKeyResponsesWithKeyIDsSQL = `
+	SELECT response_json FROM federationsender_notary_server_keys_json
+	JOIN federationsender_notary_server_keys_metadata ON
+	federationsender_notary_server_keys_metadata.notary_id = federationsender_notary_server_keys_json.notary_id
+	WHERE federationsender_notary_server_keys_json.server_name = $1 AND federationsender_notary_server_keys_metadata.key_id IN ($2)
+	GROUP BY federationsender_notary_server_keys_json.notary_id
+`
+
+// JOINs with the metadata table
+const deleteUnusedServerKeysJSONSQL = `
+	DELETE FROM federationsender_notary_server_keys_json WHERE federationsender_notary_server_keys_json.notary_id NOT IN (
+		SELECT DISTINCT notary_id FROM federationsender_notary_server_keys_metadata
+	)
+`
+
+type notaryServerKeysMetadataStatements struct {
+	db                             *sql.DB
+	upsertServerKeysStmt           *sql.Stmt
+	selectNotaryKeyResponsesStmt   *sql.Stmt
+	selectNotaryKeyMetadataStmt    *sql.Stmt
+	deleteUnusedServerKeysJSONStmt *sql.Stmt
+}
+
+func NewSQLiteNotaryServerKeysMetadataTable(db *sql.DB) (s *notaryServerKeysMetadataStatements, err error) {
+	s = &notaryServerKeysMetadataStatements{
+		db: db,
+	}
+	_, err = db.Exec(notaryServerKeysMetadataSchema)
+	if err != nil {
+		return
+	}
+
+	if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil {
+		return
+	}
+	if s.selectNotaryKeyResponsesStmt, err = db.Prepare(selectNotaryKeyResponsesSQL); err != nil {
+		return
+	}
+	if s.selectNotaryKeyMetadataStmt, err = db.Prepare(selectNotaryKeyMetadataSQL); err != nil {
+		return
+	}
+	if s.deleteUnusedServerKeysJSONStmt, err = db.Prepare(deleteUnusedServerKeysJSONSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *notaryServerKeysMetadataStatements) UpsertKey(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, keyID gomatrixserverlib.KeyID, newNotaryID tables.NotaryID, newValidUntil gomatrixserverlib.Timestamp,
+) (tables.NotaryID, error) {
+	notaryID := newNotaryID
+	// see if the existing notary ID a) exists, b) has a longer valid_until
+	var existingNotaryID tables.NotaryID
+	var existingValidUntil gomatrixserverlib.Timestamp
+	if err := txn.Stmt(s.selectNotaryKeyMetadataStmt).QueryRowContext(ctx, serverName, keyID).Scan(&existingNotaryID, &existingValidUntil); err != nil {
+		if err != sql.ErrNoRows {
+			return 0, err
+		}
+	}
+	if existingValidUntil.Time().After(newValidUntil.Time()) {
+		// the existing valid_until is valid longer, so use that.
+		return existingNotaryID, nil
+	}
+	// overwrite the notary_id for this (server_name, key_id) tuple
+	_, err := txn.Stmt(s.upsertServerKeysStmt).ExecContext(ctx, notaryID, serverName, keyID)
+	return notaryID, err
+}
+
+func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, keyIDs []gomatrixserverlib.KeyID) ([]gomatrixserverlib.ServerKeys, error) {
+	var rows *sql.Rows
+	var err error
+	if len(keyIDs) == 0 {
+		rows, err = txn.Stmt(s.selectNotaryKeyResponsesStmt).QueryContext(ctx, string(serverName))
+	} else {
+		iKeyIDs := make([]interface{}, len(keyIDs)+1)
+		iKeyIDs[0] = serverName
+		for i := range keyIDs {
+			iKeyIDs[i+1] = string(keyIDs[i])
+		}
+		sql := strings.Replace(selectNotaryKeyResponsesWithKeyIDsSQL, "($2)", sqlutil.QueryVariadicOffset(len(keyIDs), 1), 1)
+		fmt.Println(sql)
+		fmt.Println(iKeyIDs...)
+		rows, err = s.db.QueryContext(ctx, sql, iKeyIDs...)
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectNotaryKeyResponsesStmt close failed")
+	var results []gomatrixserverlib.ServerKeys
+	for rows.Next() {
+		var sk gomatrixserverlib.ServerKeys
+		var raw string
+		if err = rows.Scan(&raw); err != nil {
+			return nil, err
+		}
+		if err = json.Unmarshal([]byte(raw), &sk); err != nil {
+			return nil, err
+		}
+		results = append(results, sk)
+	}
+	return results, nil
+}
+
+func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error {
+	_, err := txn.Stmt(s.deleteUnusedServerKeysJSONStmt).ExecContext(ctx)
+	return err
+}

federationsender/storage/sqlite3/storage.go

@@ -71,6 +71,14 @@ func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationS
 	if err != nil {
 		return nil, err
 	}
+	notaryKeys, err := NewSQLiteNotaryServerKeysTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	notaryKeysMetadata, err := NewSQLiteNotaryServerKeysMetadataTable(d.db)
+	if err != nil {
+		return nil, err
+	}
 	m := sqlutil.NewMigrations()
 	deltas.LoadRemoveRoomsTable(m)
 	if err = m.RunDeltas(d.db, dbProperties); err != nil {
@@ -87,6 +95,8 @@ func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationS
 		FederationSenderBlacklist:     blacklist,
 		FederationSenderOutboundPeeks: outboundPeeks,
 		FederationSenderInboundPeeks:  inboundPeeks,
+		NotaryServerKeysJSON:          notaryKeys,
+		NotaryServerKeysMetadata:      notaryKeysMetadata,
 	}
 	if err = d.PartitionOffsetStatements.Prepare(d.db, d.writer, "federationsender"); err != nil {
 		return nil, err

federationsender/storage/tables/interface.go

@@ -22,6 +22,8 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
+type NotaryID int64
+
 type FederationSenderQueuePDUs interface {
 	InsertQueuePDU(ctx context.Context, txn *sql.Tx, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, nid int64) error
 	DeleteQueuePDUs(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, jsonNIDs []int64) error
@@ -80,3 +82,25 @@ type FederationSenderInboundPeeks interface {
 	DeleteInboundPeek(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, roomID, peekID string) (err error)
 	DeleteInboundPeeks(ctx context.Context, txn *sql.Tx, roomID string) (err error)
 }
+
+// FederationSenderNotaryServerKeysJSON contains the byte-for-byte responses from servers which contain their keys and is signed by them.
+type FederationSenderNotaryServerKeysJSON interface {
+	// InsertJSONResponse inserts a new response JSON. Useless on its own, needs querying via FederationSenderNotaryServerKeysMetadata
+	// `validUntil` should be the value of `valid_until_ts` with the 7-day check applied from:
+	//   "Servers MUST use the lesser of this field and 7 days into the future when determining if a key is valid.
+	//    This is to avoid a situation where an attacker publishes a key which is valid for a significant amount of time
+	//    without a way for the homeserver owner to revoke it.""
+	InsertJSONResponse(ctx context.Context, txn *sql.Tx, keyQueryResponseJSON gomatrixserverlib.ServerKeys, serverName gomatrixserverlib.ServerName, validUntil gomatrixserverlib.Timestamp) (NotaryID, error)
+}
+
+// FederationSenderNotaryServerKeysMetadata persists the metadata for FederationSenderNotaryServerKeysJSON
+type FederationSenderNotaryServerKeysMetadata interface {
+	// UpsertKey updates or inserts a (server_name, key_id) tuple, pointing it via NotaryID at the the response which has the longest valid_until_ts
+	// `newNotaryID` and `newValidUntil` should be the notary ID / valid_until  which has this (server_name, key_id) tuple already, e.g one you just inserted.
+	UpsertKey(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, keyID gomatrixserverlib.KeyID, newNotaryID NotaryID, newValidUntil gomatrixserverlib.Timestamp) (NotaryID, error)
+	// SelectKeys returns the signed JSON objects which contain the given key IDs. This will be at most the length of `keyIDs` and at least 1 (assuming
+	// the keys exist in the first place). If `keyIDs` is empty, the signed JSON object with the longest valid_until_ts will be returned.
+	SelectKeys(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, keyIDs []gomatrixserverlib.KeyID) ([]gomatrixserverlib.ServerKeys, error)
+	// DeleteOldJSONResponses removes all responses which are not referenced in FederationSenderNotaryServerKeysMetadata
+	DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error
+}

go.mod

@@ -5,6 +5,7 @@ replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-serve
 replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.11.1-0.20210715085246-cd5b4d5a89fe
 
 require (
+	github.com/Arceliar/ironwood v0.0.0-20210619124114-6ad55cae5031
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1
@@ -36,13 +37,14 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20210625141222-bd2b7124cee8
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20210712160706-d37cd465be8f
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20210719140634-f44a103bb12e
 	github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nats-io/nats-server/v2 v2.3.2
 	github.com/nats-io/nats.go v1.11.1-0.20210623165838-4b75fc59ae30
+	github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6
 	github.com/opentracing/opentracing-go v1.2.0
@@ -57,11 +59,11 @@ require (
 	github.com/tidwall/sjson v1.1.7
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
 	github.com/uber/jaeger-lib v2.4.0+incompatible
-	github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20210218094457-e77ca8019daa
+	github.com/yggdrasil-network/yggdrasil-go v0.4.1-0.20210715083903-52309d094c00
 	go.uber.org/atomic v1.7.0
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97
 	golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08
-	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
+	golang.org/x/net v0.0.0-20210610132358-84b48f89b13b
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
 	gopkg.in/h2non/bimg.v1 v1.1.5

go.sum

@@ -31,8 +31,10 @@ dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
-github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48=
+github.com/Arceliar/ironwood v0.0.0-20210619124114-6ad55cae5031 h1:DZVDfYhVdu+0wAiRHoY1olyNkKxIot9UjBnbQFzuUlM=
-github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
+github.com/Arceliar/ironwood v0.0.0-20210619124114-6ad55cae5031/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
+github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ=
+github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -96,6 +98,7 @@ github.com/Shopify/sarama v1.29.0/go.mod h1:2QpgD79wpdAESqNQMxNc0KYMkycd4slxGdV3
 github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
@@ -116,7 +119,9 @@ github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgw
 github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo=
 github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQyjHvw=
 github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y=
+github.com/anacrolix/missinggo/perf v1.0.0 h1:7ZOGYziGEBytW49+KmYGTaNfnwUqP1HBsy6BqESAJVw=
 github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ=
+github.com/anacrolix/sync v0.2.0 h1:oRe22/ZB+v7v/5Mbc4d2zE0AXEZy0trKyKLjqYOt6tY=
 github.com/anacrolix/sync v0.2.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g=
 github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
 github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk=
@@ -181,7 +186,7 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
-github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw=
+github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -365,6 +370,8 @@ github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHj
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
@@ -490,7 +497,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/gologme/log v1.2.0 h1:Ya5Ip/KD6FX7uH0S31QO87nCCSucKtF44TLbTtO7V4c=
 github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
@@ -592,7 +598,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hjson/hjson-go v3.0.2-0.20200316202735-d5d0e8b0617d+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
+github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0=
@@ -737,7 +743,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
-github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0/go.mod h1:rUi0/YffDo1oXBOGn1KRq7Fr07LX48XEBecQnmwjsAo=
+github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc=
 github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk=
 github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U=
 github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw=
@@ -996,7 +1002,7 @@ github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z
 github.com/lunixbochs/vtclean v0.0.0-20160125035106-4fbf7632a2c6/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
-github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -1023,8 +1029,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210625141222-bd2b7124cee8/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20210712160706-d37cd465be8f h1:k6guD5GpbnFcy2JonolQ3LhlgtwyqBBmd3nQPTwliO0=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20210719140634-f44a103bb12e h1:dYPsAU1363AEbcohNzBJHV0CkbfVa8QiMNFib/i5pS8=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20210712160706-d37cd465be8f/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20210719140634-f44a103bb12e/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b h1:5X5vdWQ13xrNkJVqaJHPsrt7rKkMJH5iac0EtfOuxSg=
 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@@ -1034,6 +1040,7 @@ github.com/mattn/go-colorable v0.0.6/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -1041,11 +1048,12 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
-github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
+github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb h1:ax2vG2unlxsjwS7PMRo4FECIfAdQLowd6ejWYwPQhBo=
 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
@@ -1084,6 +1092,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
@@ -1182,8 +1191,9 @@ github.com/neilalexander/nats-server/v2 v2.3.3-0.20210714094623-648cf26af922 h1:
 github.com/neilalexander/nats-server/v2 v2.3.3-0.20210714094623-648cf26af922/go.mod h1:dUf7Cm5z5LbciFVwWx54owyCKm8x4/hL6p7rrljhLFY=
 github.com/neilalexander/nats.go v1.11.1-0.20210715085246-cd5b4d5a89fe h1:0lmDLHkYtQOyRpX8CkB3h6aQ71soPvoUD/A/7WmkuLw=
 github.com/neilalexander/nats.go v1.11.1-0.20210715085246-cd5b4d5a89fe/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
-github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488 h1:xZk82i6JK2d0SqRIXwaxj7J/NQB6ngq0PuMx3wXBaRQ=
 github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
+github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b h1:XNm+Ks3bVziRJxcMaIbzumWEw7l52z9Rek6cMHgln1g=
+github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b/go.mod h1:ylsx0342RjGHjOoVKhR/wz/7Lhiusonihfj4QLxEMcU=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww=
@@ -1329,6 +1339,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@@ -1469,13 +1481,12 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
-github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
-github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
@@ -1500,8 +1511,8 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
-github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20210218094457-e77ca8019daa h1:YHeZ1KN4KmuAjqmBSan1JtwyoPQoklzMjMqIbaS5Ywo=
+github.com/yggdrasil-network/yggdrasil-go v0.4.1-0.20210715083903-52309d094c00 h1:bv6+5Dv7XHbThfXirMV9hh45hUH26LtuZKHrVlM/3rQ=
-github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20210218094457-e77ca8019daa/go.mod h1:G716RAw9WTLbLFI7lVj1GKTU16wb9MYl6iE9j4JlWeI=
+github.com/yggdrasil-network/yggdrasil-go v0.4.1-0.20210715083903-52309d094c00/go.mod h1:/iMJjOrXRsjlFgqhWOPhecOKi7xHmHiY4/En3A42Fog=
 github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
@@ -1577,11 +1588,11 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
@@ -1669,11 +1680,12 @@ golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210427231257-85d9c07bbe3a/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
+golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210610132358-84b48f89b13b h1:k+E048sYJHyVnsr1GDrRZWQ32D2C7lWs9JRc0bel53A=
+golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1734,12 +1746,10 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1757,6 +1767,7 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1773,11 +1784,13 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1790,9 +1803,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7-0.20210503195748-5c7c50ebbd4f h1:yQJrRE0hDxDFmZLlRaw+3vusO4fwNHgHIjUOMO7bHYI=
+golang.org/x/text v0.3.7-0.20210503195748-5c7c50ebbd4f/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1854,10 +1867,9 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.zx2c4.com/wireguard v0.0.0-20210203165646-9c7bd73be2cc/go.mod h1:r0ExowOoGFfDoLDxx+M9SYbNVsoZ0xviLL+K4f2mt+A=
-golang.zx2c4.com/wireguard v0.0.0-20210212170059-7a0fb5bbb172/go.mod h1:r0ExowOoGFfDoLDxx+M9SYbNVsoZ0xviLL+K4f2mt+A=
 golang.zx2c4.com/wireguard v0.0.0-20210510202332-9844c74f67ec/go.mod h1:a057zjmoc00UN7gVkaJt2sXVK523kMJcogDTEvPIasg=
-golang.zx2c4.com/wireguard/windows v0.3.5/go.mod h1:ATrIFNoq3rsK735WJiQzfWYyNFc9xLBhMMjW9DWIvnU=
+golang.zx2c4.com/wireguard v0.0.0-20210604143328-f9b48a961cd2/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
+golang.zx2c4.com/wireguard/windows v0.3.14/go.mod h1:3P4IEAsb+BjlKZmpUXgy74c0iX9AVwwr3WcVJ8nPgME=
 google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=

mediaapi/routing/upload.go

@@ -147,20 +147,17 @@ func (r *uploadRequest) doUpload(
 	//   r.storeFileAndMetadata(ctx, tmpDir, ...)
 	// before you return from doUpload else we will leak a temp file. We could make this nicer with a `WithTransaction` style of
 	// nested function to guarantee either storage or cleanup.
-
+	if *cfg.MaxFileSizeBytes > 0 {
-	// should not happen, but prevents any int overflows
+		if *cfg.MaxFileSizeBytes+1 <= 0 {
-	if cfg.MaxFileSizeBytes != nil && *cfg.MaxFileSizeBytes+1 <= 0 {
 			r.Logger.WithFields(log.Fields{
-			"MaxFileSizeBytes": *cfg.MaxFileSizeBytes + 1,
+				"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
-		}).Error("Error while transferring file, configured max_file_size_bytes overflows int64")
+			}).Warnf("Configured MaxFileSizeBytes overflows int64, defaulting to %d bytes", config.DefaultMaxFileSizeBytes)
-		return &util.JSONResponse{
+			cfg.MaxFileSizeBytes = &config.DefaultMaxFileSizeBytes
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.Unknown("Failed to upload"),
 		}
+		reqReader = io.LimitReader(reqReader, int64(*cfg.MaxFileSizeBytes)+1)
 	}
 
-	lr := io.LimitReader(reqReader, int64(*cfg.MaxFileSizeBytes)+1)
+	hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, reqReader, cfg.AbsBasePath)
-	hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, lr, cfg.AbsBasePath)
 	if err != nil {
 		r.Logger.WithError(err).WithFields(log.Fields{
 			"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,

roomserver/state/state.go

@@ -424,12 +424,17 @@ func (v *StateResolution) loadStateAfterEventsForNumericTuples(
 	return result, nil
 }
 
-var calculateStateDurations = prometheus.NewSummaryVec(
+var calculateStateDurations = prometheus.NewHistogramVec(
-	prometheus.SummaryOpts{
+	prometheus.HistogramOpts{
 		Namespace: "dendrite",
 		Subsystem: "roomserver",
-		Name:      "calculate_state_duration_microseconds",
+		Name:      "calculate_state_duration_milliseconds",
 		Help:      "How long it takes to calculate the state after a list of events",
+		Buckets: []float64{ // milliseconds
+			5, 10, 25, 50, 75, 100, 200, 300, 400, 500,
+			1000, 2000, 3000, 4000, 5000, 6000,
+			7000, 8000, 9000, 10000, 15000, 20000, 30000,
+		},
 	},
 	// Takes two labels:
 	//   algorithm:
@@ -496,9 +501,8 @@ func (c *calculateStateMetrics) stop(stateNID types.StateSnapshotNID, err error)
 	} else {
 		outcome = "failure"
 	}
-	endTime := time.Now()
 	calculateStateDurations.WithLabelValues(c.algorithm, outcome).Observe(
-		float64(endTime.Sub(c.startTime).Nanoseconds()) / 1000.,
+		float64(time.Since(c.startTime).Milliseconds()),
 	)
 	calculateStatePrevEventLength.WithLabelValues(c.algorithm, outcome).Observe(
 		float64(c.prevEventLength),

roomserver/storage/shared/storage.go

@@ -866,6 +866,10 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s
 		return nil, err
 	}
 	stateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, stateKey)
+	if err == sql.ErrNoRows {
+		// No rooms have a state event with this state key, otherwise we'd have an state key NID
+		return nil, nil
+	}
 	if err != nil {
 		return nil, err
 	}

setup/base.go

@@ -137,15 +137,14 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 
 	var dnsCache *gomatrixserverlib.DNSCache
 	if cfg.Global.DNSCache.Enabled {
-		lifetime := time.Second * cfg.Global.DNSCache.CacheLifetime
 		dnsCache = gomatrixserverlib.NewDNSCache(
 			cfg.Global.DNSCache.CacheSize,
-			lifetime,
+			cfg.Global.DNSCache.CacheLifetime,
 		)
 		logrus.Infof(
 			"DNS cache enabled (size %d, lifetime %s)",
 			cfg.Global.DNSCache.CacheSize,
-			lifetime,
+			cfg.Global.DNSCache.CacheLifetime,
 		)
 	}
 

setup/config/config_mediaapi.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"fmt"
-	"math"
 )
 
 type MediaAPI struct {
@@ -36,6 +35,9 @@ type MediaAPI struct {
 	ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"`
 }
 
+// DefaultMaxFileSizeBytes defines the default file size allowed in transfers
+var DefaultMaxFileSizeBytes = FileSizeBytes(10485760)
+
 func (c *MediaAPI) Defaults() {
 	c.InternalAPI.Listen = "http://localhost:7774"
 	c.InternalAPI.Connect = "http://localhost:7774"
@@ -43,8 +45,7 @@ func (c *MediaAPI) Defaults() {
 	c.Database.Defaults(5)
 	c.Database.ConnectionString = "file:mediaapi.db"
 
-	defaultMaxFileSizeBytes := FileSizeBytes(10485760)
+	c.MaxFileSizeBytes = &DefaultMaxFileSizeBytes
-	c.MaxFileSizeBytes = &defaultMaxFileSizeBytes
 	c.MaxThumbnailGenerators = 10
 	c.BasePath = "./media_store"
 }
@@ -58,11 +59,6 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
 
 	checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))
-	// allow "unlimited" file size
-	if c.MaxFileSizeBytes != nil && *c.MaxFileSizeBytes <= 0 {
-		unlimitedSize := FileSizeBytes(math.MaxInt64 - 1)
-		c.MaxFileSizeBytes = &unlimitedSize
-	}
 	checkPositive(configErrs, "media_api.max_file_size_bytes", int64(*c.MaxFileSizeBytes))
 	checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators))
 

syncapi/routing/messages.go

@@ -393,7 +393,7 @@ func (r *messagesReq) getStartEnd(events []*gomatrixserverlib.HeaderedEvent) (st
 		start = *r.from
 		if events[len(events)-1].Type() == gomatrixserverlib.MRoomCreate {
 			// NOTSPEC: We've hit the beginning of the room so there's really nowhere
-			// else to go. This seems to fix Riot iOS from looping on /messages endlessly.
+			// else to go. This seems to fix Element iOS from looping on /messages endlessly.
 			end = types.TopologyToken{}
 		} else {
 			end, err = r.db.EventPositionInTopology(

sytest-whitelist

@@ -520,6 +520,8 @@ Inviting an AS-hosted user asks the AS server
 Can generate a openid access_token that can be exchanged for information about a user
 Invalid openid access tokens are rejected
 Requests to userinfo without access tokens are rejected
+'ban' event respects room powerlevel
+Non-present room members cannot ban others
 POST /_synapse/admin/v1/register with shared secret
 POST /_synapse/admin/v1/register admin with shared secret
 POST /_synapse/admin/v1/register with shared secret downcases capitals
@@ -530,3 +532,8 @@ Inbound federation can receive invite rejections
 Inbound federation can receive invite and reject when remote replies with a 403
 Inbound federation can receive invite and reject when remote replies with a 500
 Inbound federation can receive invite and reject when remote is unreachable
+Remote servers cannot set power levels in rooms without existing powerlevels
+Remote servers should reject attempts by non-creators to set the power levels
+Federation handles empty auth_events in state_ids sanely
+Key notary server should return an expired key if it can't find any others
+Key notary server must not overwrite a valid key with a spurious result from the origin server