Merge branch 'master' into neilalexander/excludesync

README.md

@@ -75,7 +75,7 @@ Then point your favourite Matrix client at `http://localhost:8008`.
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of October 2020 we're at around 56% CS API coverage and 77% Federation coverage, though check
+updates with CI. As of October 2020 we're at around 57% CS API coverage and 81% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably:
  - Receipts

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7
 	github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.2
@@ -40,6 +40,7 @@ require (
 	github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee
 	go.uber.org/atomic v1.6.0
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
+	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
 	gopkg.in/h2non/bimg.v1 v1.1.4
 	gopkg.in/yaml.v2 v2.3.0
 )

go.sum

@@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 h1:GF1PxbvImWDoz1DQZNMoaYtIqQXtyLAtmQOzwwmw1OI=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7 h1:YPuewGCKaJh08NslYAhyGiLw2tg6ew9LtkW7Xr+4uTU=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=

internal/setup/base.go

@@ -15,8 +15,10 @@
 package setup
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"time"
@@ -25,6 +27,8 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
@@ -107,7 +111,22 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 		logrus.WithError(err).Warnf("Failed to create cache")
 	}
 
-	apiClient := http.Client{Timeout: time.Minute * 10}
+	apiClient := http.Client{
+		Timeout: time.Minute * 10,
+		Transport: &http2.Transport{
+			AllowHTTP: true,
+			DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) {
+				// Ordinarily HTTP/2 would expect TLS, but the remote listener is
+				// H2C-enabled (HTTP/2 without encryption). Overriding the DialTLS
+				// function with a plain Dial allows us to trick the HTTP client
+				// into establishing a HTTP/2 connection without TLS.
+				// TODO: Eventually we will want to look at authenticating and
+				// encrypting these internal HTTP APIs, at which point we will have
+				// to reconsider H2C and change all this anyway.
+				return net.Dial(network, addr)
+			},
+		},
+	}
 	client := http.Client{Timeout: HTTPClientTimeout}
 	if cfg.FederationSender.Proxy.Enabled {
 		client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{
@@ -269,10 +288,17 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 	internalServ := externalServ
 
 	if internalAddr != NoListener && externalAddr != internalAddr {
+		// H2C allows us to accept HTTP/2 connections without TLS
+		// encryption. Since we don't currently require any form of
+		// authentication or encryption on these internal HTTP APIs,
+		// H2C gives us all of the advantages of HTTP/2 (such as
+		// stream multiplexing and avoiding head-of-line blocking)
+		// without enabling TLS.
+		internalH2S := &http2.Server{}
 		internalRouter = mux.NewRouter().SkipClean(true).UseEncodedPath()
 		internalServ = &http.Server{
 			Addr:    string(internalAddr),
-			Handler: internalRouter,
+			Handler: h2c.NewHandler(internalRouter, internalH2S),
 		}
 	}
 

roomserver/storage/shared/storage.go

@@ -27,23 +27,24 @@ import (
 const redactionsArePermanent = true
 
 type Database struct {
-	DB                  *sql.DB
+	DB                         *sql.DB
-	Cache               caching.RoomServerCaches
+	Cache                      caching.RoomServerCaches
-	Writer              sqlutil.Writer
+	Writer                     sqlutil.Writer
-	EventsTable         tables.Events
+	EventsTable                tables.Events
-	EventJSONTable      tables.EventJSON
+	EventJSONTable             tables.EventJSON
-	EventTypesTable     tables.EventTypes
+	EventTypesTable            tables.EventTypes
-	EventStateKeysTable tables.EventStateKeys
+	EventStateKeysTable        tables.EventStateKeys
-	RoomsTable          tables.Rooms
+	RoomsTable                 tables.Rooms
-	TransactionsTable   tables.Transactions
+	TransactionsTable          tables.Transactions
-	StateSnapshotTable  tables.StateSnapshot
+	StateSnapshotTable         tables.StateSnapshot
-	StateBlockTable     tables.StateBlock
+	StateBlockTable            tables.StateBlock
-	RoomAliasesTable    tables.RoomAliases
+	RoomAliasesTable           tables.RoomAliases
-	PrevEventsTable     tables.PreviousEvents
+	PrevEventsTable            tables.PreviousEvents
-	InvitesTable        tables.Invites
+	InvitesTable               tables.Invites
-	MembershipTable     tables.Membership
+	MembershipTable            tables.Membership
-	PublishedTable      tables.Published
+	PublishedTable             tables.Published
-	RedactionsTable     tables.Redactions
+	RedactionsTable            tables.Redactions
+	GetLatestEventsForUpdateFn func(ctx context.Context, roomInfo types.RoomInfo) (*LatestEventsUpdater, error)
 }
 
 func (d *Database) SupportsConcurrentRoomInputs() bool {
@@ -372,6 +373,9 @@ func (d *Database) MembershipUpdater(
 func (d *Database) GetLatestEventsForUpdate(
 	ctx context.Context, roomInfo types.RoomInfo,
 ) (*LatestEventsUpdater, error) {
+	if d.GetLatestEventsForUpdateFn != nil {
+		return d.GetLatestEventsForUpdateFn(ctx, roomInfo)
+	}
 	txn, err := d.DB.Begin()
 	if err != nil {
 		return nil, err

roomserver/storage/sqlite3/storage.go

@@ -120,23 +120,24 @@ func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches)
 		return nil, err
 	}
 	d.Database = shared.Database{
-		DB:                  d.db,
+		DB:                         d.db,
-		Cache:               cache,
+		Cache:                      cache,
-		Writer:              d.writer,
+		Writer:                     d.writer,
-		EventsTable:         d.events,
+		EventsTable:                d.events,
-		EventTypesTable:     d.eventTypes,
+		EventTypesTable:            d.eventTypes,
-		EventStateKeysTable: d.eventStateKeys,
+		EventStateKeysTable:        d.eventStateKeys,
-		EventJSONTable:      d.eventJSON,
+		EventJSONTable:             d.eventJSON,
-		RoomsTable:          d.rooms,
+		RoomsTable:                 d.rooms,
-		TransactionsTable:   d.transactions,
+		TransactionsTable:          d.transactions,
-		StateBlockTable:     stateBlock,
+		StateBlockTable:            stateBlock,
-		StateSnapshotTable:  stateSnapshot,
+		StateSnapshotTable:         stateSnapshot,
-		PrevEventsTable:     d.prevEvents,
+		PrevEventsTable:            d.prevEvents,
-		RoomAliasesTable:    roomAliases,
+		RoomAliasesTable:           roomAliases,
-		InvitesTable:        d.invites,
+		InvitesTable:               d.invites,
-		MembershipTable:     d.membership,
+		MembershipTable:            d.membership,
-		PublishedTable:      published,
+		PublishedTable:             published,
-		RedactionsTable:     redactions,
+		RedactionsTable:            redactions,
+		GetLatestEventsForUpdateFn: d.GetLatestEventsForUpdate,
 	}
 	return &d, nil
 }

sytest-whitelist

@@ -485,3 +485,4 @@ Event with an invalid signature in the send_join response should not cause room
 Inbound federation rejects typing notifications from wrong remote
 Should not be able to take over the room by pretending there is no PL event
 Can get rooms/{roomId}/state for a departed room (SPEC-216)
+Users cannot set notifications powerlevel higher than their own