Merge branch 'master' into event-redaction

CONTRIBUTING.md

@@ -23,11 +23,10 @@ run](scripts/build-test-lint.sh).
 ## Continuous Integration
 
 When a Pull Request is submitted, continuous integration jobs are run
-automatically to ensure the code builds and is relatively well-written. Checks
+automatically to ensure the code builds and is relatively well-written. The
-are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/) and
+jobs are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/),
-[CircleCI](https://circleci.com/gh/matrix-org/dendrite/). The Buildkite
+and the Buildkite pipeline configuration can be found in Matrix.org's
-pipeline can be found in Matrix.org's [pipelines
+[pipelines repository](https://github.com/matrix-org/pipelines).
-repository](https://github.com/matrix-org/pipelines).
 
 If a job fails, click the "details" button and you should be taken to the job's
 logs.
@@ -44,16 +43,20 @@ To save waiting for CI to finish after every commit, it is ideal to run the
 checks locally before pushing, fixing errors first. This also saves other
 people time as only so many PRs can be tested at a given time.
 
-To execute what Buildkite tests, simply run `./scripts/build-test-lint.sh`.
+To execute what Buildkite tests, first run `./scripts/build-test-lint.sh`;
-This script will build the code, lint it, and run `go test ./...` with race
+this script will build the code, lint it, and run `go test ./...` with race
 condition checking enabled. If something needs to be changed, fix it and then
 run the script again until it no longer complains. Be warned that the linting
 can take a significant amount of CPU and RAM.
 
-CircleCI simply runs [Sytest](https://github.com/matrix-org/sytest) with a test
+Once the code builds, run [Sytest](https://github.com/matrix-org/sytest)
-whitelist. See
+according to the guide in
 [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md#using-a-sytest-docker-image)
-for instructions on setting it up to run locally.
+so you can see whether something is being broken and whether there are newly
+passing tests.
+
+If these two steps report no problems, the code should be able to pass the CI
+tests.
 
 
 ## Picking Things To Do

README.md

@@ -1,4 +1,4 @@
-# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org)
+# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org)
 
 Dendrite will be a matrix homeserver written in go.
 

clientapi/clientapi.go

@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	"github.com/matrix-org/dendrite/common/basecomponent"
 	"github.com/matrix-org/dendrite/common/transactions"
+	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	typingServerAPI "github.com/matrix-org/dendrite/typingserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -43,6 +44,7 @@ func SetupClientAPIComponent(
 	typingInputAPI typingServerAPI.TypingServerInputAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	transactionsCache *transactions.Cache,
+	fedSenderAPI federationSenderAPI.FederationSenderQueryAPI,
 ) {
 	roomserverProducer := producers.NewRoomserverProducer(inputAPI)
 	typingProducer := producers.NewTypingServerProducer(typingInputAPI)
@@ -67,6 +69,6 @@ func SetupClientAPIComponent(
 	routing.Setup(
 		base.APIMux, *base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI,
 		accountsDB, deviceDB, federation, *keyRing, userUpdateProducer,
-		syncProducer, typingProducer, transactionsCache,
+		syncProducer, typingProducer, transactionsCache, fedSenderAPI,
 	)
 }

clientapi/routing/directory.go

@@ -22,12 +22,24 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/common/config"
+	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
+type roomDirectoryResponse struct {
+	RoomID  string   `json:"room_id"`
+	Servers []string `json:"servers"`
+}
+
+func (r *roomDirectoryResponse) fillServers(servers []gomatrixserverlib.ServerName) {
+	r.Servers = make([]string, len(servers))
+	for i, s := range servers {
+		r.Servers[i] = string(s)
+	}
+}
+
 // DirectoryRoom looks up a room alias
 func DirectoryRoom(
 	req *http.Request,
@@ -35,6 +47,7 @@ func DirectoryRoom(
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.Dendrite,
 	rsAPI roomserverAPI.RoomserverAliasAPI,
+	fedSenderAPI federationSenderAPI.FederationSenderQueryAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
 	if err != nil {
@@ -44,41 +57,32 @@ func DirectoryRoom(
 		}
 	}
 
-	if domain == cfg.Matrix.ServerName {
+	var res roomDirectoryResponse
-		// Query the roomserver API to check if the alias exists locally
+
+	// Query the roomserver API to check if the alias exists locally.
 	queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
 	var queryRes roomserverAPI.GetRoomIDForAliasResponse
 	if err = rsAPI.GetRoomIDForAlias(req.Context(), &queryReq, &queryRes); err != nil {
 		return httputil.LogThenError(req, err)
 	}
 
-		// List any roomIDs found associated with this alias
+	res.RoomID = queryRes.RoomID
-		if len(queryRes.RoomID) > 0 {
+
-			return util.JSONResponse{
+	if res.RoomID == "" {
-				Code: http.StatusOK,
+		// If we don't know it locally, do a federation query.
-				JSON: queryRes,
+		// But don't send the query to ourselves.
-			}
+		if domain != cfg.Matrix.ServerName {
-		}
+			fedRes, fedErr := federation.LookupRoomAlias(req.Context(), domain, roomAlias)
-	} else {
+			if fedErr != nil {
-		// Query the federation for this room alias
-		resp, err := federation.LookupRoomAlias(req.Context(), domain, roomAlias)
-		if err != nil {
-			switch err.(type) {
-			case gomatrix.HTTPError:
-			default:
 				// TODO: Return 502 if the remote server errored.
 				// TODO: Return 504 if the remote server timed out.
-				return httputil.LogThenError(req, err)
+				return httputil.LogThenError(req, fedErr)
-			}
-		}
-		if len(resp.RoomID) > 0 {
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: resp,
-			}
 			}
+			res.RoomID = fedRes.RoomID
+			res.fillServers(fedRes.Servers)
 		}
 
+		if res.RoomID == "" {
 			return util.JSONResponse{
 				Code: http.StatusNotFound,
 				JSON: jsonerror.NotFound(
@@ -86,6 +90,20 @@ func DirectoryRoom(
 				),
 			}
 		}
+	} else {
+		joinedHostsReq := federationSenderAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: res.RoomID}
+		var joinedHostsRes federationSenderAPI.QueryJoinedHostServerNamesInRoomResponse
+		if err = fedSenderAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &joinedHostsReq, &joinedHostsRes); err != nil {
+			return httputil.LogThenError(req, err)
+		}
+		res.fillServers(joinedHostsRes.ServerNames)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}
 
 // SetLocalAlias implements PUT /directory/room/{roomAlias}
 // TODO: Check if the user has the power level to set an alias

clientapi/routing/routing.go

@@ -30,6 +30,7 @@ import (
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/common/transactions"
+	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -59,6 +60,7 @@ func Setup(
 	syncProducer *producers.SyncAPIProducer,
 	typingProducer *producers.TypingServerProducer,
 	transactionsCache *transactions.Cache,
+	federationSender federationSenderAPI.FederationSenderQueryAPI,
 ) {
 
 	apiMux.Handle("/_matrix/client/versions",
@@ -207,7 +209,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI)
+			return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI, federationSender)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 

cmd/dendrite-client-api-server/main.go

@@ -37,11 +37,12 @@ func main() {
 
 	asQuery := base.CreateHTTPAppServiceAPIs()
 	alias, input, query := base.CreateHTTPRoomserverAPIs()
+	fedSenderAPI := base.CreateHTTPFederationSenderAPIs()
 	typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache())
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB, federation, &keyRing,
-		alias, input, query, typingInputAPI, asQuery, transactions.New(),
+		alias, input, query, typingInputAPI, asQuery, transactions.New(), fedSenderAPI,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Listen.ClientAPI))

cmd/dendrite-monolith-server/main.go

@@ -60,14 +60,14 @@ func main() {
 	asQuery := appservice.SetupAppServiceAPIComponent(
 		base, accountDB, deviceDB, federation, alias, query, transactions.New(),
 	)
+	fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query)
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB,
 		federation, &keyRing, alias, input, query,
-		typingInputAPI, asQuery, transactions.New(),
+		typingInputAPI, asQuery, transactions.New(), fedSenderAPI,
 	)
 	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery)
-	federationsender.SetupFederationSenderComponent(base, federation, query)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
 	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB)
 	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query)

common/basecomponent/base.go

@@ -32,6 +32,7 @@ import (
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/common/config"
+	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	typingServerAPI "github.com/matrix-org/dendrite/typingserver/api"
 	"github.com/sirupsen/logrus"
@@ -107,6 +108,12 @@ func (b *BaseDendrite) CreateHTTPTypingServerAPIs() typingServerAPI.TypingServer
 	return typingServerAPI.NewTypingServerInputAPIHTTP(b.Cfg.TypingServerURL(), nil)
 }
 
+// CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting
+// the federation sender over HTTP
+func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI {
+	return federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), nil)
+}
+
 // CreateDeviceDB creates a new instance of the device database. Should only be
 // called once per component.
 func (b *BaseDendrite) CreateDeviceDB() *devices.Database {

common/config/config.go

@@ -678,6 +678,15 @@ func (config *Dendrite) TypingServerURL() string {
 	return "http://" + string(config.Listen.TypingServer)
 }
 
+// FederationSenderURL returns an HTTP URL for where the federation sender is listening.
+func (config *Dendrite) FederationSenderURL() string {
+	// Hard code the typing server to talk HTTP for now.
+	// If we support HTTPS we need to think of a practical way to do certificate validation.
+	// People setting up servers shouldn't need to get a certificate valid for the public
+	// internet for an internal API.
+	return "http://" + string(config.Listen.FederationSender)
+}
+
 // SetupTracing configures the opentracing using the supplied configuration.
 func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) {
 	return config.Tracing.Jaeger.InitGlobalTracer(

testfile

@@ -172,6 +172,16 @@ Outbound federation can query profile data
 /event/ does not allow access to events before the user joined
 Federation key API allows unsigned requests for keys
 Can paginate public room list
+GET /directory/room/:room_alias yields room ID
+PUT /directory/room/:room_alias creates alias
+Room aliases can contain Unicode
+Creators can delete alias
+Alias creators can delete alias with no ops
+Alias creators can delete canonical alias with no ops
+Regular users cannot create room aliases within the AS namespace
+Deleting a non-existent alias should return a 404
+Users can't delete other's aliases
+Outbound federation can query room alias directory
 POST /rooms/:room_id/redact/:event_id as original message sender redacts message
 POST /rooms/:room_id/redact/:event_id as random user does not redact message
 POST /redact disallows redaction of event in different room