From 49fd47c86313f036da3a9549e7d14def166f4ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Behouba=20Manass=C3=A9?= Date: Mon, 30 Sep 2019 19:25:04 +0300 Subject: [PATCH 1/7] selectAccountDataByType return ClientEvent pointer instead of slice of ClientEvent (#798) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pull request is an attempt to fix #773. Signed-off-by: Kouame Behouba Manassé behouba@gmail.com --- .../storage/accounts/account_data_table.go | 25 ++++++------------- clientapi/auth/storage/accounts/storage.go | 4 +-- clientapi/routing/room_tagging.go | 14 +++++------ syncapi/sync/requestpool.go | 4 +-- 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/clientapi/auth/storage/accounts/account_data_table.go b/clientapi/auth/storage/accounts/account_data_table.go index 0d73cb312..0d6ad0933 100644 --- a/clientapi/auth/storage/accounts/account_data_table.go +++ b/clientapi/auth/storage/accounts/account_data_table.go @@ -120,28 +120,17 @@ func (s *accountDataStatements) selectAccountData( func (s *accountDataStatements) selectAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data []gomatrixserverlib.ClientEvent, err error) { - data = []gomatrixserverlib.ClientEvent{} - +) (data *gomatrixserverlib.ClientEvent, err error) { stmt := s.selectAccountDataByTypeStmt - rows, err := stmt.QueryContext(ctx, localpart, roomID, dataType) - if err != nil { + var content []byte + + if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { return } - for rows.Next() { - var content []byte - - if err = rows.Scan(&content); err != nil { - return - } - - ac := gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - - data = append(data, ac) + data = &gomatrixserverlib.ClientEvent{ + Type: dataType, + Content: content, } return diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 41d75daad..020a38376 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -263,11 +263,11 @@ func (d *Database) GetAccountData(ctx context.Context, localpart string) ( // GetAccountDataByType returns account data matching a given // localpart, room ID and type. -// If no account data could be found, returns an empty array +// If no account data could be found, returns nil // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data []gomatrixserverlib.ClientEvent, err error) { +) (data *gomatrixserverlib.ClientEvent, err error) { return d.accountDatas.selectAccountDataByType( ctx, localpart, roomID, dataType, ) diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index 6e7324cd8..487081c53 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -59,7 +59,7 @@ func GetTags( return httputil.LogThenError(req, err) } - if len(data) == 0 { + if data == nil { return util.JSONResponse{ Code: http.StatusOK, JSON: struct{}{}, @@ -68,7 +68,7 @@ func GetTags( return util.JSONResponse{ Code: http.StatusOK, - JSON: data[0].Content, + JSON: data.Content, } } @@ -103,8 +103,8 @@ func PutTag( } var tagContent gomatrix.TagContent - if len(data) > 0 { - if err = json.Unmarshal(data[0].Content, &tagContent); err != nil { + if data != nil { + if err = json.Unmarshal(data.Content, &tagContent); err != nil { return httputil.LogThenError(req, err) } } else { @@ -155,7 +155,7 @@ func DeleteTag( } // If there are no tags in the database, exit - if len(data) == 0 { + if data == nil { // Spec only defines 200 responses for this endpoint so we don't return anything else. return util.JSONResponse{ Code: http.StatusOK, @@ -164,7 +164,7 @@ func DeleteTag( } var tagContent gomatrix.TagContent - err = json.Unmarshal(data[0].Content, &tagContent) + err = json.Unmarshal(data.Content, &tagContent) if err != nil { return httputil.LogThenError(req, err) } @@ -204,7 +204,7 @@ func obtainSavedTags( userID string, roomID string, accountDB *accounts.Database, -) (string, []gomatrixserverlib.ClientEvent, error) { +) (string, *gomatrixserverlib.ClientEvent, error) { localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { return "", nil, err diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 6b95f4698..94a369001 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -196,13 +196,13 @@ func (rp *RequestPool) appendAccountData( events := []gomatrixserverlib.ClientEvent{} // Request the missing data from the database for _, dataType := range dataTypes { - evs, err := rp.accountDB.GetAccountDataByType( + event, err := rp.accountDB.GetAccountDataByType( req.ctx, localpart, roomID, dataType, ) if err != nil { return nil, err } - events = append(events, evs...) + events = append(events, *event) } // Append the data to the response From e239fb10f36662d38ab9d1cad5a0bc65518c1704 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 2 Oct 2019 00:09:47 +0800 Subject: [PATCH 2/7] Add missing servers field in /directory/room/:alias response (#732) --- clientapi/clientapi.go | 4 +- clientapi/routing/directory.go | 80 ++++++++++++++++---------- clientapi/routing/routing.go | 4 +- cmd/dendrite-client-api-server/main.go | 3 +- cmd/dendrite-monolith-server/main.go | 4 +- common/basecomponent/base.go | 7 +++ common/config/config.go | 9 +++ testfile | 10 ++++ 8 files changed, 85 insertions(+), 36 deletions(-) diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 5b6e21c80..f3f3e08cf 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/routing" "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/transactions" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" typingServerAPI "github.com/matrix-org/dendrite/typingserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -43,6 +44,7 @@ func SetupClientAPIComponent( typingInputAPI typingServerAPI.TypingServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, + fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI) typingProducer := producers.NewTypingServerProducer(typingInputAPI) @@ -67,6 +69,6 @@ func SetupClientAPIComponent( routing.Setup( base.APIMux, *base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, - syncProducer, typingProducer, transactionsCache, + syncProducer, typingProducer, transactionsCache, fedSenderAPI, ) } diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 0d91d0426..574b275d6 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -22,12 +22,24 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) +type roomDirectoryResponse struct { + RoomID string `json:"room_id"` + Servers []string `json:"servers"` +} + +func (r *roomDirectoryResponse) fillServers(servers []gomatrixserverlib.ServerName) { + r.Servers = make([]string, len(servers)) + for i, s := range servers { + r.Servers[i] = string(s) + } +} + // DirectoryRoom looks up a room alias func DirectoryRoom( req *http.Request, @@ -35,6 +47,7 @@ func DirectoryRoom( federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverAliasAPI, + fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) if err != nil { @@ -44,46 +57,51 @@ func DirectoryRoom( } } - if domain == cfg.Matrix.ServerName { - // Query the roomserver API to check if the alias exists locally - queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} - var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = rsAPI.GetRoomIDForAlias(req.Context(), &queryReq, &queryRes); err != nil { - return httputil.LogThenError(req, err) + var res roomDirectoryResponse + + // Query the roomserver API to check if the alias exists locally. + queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} + var queryRes roomserverAPI.GetRoomIDForAliasResponse + if err = rsAPI.GetRoomIDForAlias(req.Context(), &queryReq, &queryRes); err != nil { + return httputil.LogThenError(req, err) + } + + res.RoomID = queryRes.RoomID + + if res.RoomID == "" { + // If we don't know it locally, do a federation query. + // But don't send the query to ourselves. + if domain != cfg.Matrix.ServerName { + fedRes, fedErr := federation.LookupRoomAlias(req.Context(), domain, roomAlias) + if fedErr != nil { + // TODO: Return 502 if the remote server errored. + // TODO: Return 504 if the remote server timed out. + return httputil.LogThenError(req, fedErr) + } + res.RoomID = fedRes.RoomID + res.fillServers(fedRes.Servers) } - // List any roomIDs found associated with this alias - if len(queryRes.RoomID) > 0 { + if res.RoomID == "" { return util.JSONResponse{ - Code: http.StatusOK, - JSON: queryRes, + Code: http.StatusNotFound, + JSON: jsonerror.NotFound( + fmt.Sprintf("Room alias %s not found", roomAlias), + ), } } } else { - // Query the federation for this room alias - resp, err := federation.LookupRoomAlias(req.Context(), domain, roomAlias) - if err != nil { - switch err.(type) { - case gomatrix.HTTPError: - default: - // TODO: Return 502 if the remote server errored. - // TODO: Return 504 if the remote server timed out. - return httputil.LogThenError(req, err) - } - } - if len(resp.RoomID) > 0 { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: resp, - } + joinedHostsReq := federationSenderAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: res.RoomID} + var joinedHostsRes federationSenderAPI.QueryJoinedHostServerNamesInRoomResponse + if err = fedSenderAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &joinedHostsReq, &joinedHostsRes); err != nil { + return httputil.LogThenError(req, err) } + res.fillServers(joinedHostsRes.ServerNames) } return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound( - fmt.Sprintf("Room alias %s not found", roomAlias), - ), + Code: http.StatusOK, + JSON: res, } } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index d4b323a2d..4a36661db 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/common/transactions" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -59,6 +60,7 @@ func Setup( syncProducer *producers.SyncAPIProducer, typingProducer *producers.TypingServerProducer, transactionsCache *transactions.Cache, + federationSender federationSenderAPI.FederationSenderQueryAPI, ) { apiMux.Handle("/_matrix/client/versions", @@ -185,7 +187,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI) + return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI, federationSender) }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index dd0656441..e273ecadb 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -37,11 +37,12 @@ func main() { asQuery := base.CreateHTTPAppServiceAPIs() alias, input, query := base.CreateHTTPRoomserverAPIs() + fedSenderAPI := base.CreateHTTPFederationSenderAPIs() typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, - alias, input, query, typingInputAPI, asQuery, transactions.New(), + alias, input, query, typingInputAPI, asQuery, transactions.New(), fedSenderAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 87a625b8b..0a320616e 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -60,14 +60,14 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, alias, input, query, - typingInputAPI, asQuery, transactions.New(), + typingInputAPI, asQuery, transactions.New(), fedSenderAPI, ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery) - federationsender.SetupFederationSenderComponent(base, federation, query) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query) diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 6a20aca3b..503134b23 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -32,6 +32,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/common/config" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" typingServerAPI "github.com/matrix-org/dendrite/typingserver/api" "github.com/sirupsen/logrus" @@ -107,6 +108,12 @@ func (b *BaseDendrite) CreateHTTPTypingServerAPIs() typingServerAPI.TypingServer return typingServerAPI.NewTypingServerInputAPIHTTP(b.Cfg.TypingServerURL(), nil) } +// CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting +// the federation sender over HTTP +func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI { + return federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), nil) +} + // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() *devices.Database { diff --git a/common/config/config.go b/common/config/config.go index 40232fb03..16cf26408 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -678,6 +678,15 @@ func (config *Dendrite) TypingServerURL() string { return "http://" + string(config.Listen.TypingServer) } +// FederationSenderURL returns an HTTP URL for where the federation sender is listening. +func (config *Dendrite) FederationSenderURL() string { + // Hard code the typing server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.FederationSender) +} + // SetupTracing configures the opentracing using the supplied configuration. func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { return config.Tracing.Jaeger.InitGlobalTracer( diff --git a/testfile b/testfile index 38506d717..5ad3426e9 100644 --- a/testfile +++ b/testfile @@ -172,3 +172,13 @@ Outbound federation can query profile data /event/ does not allow access to events before the user joined Federation key API allows unsigned requests for keys Can paginate public room list +GET /directory/room/:room_alias yields room ID +PUT /directory/room/:room_alias creates alias +Room aliases can contain Unicode +Creators can delete alias +Alias creators can delete alias with no ops +Alias creators can delete canonical alias with no ops +Regular users cannot create room aliases within the AS namespace +Deleting a non-existent alias should return a 404 +Users can't delete other's aliases +Outbound federation can query room alias directory From 3e2bb8bf2fac3173f45a98370eebfc266f55c958 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 2 Oct 2019 11:01:52 +0800 Subject: [PATCH 3/7] Clean up CircleCI leftovers (#801) --- CONTRIBUTING.md | 23 +++++++++++++---------- README.md | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa815f15b..4d413a29c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,11 +23,10 @@ run](scripts/build-test-lint.sh). ## Continuous Integration When a Pull Request is submitted, continuous integration jobs are run -automatically to ensure the code builds and is relatively well-written. Checks -are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/) and -[CircleCI](https://circleci.com/gh/matrix-org/dendrite/). The Buildkite -pipeline can be found in Matrix.org's [pipelines -repository](https://github.com/matrix-org/pipelines). +automatically to ensure the code builds and is relatively well-written. The +jobs are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/), +and the Buildkite pipeline configuration can be found in Matrix.org's +[pipelines repository](https://github.com/matrix-org/pipelines). If a job fails, click the "details" button and you should be taken to the job's logs. @@ -44,16 +43,20 @@ To save waiting for CI to finish after every commit, it is ideal to run the checks locally before pushing, fixing errors first. This also saves other people time as only so many PRs can be tested at a given time. -To execute what Buildkite tests, simply run `./scripts/build-test-lint.sh`. -This script will build the code, lint it, and run `go test ./...` with race +To execute what Buildkite tests, first run `./scripts/build-test-lint.sh`; +this script will build the code, lint it, and run `go test ./...` with race condition checking enabled. If something needs to be changed, fix it and then run the script again until it no longer complains. Be warned that the linting can take a significant amount of CPU and RAM. -CircleCI simply runs [Sytest](https://github.com/matrix-org/sytest) with a test -whitelist. See +Once the code builds, run [Sytest](https://github.com/matrix-org/sytest) +according to the guide in [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md#using-a-sytest-docker-image) -for instructions on setting it up to run locally. +so you can see whether something is being broken and whether there are newly +passing tests. + +If these two steps report no problems, the code should be able to pass the CI +tests. ## Picking Things To Do diff --git a/README.md b/README.md index 4e628c0ff..2dadb1f4f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) +# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) Dendrite will be a matrix homeserver written in go. From 7d77538ca4dad73594bc20d11b7f85dc0693d7a5 Mon Sep 17 00:00:00 2001 From: aditsachde <23707194+aditsachde@users.noreply.github.com> Date: Wed, 2 Oct 2019 05:29:27 -0400 Subject: [PATCH 4/7] patch dendrite microservices with bind config (#795) This PR adds a block in the dendrite config for the services to bind to. The microservices should bind to the addresses in the bind block, and will be contacted at the address in the listen block. This fixes an issue with the microservices and kubernetes services. --- cmd/dendrite-appservice-server/main.go | 3 ++- cmd/dendrite-client-api-server/main.go | 3 ++- cmd/dendrite-federation-api-server/main.go | 3 ++- cmd/dendrite-federation-sender-server/main.go | 3 ++- cmd/dendrite-media-api-server/main.go | 3 ++- cmd/dendrite-public-rooms-api-server/main.go | 3 ++- cmd/dendrite-room-server/main.go | 3 ++- cmd/dendrite-sync-api-server/main.go | 3 ++- cmd/dendrite-typing-server/main.go | 3 ++- common/basecomponent/base.go | 11 ++++++++++- common/config/config.go | 14 ++++++++++++++ common/test/config.go | 10 ++++++++++ 12 files changed, 52 insertions(+), 10 deletions(-) diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index dcaea5138..f203969f4 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -35,5 +35,6 @@ func main() { base, accountDB, deviceDB, federation, alias, query, cache, ) - base.SetupAndServeHTTP(string(base.Cfg.Listen.FederationSender)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) + } diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index e273ecadb..2bde0f4cf 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -45,5 +45,6 @@ func main() { alias, input, query, typingInputAPI, asQuery, transactions.New(), fedSenderAPI, ) - base.SetupAndServeHTTP(string(base.Cfg.Listen.ClientAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) + } diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 014ed3343..c83845d25 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -39,5 +39,6 @@ func main() { alias, input, query, asQuery, ) - base.SetupAndServeHTTP(string(base.Cfg.Listen.FederationAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) + } diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 59b98e5bb..71fc0b015 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -32,5 +32,6 @@ func main() { base, federation, query, ) - base.SetupAndServeHTTP(string(base.Cfg.Listen.FederationSender)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) + } diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 718bb6f1b..a818db73a 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -28,5 +28,6 @@ func main() { mediaapi.SetupMediaAPIComponent(base, deviceDB) - base.SetupAndServeHTTP(string(base.Cfg.Listen.MediaAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI)) + } diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 63e1f40b5..b60eed92a 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,5 +28,6 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB) - base.SetupAndServeHTTP(string(base.Cfg.Listen.PublicRoomsAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) + } diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index a5942544d..41b705755 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -28,5 +28,6 @@ func main() { roomserver.SetupRoomServerComponent(base) - base.SetupAndServeHTTP(string(base.Cfg.Listen.RoomServer)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) + } diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 343d3567d..1c47ec525 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -31,5 +31,6 @@ func main() { syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query) - base.SetupAndServeHTTP(string(base.Cfg.Listen.SyncAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) + } diff --git a/cmd/dendrite-typing-server/main.go b/cmd/dendrite-typing-server/main.go index 4eb0823a9..461eb7144 100644 --- a/cmd/dendrite-typing-server/main.go +++ b/cmd/dendrite-typing-server/main.go @@ -32,5 +32,6 @@ func main() { typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) - base.SetupAndServeHTTP(string(base.Cfg.Listen.TypingServer)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.TypingServer), string(base.Cfg.Listen.TypingServer)) + } diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 503134b23..b05ec42db 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -157,7 +157,16 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on // ApiMux under /api/ and adds a prometheus handler under /metrics. -func (b *BaseDendrite) SetupAndServeHTTP(addr string) { +func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { + // If a separate bind address is defined, listen on that. Otherwise use + // the listen address + var addr string + if bindaddr != "" { + addr = bindaddr + } else { + addr = listenaddr + } + common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(b.APIMux)) logrus.Infof("Starting %s server on %s", b.componentName, addr) diff --git a/common/config/config.go b/common/config/config.go index 16cf26408..0332d0358 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -196,6 +196,20 @@ type Dendrite struct { // The internal addresses the components will listen on. // These should not be exposed externally as they expose metrics and debugging APIs. + // Falls back to addresses listed in Listen if not specified + Bind struct { + MediaAPI Address `yaml:"media_api"` + ClientAPI Address `yaml:"client_api"` + FederationAPI Address `yaml:"federation_api"` + AppServiceAPI Address `yaml:"appservice_api"` + SyncAPI Address `yaml:"sync_api"` + RoomServer Address `yaml:"room_server"` + FederationSender Address `yaml:"federation_sender"` + PublicRoomsAPI Address `yaml:"public_rooms_api"` + TypingServer Address `yaml:"typing_server"` + } `yaml:"bind"` + + // The addresses for talking to other microservices. Listen struct { MediaAPI Address `yaml:"media_api"` ClientAPI Address `yaml:"client_api"` diff --git a/common/test/config.go b/common/test/config.go index 08a1b398f..668e9a26e 100644 --- a/common/test/config.go +++ b/common/test/config.go @@ -106,6 +106,16 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Listen.PublicRoomsAPI = assignAddress() cfg.Listen.TypingServer = assignAddress() + // Bind to the same address as the listen address + // All microservices are run on the same host in testing + cfg.Bind.ClientAPI = cfg.Listen.ClientAPI + cfg.Bind.FederationAPI = cfg.Listen.FederationAPI + cfg.Bind.MediaAPI = cfg.Listen.MediaAPI + cfg.Bind.RoomServer = cfg.Listen.RoomServer + cfg.Bind.SyncAPI = cfg.Listen.SyncAPI + cfg.Bind.PublicRoomsAPI = cfg.Listen.PublicRoomsAPI + cfg.Bind.TypingServer = cfg.Listen.TypingServer + return &cfg, port, nil } From 9a56cf8439304540d0cdae18b30df82962c065cb Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 4 Oct 2019 17:08:00 +0100 Subject: [PATCH 5/7] Add AppService to test config and bind addrs (#800) Because our unit tests are often forgotten about :/ --- common/test/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/test/config.go b/common/test/config.go index 668e9a26e..693555619 100644 --- a/common/test/config.go +++ b/common/test/config.go @@ -91,6 +91,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con // the table names are globally unique. But we might not want to // rely on that in the future. cfg.Database.Account = config.DataSource(database) + cfg.Database.AppService = config.DataSource(database) cfg.Database.Device = config.DataSource(database) cfg.Database.MediaAPI = config.DataSource(database) cfg.Database.RoomServer = config.DataSource(database) @@ -99,6 +100,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Database.PublicRoomsAPI = config.DataSource(database) cfg.Listen.ClientAPI = assignAddress() + cfg.Listen.AppServiceAPI = assignAddress() cfg.Listen.FederationAPI = assignAddress() cfg.Listen.MediaAPI = assignAddress() cfg.Listen.RoomServer = assignAddress() From e959927d0ad20f53c51944abb751b7d40fad6aea Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Mon, 7 Oct 2019 20:15:58 +0800 Subject: [PATCH 6/7] selectAccountDataByType shouldn't error when no rows (#804) Signed-off-by: Alex Chen --- clientapi/auth/storage/accounts/account_data_table.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clientapi/auth/storage/accounts/account_data_table.go b/clientapi/auth/storage/accounts/account_data_table.go index 0d6ad0933..080ca3f38 100644 --- a/clientapi/auth/storage/accounts/account_data_table.go +++ b/clientapi/auth/storage/accounts/account_data_table.go @@ -125,6 +125,10 @@ func (s *accountDataStatements) selectAccountDataByType( var content []byte if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return } From 145921f2071660af7d8e1ea2ea015949a0b03c59 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 8 Oct 2019 12:20:37 +0100 Subject: [PATCH 7/7] Pin golangci-lint version to non-broken one (#809) --- scripts/find-lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh index 25b311f94..ca991b62c 100755 --- a/scripts/find-lint.sh +++ b/scripts/find-lint.sh @@ -27,7 +27,7 @@ echo "Installing golangci-lint..." # TODO: Once go 1.13 is out, use go get's -mod=readonly option # https://github.com/golang/go/issues/30667 cp go.mod go.mod.bak && cp go.sum go.sum.bak -go get github.com/golangci/golangci-lint/cmd/golangci-lint +go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1 echo "Looking for lint..." golangci-lint run $args