From 49fd47c86313f036da3a9549e7d14def166f4ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Behouba=20Manass=C3=A9?= Date: Mon, 30 Sep 2019 19:25:04 +0300 Subject: [PATCH 01/19] selectAccountDataByType return ClientEvent pointer instead of slice of ClientEvent (#798) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pull request is an attempt to fix #773. Signed-off-by: Kouame Behouba Manassé behouba@gmail.com --- .../storage/accounts/account_data_table.go | 25 ++++++------------- clientapi/auth/storage/accounts/storage.go | 4 +-- clientapi/routing/room_tagging.go | 14 +++++------ syncapi/sync/requestpool.go | 4 +-- 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/clientapi/auth/storage/accounts/account_data_table.go b/clientapi/auth/storage/accounts/account_data_table.go index 0d73cb312..0d6ad0933 100644 --- a/clientapi/auth/storage/accounts/account_data_table.go +++ b/clientapi/auth/storage/accounts/account_data_table.go @@ -120,28 +120,17 @@ func (s *accountDataStatements) selectAccountData( func (s *accountDataStatements) selectAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data []gomatrixserverlib.ClientEvent, err error) { - data = []gomatrixserverlib.ClientEvent{} - +) (data *gomatrixserverlib.ClientEvent, err error) { stmt := s.selectAccountDataByTypeStmt - rows, err := stmt.QueryContext(ctx, localpart, roomID, dataType) - if err != nil { + var content []byte + + if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { return } - for rows.Next() { - var content []byte - - if err = rows.Scan(&content); err != nil { - return - } - - ac := gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - - data = append(data, ac) + data = &gomatrixserverlib.ClientEvent{ + Type: dataType, + Content: content, } return diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 41d75daad..020a38376 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -263,11 +263,11 @@ func (d *Database) GetAccountData(ctx context.Context, localpart string) ( // GetAccountDataByType returns account data matching a given // localpart, room ID and type. -// If no account data could be found, returns an empty array +// If no account data could be found, returns nil // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data []gomatrixserverlib.ClientEvent, err error) { +) (data *gomatrixserverlib.ClientEvent, err error) { return d.accountDatas.selectAccountDataByType( ctx, localpart, roomID, dataType, ) diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index 6e7324cd8..487081c53 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -59,7 +59,7 @@ func GetTags( return httputil.LogThenError(req, err) } - if len(data) == 0 { + if data == nil { return util.JSONResponse{ Code: http.StatusOK, JSON: struct{}{}, @@ -68,7 +68,7 @@ func GetTags( return util.JSONResponse{ Code: http.StatusOK, - JSON: data[0].Content, + JSON: data.Content, } } @@ -103,8 +103,8 @@ func PutTag( } var tagContent gomatrix.TagContent - if len(data) > 0 { - if err = json.Unmarshal(data[0].Content, &tagContent); err != nil { + if data != nil { + if err = json.Unmarshal(data.Content, &tagContent); err != nil { return httputil.LogThenError(req, err) } } else { @@ -155,7 +155,7 @@ func DeleteTag( } // If there are no tags in the database, exit - if len(data) == 0 { + if data == nil { // Spec only defines 200 responses for this endpoint so we don't return anything else. return util.JSONResponse{ Code: http.StatusOK, @@ -164,7 +164,7 @@ func DeleteTag( } var tagContent gomatrix.TagContent - err = json.Unmarshal(data[0].Content, &tagContent) + err = json.Unmarshal(data.Content, &tagContent) if err != nil { return httputil.LogThenError(req, err) } @@ -204,7 +204,7 @@ func obtainSavedTags( userID string, roomID string, accountDB *accounts.Database, -) (string, []gomatrixserverlib.ClientEvent, error) { +) (string, *gomatrixserverlib.ClientEvent, error) { localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { return "", nil, err diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 6b95f4698..94a369001 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -196,13 +196,13 @@ func (rp *RequestPool) appendAccountData( events := []gomatrixserverlib.ClientEvent{} // Request the missing data from the database for _, dataType := range dataTypes { - evs, err := rp.accountDB.GetAccountDataByType( + event, err := rp.accountDB.GetAccountDataByType( req.ctx, localpart, roomID, dataType, ) if err != nil { return nil, err } - events = append(events, evs...) + events = append(events, *event) } // Append the data to the response From e239fb10f36662d38ab9d1cad5a0bc65518c1704 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 2 Oct 2019 00:09:47 +0800 Subject: [PATCH 02/19] Add missing servers field in /directory/room/:alias response (#732) --- clientapi/clientapi.go | 4 +- clientapi/routing/directory.go | 80 ++++++++++++++++---------- clientapi/routing/routing.go | 4 +- cmd/dendrite-client-api-server/main.go | 3 +- cmd/dendrite-monolith-server/main.go | 4 +- common/basecomponent/base.go | 7 +++ common/config/config.go | 9 +++ testfile | 10 ++++ 8 files changed, 85 insertions(+), 36 deletions(-) diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 5b6e21c80..f3f3e08cf 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/routing" "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/transactions" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" typingServerAPI "github.com/matrix-org/dendrite/typingserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -43,6 +44,7 @@ func SetupClientAPIComponent( typingInputAPI typingServerAPI.TypingServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, + fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI) typingProducer := producers.NewTypingServerProducer(typingInputAPI) @@ -67,6 +69,6 @@ func SetupClientAPIComponent( routing.Setup( base.APIMux, *base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, - syncProducer, typingProducer, transactionsCache, + syncProducer, typingProducer, transactionsCache, fedSenderAPI, ) } diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 0d91d0426..574b275d6 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -22,12 +22,24 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) +type roomDirectoryResponse struct { + RoomID string `json:"room_id"` + Servers []string `json:"servers"` +} + +func (r *roomDirectoryResponse) fillServers(servers []gomatrixserverlib.ServerName) { + r.Servers = make([]string, len(servers)) + for i, s := range servers { + r.Servers[i] = string(s) + } +} + // DirectoryRoom looks up a room alias func DirectoryRoom( req *http.Request, @@ -35,6 +47,7 @@ func DirectoryRoom( federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverAliasAPI, + fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) if err != nil { @@ -44,46 +57,51 @@ func DirectoryRoom( } } - if domain == cfg.Matrix.ServerName { - // Query the roomserver API to check if the alias exists locally - queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} - var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = rsAPI.GetRoomIDForAlias(req.Context(), &queryReq, &queryRes); err != nil { - return httputil.LogThenError(req, err) + var res roomDirectoryResponse + + // Query the roomserver API to check if the alias exists locally. + queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} + var queryRes roomserverAPI.GetRoomIDForAliasResponse + if err = rsAPI.GetRoomIDForAlias(req.Context(), &queryReq, &queryRes); err != nil { + return httputil.LogThenError(req, err) + } + + res.RoomID = queryRes.RoomID + + if res.RoomID == "" { + // If we don't know it locally, do a federation query. + // But don't send the query to ourselves. + if domain != cfg.Matrix.ServerName { + fedRes, fedErr := federation.LookupRoomAlias(req.Context(), domain, roomAlias) + if fedErr != nil { + // TODO: Return 502 if the remote server errored. + // TODO: Return 504 if the remote server timed out. + return httputil.LogThenError(req, fedErr) + } + res.RoomID = fedRes.RoomID + res.fillServers(fedRes.Servers) } - // List any roomIDs found associated with this alias - if len(queryRes.RoomID) > 0 { + if res.RoomID == "" { return util.JSONResponse{ - Code: http.StatusOK, - JSON: queryRes, + Code: http.StatusNotFound, + JSON: jsonerror.NotFound( + fmt.Sprintf("Room alias %s not found", roomAlias), + ), } } } else { - // Query the federation for this room alias - resp, err := federation.LookupRoomAlias(req.Context(), domain, roomAlias) - if err != nil { - switch err.(type) { - case gomatrix.HTTPError: - default: - // TODO: Return 502 if the remote server errored. - // TODO: Return 504 if the remote server timed out. - return httputil.LogThenError(req, err) - } - } - if len(resp.RoomID) > 0 { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: resp, - } + joinedHostsReq := federationSenderAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: res.RoomID} + var joinedHostsRes federationSenderAPI.QueryJoinedHostServerNamesInRoomResponse + if err = fedSenderAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &joinedHostsReq, &joinedHostsRes); err != nil { + return httputil.LogThenError(req, err) } + res.fillServers(joinedHostsRes.ServerNames) } return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound( - fmt.Sprintf("Room alias %s not found", roomAlias), - ), + Code: http.StatusOK, + JSON: res, } } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index d4b323a2d..4a36661db 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/common/transactions" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -59,6 +60,7 @@ func Setup( syncProducer *producers.SyncAPIProducer, typingProducer *producers.TypingServerProducer, transactionsCache *transactions.Cache, + federationSender federationSenderAPI.FederationSenderQueryAPI, ) { apiMux.Handle("/_matrix/client/versions", @@ -185,7 +187,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI) + return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI, federationSender) }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index dd0656441..e273ecadb 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -37,11 +37,12 @@ func main() { asQuery := base.CreateHTTPAppServiceAPIs() alias, input, query := base.CreateHTTPRoomserverAPIs() + fedSenderAPI := base.CreateHTTPFederationSenderAPIs() typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, - alias, input, query, typingInputAPI, asQuery, transactions.New(), + alias, input, query, typingInputAPI, asQuery, transactions.New(), fedSenderAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 87a625b8b..0a320616e 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -60,14 +60,14 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, alias, input, query, - typingInputAPI, asQuery, transactions.New(), + typingInputAPI, asQuery, transactions.New(), fedSenderAPI, ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery) - federationsender.SetupFederationSenderComponent(base, federation, query) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query) diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 6a20aca3b..503134b23 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -32,6 +32,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/common/config" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" typingServerAPI "github.com/matrix-org/dendrite/typingserver/api" "github.com/sirupsen/logrus" @@ -107,6 +108,12 @@ func (b *BaseDendrite) CreateHTTPTypingServerAPIs() typingServerAPI.TypingServer return typingServerAPI.NewTypingServerInputAPIHTTP(b.Cfg.TypingServerURL(), nil) } +// CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting +// the federation sender over HTTP +func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI { + return federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), nil) +} + // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() *devices.Database { diff --git a/common/config/config.go b/common/config/config.go index 40232fb03..16cf26408 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -678,6 +678,15 @@ func (config *Dendrite) TypingServerURL() string { return "http://" + string(config.Listen.TypingServer) } +// FederationSenderURL returns an HTTP URL for where the federation sender is listening. +func (config *Dendrite) FederationSenderURL() string { + // Hard code the typing server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.FederationSender) +} + // SetupTracing configures the opentracing using the supplied configuration. func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { return config.Tracing.Jaeger.InitGlobalTracer( diff --git a/testfile b/testfile index 38506d717..5ad3426e9 100644 --- a/testfile +++ b/testfile @@ -172,3 +172,13 @@ Outbound federation can query profile data /event/ does not allow access to events before the user joined Federation key API allows unsigned requests for keys Can paginate public room list +GET /directory/room/:room_alias yields room ID +PUT /directory/room/:room_alias creates alias +Room aliases can contain Unicode +Creators can delete alias +Alias creators can delete alias with no ops +Alias creators can delete canonical alias with no ops +Regular users cannot create room aliases within the AS namespace +Deleting a non-existent alias should return a 404 +Users can't delete other's aliases +Outbound federation can query room alias directory From 3e2bb8bf2fac3173f45a98370eebfc266f55c958 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 2 Oct 2019 11:01:52 +0800 Subject: [PATCH 03/19] Clean up CircleCI leftovers (#801) --- CONTRIBUTING.md | 23 +++++++++++++---------- README.md | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa815f15b..4d413a29c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,11 +23,10 @@ run](scripts/build-test-lint.sh). ## Continuous Integration When a Pull Request is submitted, continuous integration jobs are run -automatically to ensure the code builds and is relatively well-written. Checks -are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/) and -[CircleCI](https://circleci.com/gh/matrix-org/dendrite/). The Buildkite -pipeline can be found in Matrix.org's [pipelines -repository](https://github.com/matrix-org/pipelines). +automatically to ensure the code builds and is relatively well-written. The +jobs are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/), +and the Buildkite pipeline configuration can be found in Matrix.org's +[pipelines repository](https://github.com/matrix-org/pipelines). If a job fails, click the "details" button and you should be taken to the job's logs. @@ -44,16 +43,20 @@ To save waiting for CI to finish after every commit, it is ideal to run the checks locally before pushing, fixing errors first. This also saves other people time as only so many PRs can be tested at a given time. -To execute what Buildkite tests, simply run `./scripts/build-test-lint.sh`. -This script will build the code, lint it, and run `go test ./...` with race +To execute what Buildkite tests, first run `./scripts/build-test-lint.sh`; +this script will build the code, lint it, and run `go test ./...` with race condition checking enabled. If something needs to be changed, fix it and then run the script again until it no longer complains. Be warned that the linting can take a significant amount of CPU and RAM. -CircleCI simply runs [Sytest](https://github.com/matrix-org/sytest) with a test -whitelist. See +Once the code builds, run [Sytest](https://github.com/matrix-org/sytest) +according to the guide in [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md#using-a-sytest-docker-image) -for instructions on setting it up to run locally. +so you can see whether something is being broken and whether there are newly +passing tests. + +If these two steps report no problems, the code should be able to pass the CI +tests. ## Picking Things To Do diff --git a/README.md b/README.md index 4e628c0ff..2dadb1f4f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) +# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) Dendrite will be a matrix homeserver written in go. From 7d77538ca4dad73594bc20d11b7f85dc0693d7a5 Mon Sep 17 00:00:00 2001 From: aditsachde <23707194+aditsachde@users.noreply.github.com> Date: Wed, 2 Oct 2019 05:29:27 -0400 Subject: [PATCH 04/19] patch dendrite microservices with bind config (#795) This PR adds a block in the dendrite config for the services to bind to. The microservices should bind to the addresses in the bind block, and will be contacted at the address in the listen block. This fixes an issue with the microservices and kubernetes services. --- cmd/dendrite-appservice-server/main.go | 3 ++- cmd/dendrite-client-api-server/main.go | 3 ++- cmd/dendrite-federation-api-server/main.go | 3 ++- cmd/dendrite-federation-sender-server/main.go | 3 ++- cmd/dendrite-media-api-server/main.go | 3 ++- cmd/dendrite-public-rooms-api-server/main.go | 3 ++- cmd/dendrite-room-server/main.go | 3 ++- cmd/dendrite-sync-api-server/main.go | 3 ++- cmd/dendrite-typing-server/main.go | 3 ++- common/basecomponent/base.go | 11 ++++++++++- common/config/config.go | 14 ++++++++++++++ common/test/config.go | 10 ++++++++++ 12 files changed, 52 insertions(+), 10 deletions(-) diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index dcaea5138..f203969f4 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -35,5 +35,6 @@ func main() { base, accountDB, deviceDB, federation, alias, query, cache, ) - base.SetupAndServeHTTP(string(base.Cfg.Listen.FederationSender)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) + } diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index e273ecadb..2bde0f4cf 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -45,5 +45,6 @@ func main() { alias, input, query, typingInputAPI, asQuery, transactions.New(), fedSenderAPI, ) - base.SetupAndServeHTTP(string(base.Cfg.Listen.ClientAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) + } diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 014ed3343..c83845d25 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -39,5 +39,6 @@ func main() { alias, input, query, asQuery, ) - base.SetupAndServeHTTP(string(base.Cfg.Listen.FederationAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) + } diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 59b98e5bb..71fc0b015 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -32,5 +32,6 @@ func main() { base, federation, query, ) - base.SetupAndServeHTTP(string(base.Cfg.Listen.FederationSender)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) + } diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 718bb6f1b..a818db73a 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -28,5 +28,6 @@ func main() { mediaapi.SetupMediaAPIComponent(base, deviceDB) - base.SetupAndServeHTTP(string(base.Cfg.Listen.MediaAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI)) + } diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 63e1f40b5..b60eed92a 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,5 +28,6 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB) - base.SetupAndServeHTTP(string(base.Cfg.Listen.PublicRoomsAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) + } diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index a5942544d..41b705755 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -28,5 +28,6 @@ func main() { roomserver.SetupRoomServerComponent(base) - base.SetupAndServeHTTP(string(base.Cfg.Listen.RoomServer)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) + } diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 343d3567d..1c47ec525 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -31,5 +31,6 @@ func main() { syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query) - base.SetupAndServeHTTP(string(base.Cfg.Listen.SyncAPI)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) + } diff --git a/cmd/dendrite-typing-server/main.go b/cmd/dendrite-typing-server/main.go index 4eb0823a9..461eb7144 100644 --- a/cmd/dendrite-typing-server/main.go +++ b/cmd/dendrite-typing-server/main.go @@ -32,5 +32,6 @@ func main() { typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) - base.SetupAndServeHTTP(string(base.Cfg.Listen.TypingServer)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.TypingServer), string(base.Cfg.Listen.TypingServer)) + } diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 503134b23..b05ec42db 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -157,7 +157,16 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on // ApiMux under /api/ and adds a prometheus handler under /metrics. -func (b *BaseDendrite) SetupAndServeHTTP(addr string) { +func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { + // If a separate bind address is defined, listen on that. Otherwise use + // the listen address + var addr string + if bindaddr != "" { + addr = bindaddr + } else { + addr = listenaddr + } + common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(b.APIMux)) logrus.Infof("Starting %s server on %s", b.componentName, addr) diff --git a/common/config/config.go b/common/config/config.go index 16cf26408..0332d0358 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -196,6 +196,20 @@ type Dendrite struct { // The internal addresses the components will listen on. // These should not be exposed externally as they expose metrics and debugging APIs. + // Falls back to addresses listed in Listen if not specified + Bind struct { + MediaAPI Address `yaml:"media_api"` + ClientAPI Address `yaml:"client_api"` + FederationAPI Address `yaml:"federation_api"` + AppServiceAPI Address `yaml:"appservice_api"` + SyncAPI Address `yaml:"sync_api"` + RoomServer Address `yaml:"room_server"` + FederationSender Address `yaml:"federation_sender"` + PublicRoomsAPI Address `yaml:"public_rooms_api"` + TypingServer Address `yaml:"typing_server"` + } `yaml:"bind"` + + // The addresses for talking to other microservices. Listen struct { MediaAPI Address `yaml:"media_api"` ClientAPI Address `yaml:"client_api"` diff --git a/common/test/config.go b/common/test/config.go index 08a1b398f..668e9a26e 100644 --- a/common/test/config.go +++ b/common/test/config.go @@ -106,6 +106,16 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Listen.PublicRoomsAPI = assignAddress() cfg.Listen.TypingServer = assignAddress() + // Bind to the same address as the listen address + // All microservices are run on the same host in testing + cfg.Bind.ClientAPI = cfg.Listen.ClientAPI + cfg.Bind.FederationAPI = cfg.Listen.FederationAPI + cfg.Bind.MediaAPI = cfg.Listen.MediaAPI + cfg.Bind.RoomServer = cfg.Listen.RoomServer + cfg.Bind.SyncAPI = cfg.Listen.SyncAPI + cfg.Bind.PublicRoomsAPI = cfg.Listen.PublicRoomsAPI + cfg.Bind.TypingServer = cfg.Listen.TypingServer + return &cfg, port, nil } From 9a56cf8439304540d0cdae18b30df82962c065cb Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 4 Oct 2019 17:08:00 +0100 Subject: [PATCH 05/19] Add AppService to test config and bind addrs (#800) Because our unit tests are often forgotten about :/ --- common/test/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/test/config.go b/common/test/config.go index 668e9a26e..693555619 100644 --- a/common/test/config.go +++ b/common/test/config.go @@ -91,6 +91,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con // the table names are globally unique. But we might not want to // rely on that in the future. cfg.Database.Account = config.DataSource(database) + cfg.Database.AppService = config.DataSource(database) cfg.Database.Device = config.DataSource(database) cfg.Database.MediaAPI = config.DataSource(database) cfg.Database.RoomServer = config.DataSource(database) @@ -99,6 +100,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Database.PublicRoomsAPI = config.DataSource(database) cfg.Listen.ClientAPI = assignAddress() + cfg.Listen.AppServiceAPI = assignAddress() cfg.Listen.FederationAPI = assignAddress() cfg.Listen.MediaAPI = assignAddress() cfg.Listen.RoomServer = assignAddress() From e959927d0ad20f53c51944abb751b7d40fad6aea Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Mon, 7 Oct 2019 20:15:58 +0800 Subject: [PATCH 06/19] selectAccountDataByType shouldn't error when no rows (#804) Signed-off-by: Alex Chen --- clientapi/auth/storage/accounts/account_data_table.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clientapi/auth/storage/accounts/account_data_table.go b/clientapi/auth/storage/accounts/account_data_table.go index 0d6ad0933..080ca3f38 100644 --- a/clientapi/auth/storage/accounts/account_data_table.go +++ b/clientapi/auth/storage/accounts/account_data_table.go @@ -125,6 +125,10 @@ func (s *accountDataStatements) selectAccountDataByType( var content []byte if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return } From 145921f2071660af7d8e1ea2ea015949a0b03c59 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 8 Oct 2019 12:20:37 +0100 Subject: [PATCH 07/19] Pin golangci-lint version to non-broken one (#809) --- scripts/find-lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh index 25b311f94..ca991b62c 100755 --- a/scripts/find-lint.sh +++ b/scripts/find-lint.sh @@ -27,7 +27,7 @@ echo "Installing golangci-lint..." # TODO: Once go 1.13 is out, use go get's -mod=readonly option # https://github.com/golang/go/issues/30667 cp go.mod go.mod.bak && cp go.sum go.sum.bak -go get github.com/golangci/golangci-lint/cmd/golangci-lint +go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1 echo "Looking for lint..." golangci-lint run $args From 8fb2c9c33c6bd7b1a58685668eedcbb57607f8c5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Dec 2019 16:47:45 +0000 Subject: [PATCH 08/19] Replace deprecated prometheus.InstrumentHandler and unsafe time.Ticker --- common/httpapi.go | 12 +++++- common/transactions/transactions.go | 2 +- go.mod | 26 ++++++----- go.sum | 67 +++++++++++++++++++++++++++++ mediaapi/routing/routing.go | 51 +++++++++++++--------- 5 files changed, 124 insertions(+), 34 deletions(-) diff --git a/common/httpapi.go b/common/httpapi.go index bf634ff4a..59b303b6d 100644 --- a/common/httpapi.go +++ b/common/httpapi.go @@ -11,6 +11,7 @@ import ( opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -59,7 +60,16 @@ func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request) } } - return prometheus.InstrumentHandler(metricsName, http.HandlerFunc(withSpan)) + return promhttp.InstrumentHandlerCounter( + promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: metricsName, + Help: "Total number of http requests for HTML resources", + }, + []string{"code"}, + ), + http.HandlerFunc(withSpan), + ) } // MakeInternalAPI turns a util.JSONRequestHandler function into an http.Handler. diff --git a/common/transactions/transactions.go b/common/transactions/transactions.go index 80b403a98..d2eb0f27f 100644 --- a/common/transactions/transactions.go +++ b/common/transactions/transactions.go @@ -85,7 +85,7 @@ func (t *Cache) AddTransaction(accessToken, txnID string, res *util.JSONResponse // It guarantees that an entry will be present in cache for at least cleanupPeriod & at most 2 * cleanupPeriod. // This cycles the txnMaps forward, i.e. back map is assigned the front and front is assigned an empty map. func cacheCleanService(t *Cache) { - ticker := time.Tick(t.cleanupPeriod) + ticker := time.NewTicker(t.cleanupPeriod).C for range ticker { t.Lock() t.txnsMaps[1] = t.txnsMaps[0] diff --git a/go.mod b/go.mod index d51f0a33e..d2cb80bb8 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,16 @@ module github.com/matrix-org/dendrite require ( github.com/Shopify/sarama v0.0.0-20170127151855-574d3147eee3 github.com/alecthomas/gometalinter v2.0.2+incompatible - github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf + github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7 - github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a + github.com/beorn7/perks v1.0.1 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b github.com/davecgh/go-spew v1.1.1 github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 github.com/eapache/queue v1.1.0 - github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b + github.com/golang/protobuf v1.3.2 github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 github.com/gorilla/context v1.1.1 @@ -28,18 +28,20 @@ require ( github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/nicksnyder/go-i18n v1.8.1 github.com/opentracing/opentracing-go v0.0.0-20170806192116-8ebe5d4e236e github.com/pelletier/go-toml v0.0.0-20170904195809-1d6b12b7cb29 github.com/pierrec/lz4 v0.0.0-20161206202305-5c9560bfa9ac github.com/pierrec/xxHash v0.0.0-20160112165351-5a004441f897 - github.com/pkg/errors v0.0.0-20170505043639-c605e284fe17 + github.com/pkg/errors v0.8.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v0.0.0-20180519192340-c51dc758d4bb - github.com/prometheus/client_model v0.0.0-20150212101744-fa8ad6fec335 - github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4 - github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537 + github.com/prometheus/client_golang v1.2.1 + github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 + github.com/prometheus/common v0.7.0 + github.com/prometheus/procfs v0.0.5 github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 github.com/sirupsen/logrus v1.4.2 github.com/stretchr/objx v0.2.0 // indirect @@ -54,9 +56,9 @@ require ( go.uber.org/atomic v1.3.0 go.uber.org/multierr v0.0.0-20170829224307-fb7d312c2c04 go.uber.org/zap v1.7.1 - golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 - golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 - golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 + golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 + golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 + golang.org/x/sys v0.0.0-20191010194322-b09406accb47 gopkg.in/Shopify/sarama.v1 v1.11.0 gopkg.in/airbrake/gobrake.v2 v2.0.9 gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3 @@ -65,3 +67,5 @@ require ( gopkg.in/macaroon.v2 v2.1.0 gopkg.in/yaml.v2 v2.2.2 ) + +go 1.13 diff --git a/go.sum b/go.sum index 56781c9a6..962d0f5bb 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,19 @@ github.com/Shopify/sarama v0.0.0-20170127151855-574d3147eee3 h1:j6BAEHYn1kUyW2j7kY0mOJ/R8A0qWwXpvUAEHGemm/g= github.com/Shopify/sarama v0.0.0-20170127151855-574d3147eee3/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/alecthomas/gometalinter v2.0.2+incompatible/go.mod h1:qfIpQGGz3d+NmgyPBqv+LSh50emm1pt72EtcX2vKYQk= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a h1:BtpsbiV638WQZwhA98cEZw2BsbnQJrbd0BI7tsy0W1c= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= +github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ= @@ -19,24 +28,41 @@ github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934/go.mod h1 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b h1:fE/yi9pibxGEc0gSJuEShcsBXE2d5FW3OudsjE9tKzQ= github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0 h1:FMElzTwkd/2jQ2QzLEzt97JRgvFhYhnYiaQSwZ7tuyU= github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.3.0 h1:HwSEKGN6U5T2aAQTfu5pW8fiwjSp3IgwdRbkICydk/c= github.com/gorilla/mux v1.3.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b/go.mod h1:HWG7INeOG1ZE17I/S8eeb+svquXmBS/hf1Obi6hJUyQ= github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe/go.mod h1:VqeqQrZmZr9G4WdLw4ei9tAHU54iJRkfoFHvTTQn4jQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -68,7 +94,13 @@ github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0= github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= @@ -82,19 +114,38 @@ github.com/pierrec/xxHash v0.0.0-20160112165351-5a004441f897 h1:jp3jc/PyyTrTKjJJ github.com/pierrec/xxHash v0.0.0-20160112165351-5a004441f897/go.mod h1:w2waW5Zoa/Wc4Yqe0wgrIYAGKqRMf7czn2HNKXmuL+I= github.com/pkg/errors v0.0.0-20170505043639-c605e284fe17 h1:chPfVn+gpAM5CTpTyVU9j8J+xgRGwmoDlNDLjKnJiYo= github.com/pkg/errors v0.0.0-20170505043639-c605e284fe17/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.0.0-20180519192340-c51dc758d4bb h1:ghXIh3jvLRo/h3y2O7wBgcmH1th5NXQ4XHwK5CgI//I= github.com/prometheus/client_golang v0.0.0-20180519192340-c51dc758d4bb/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= +github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_model v0.0.0-20150212101744-fa8ad6fec335 h1:0E/5GnGmzoDCtmzTycjGDWW33H0UBmAhR0h+FC8hWLs= github.com/prometheus/client_model v0.0.0-20150212101744-fa8ad6fec335/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4 h1:bZG2YNnM/Fjd3kiqaVt13Apkhzz24wBKlxQ+URiggXk= github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537 h1:Lq69k27tHOmljEqDOHDy3b6kQyEie2yWeAiF0OKFMJ8= github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 h1:gwcdIpH6NU2iF8CmcqD+CP6+1CkRBOhHaPR+iu6raBY= github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 h1:+8J/sCAVv2Y9Ct1BKszDFJEVWv6Aynr2O4FYGUg6+Mc= github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= @@ -104,6 +155,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tidwall/gjson v1.0.2 h1:5BsM7kyEAHAUGEGDkEKO9Mdyiuw6QQ6TSDdarP0Nnmk= github.com/tidwall/gjson v1.0.2/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= @@ -132,28 +184,43 @@ golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfM golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3 h1:tTDpczhDVjW6WN3DinzKcw5juwkDTVn22I7MNlfxSXM= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926 h1:PY6OU86NqbyZiOzaPnDw6oOjAGtYQqIua16z6y9QkwE= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/Shopify/sarama.v1 v1.11.0 h1:/3kaCyeYaPbr59IBjeqhIcUOB1vXlIVqXAYa5g5C5F0= gopkg.in/Shopify/sarama.v1 v1.11.0/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= +gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/macaroon.v2 v2.0.0/go.mod h1:+I6LnTMkm/uV5ew/0nsulNjL16SK4+C8yDmRUzHR17I= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1DyJ//wxKX1Cdh8LhoAw= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 5bcce1772..232a57fd1 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -29,6 +29,8 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" ) const pathPrefixR0 = "/_matrix/media/r0" @@ -83,26 +85,33 @@ func makeDownloadAPI( activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) http.HandlerFunc { - return prometheus.InstrumentHandler(name, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - req = util.RequestWithLogging(req) + return promhttp.InstrumentHandlerCounter( + promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: name, + Help: "Total number of media_api requests for either thumbnails or full downloads", + }, + []string{"code"}, + ), http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + req = util.RequestWithLogging(req) - // Set common headers returned regardless of the outcome of the request - util.SetCORSHeaders(w) - // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors - w.Header().Set("Content-Type", "application/json") - - vars, _ := common.URLDecodeMapValues(mux.Vars(req)) - Download( - w, - req, - gomatrixserverlib.ServerName(vars["serverName"]), - types.MediaID(vars["mediaId"]), - cfg, - db, - client, - activeRemoteRequests, - activeThumbnailGeneration, - name == "thumbnail", - ) - })) + // Set common headers returned regardless of the outcome of the request + util.SetCORSHeaders(w) + // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors + w.Header().Set("Content-Type", "application/json") + vars := mux.Vars(req) + Download( + w, + req, + gomatrixserverlib.ServerName(vars["serverName"]), + types.MediaID(vars["mediaId"]), + cfg, + db, + client, + activeRemoteRequests, + activeThumbnailGeneration, + name == "thumbnail", + ) + }, + )) } From de5d463dd2b5f8955b714d440d02777269255ed0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Dec 2019 16:54:21 +0000 Subject: [PATCH 09/19] goimports --- go.mod | 2 -- mediaapi/routing/routing.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d2cb80bb8..723bf99d3 100644 --- a/go.mod +++ b/go.mod @@ -28,8 +28,6 @@ require ( github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/nicksnyder/go-i18n v1.8.1 github.com/opentracing/opentracing-go v0.0.0-20170806192116-8ebe5d4e236e diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 232a57fd1..bc919bacd 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -113,5 +113,5 @@ func makeDownloadAPI( name == "thumbnail", ) }, - )) + )) } From 2baf5baa75e1a3e1834bd2dde2fa3601ab834f0d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Dec 2019 17:05:39 +0000 Subject: [PATCH 10/19] re-add temporarily missing deps? --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 723bf99d3..d2cb80bb8 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,8 @@ require ( github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/nicksnyder/go-i18n v1.8.1 github.com/opentracing/opentracing-go v0.0.0-20170806192116-8ebe5d4e236e From 9b5d6c9745bf8cb1c5a2057ee6218640c82e5250 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 18 Dec 2019 15:10:53 +0000 Subject: [PATCH 11/19] Refactor InstrumentHandlerCounter definition --- mediaapi/routing/routing.go | 56 ++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index bc919bacd..3a8cb9ca6 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -85,33 +85,33 @@ func makeDownloadAPI( activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) http.HandlerFunc { - return promhttp.InstrumentHandlerCounter( - promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: name, - Help: "Total number of media_api requests for either thumbnails or full downloads", - }, - []string{"code"}, - ), http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - req = util.RequestWithLogging(req) - - // Set common headers returned regardless of the outcome of the request - util.SetCORSHeaders(w) - // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors - w.Header().Set("Content-Type", "application/json") - vars := mux.Vars(req) - Download( - w, - req, - gomatrixserverlib.ServerName(vars["serverName"]), - types.MediaID(vars["mediaId"]), - cfg, - db, - client, - activeRemoteRequests, - activeThumbnailGeneration, - name == "thumbnail", - ) + counterVec := promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: name, + Help: "Total number of media_api requests for either thumbnails or full downloads", }, - )) + []string{"code"}, + ) + httpHandler := func(w http.ResponseWriter, req *http.Request) { + req = util.RequestWithLogging(req) + + // Set common headers returned regardless of the outcome of the request + util.SetCORSHeaders(w) + // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors + w.Header().Set("Content-Type", "application/json") + vars := mux.Vars(req) + Download( + w, + req, + gomatrixserverlib.ServerName(vars["serverName"]), + types.MediaID(vars["mediaId"]), + cfg, + db, + client, + activeRemoteRequests, + activeThumbnailGeneration, + name == "thumbnail", + ) + } + return promhttp.InstrumentHandlerCounter(counterVec, http.HandlerFunc(httpHandler)) } From f392ce8ed744654c25f9ea58ed609db8edb25b68 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 18 Dec 2019 15:29:27 +0000 Subject: [PATCH 12/19] URL decode args --- mediaapi/routing/routing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 3a8cb9ca6..0f2266640 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -99,7 +99,7 @@ func makeDownloadAPI( util.SetCORSHeaders(w) // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") - vars := mux.Vars(req) + vars, _ := common.URLDecodeMapValues(mux.Vars(req)) Download( w, req, From b4d638cd04d64bf574d8af75fc22efb72b4a0798 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 20 Dec 2019 13:24:57 +0000 Subject: [PATCH 13/19] Return server names (#833) * Remove unnecessary map->array processing * Return server names in room federation directory query * Knock off a TODO --- cmd/dendrite-federation-api-server/main.go | 3 ++- cmd/dendrite-monolith-server/main.go | 2 +- federationapi/federationapi.go | 5 ++++- federationapi/routing/query.go | 11 +++++++++-- federationapi/routing/routing.go | 4 +++- federationsender/query/query.go | 9 +++------ 6 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index c83845d25..367f5dc0c 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -29,6 +29,7 @@ func main() { deviceDB := base.CreateDeviceDB() keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() + federationSender := base.CreateHTTPFederationSenderAPIs() keyRing := keydb.CreateKeyRing(federation.Client, keyDB) alias, input, query := base.CreateHTTPRoomserverAPIs() @@ -36,7 +37,7 @@ func main() { federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, &keyRing, - alias, input, query, asQuery, + alias, input, query, asQuery, federationSender, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 0a320616e..5ea6b154a 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -67,7 +67,7 @@ func main() { federation, &keyRing, alias, input, query, typingInputAPI, asQuery, transactions.New(), fedSenderAPI, ) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query) diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 87402d976..c2aef06c8 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/common/basecomponent" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" // TODO: Are we really wanting to pull in the producer from clientapi @@ -39,11 +40,13 @@ func SetupFederationAPIComponent( inputAPI roomserverAPI.RoomserverInputAPI, queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI) routing.Setup( base.APIMux, *base.Cfg, queryAPI, aliasAPI, asAPI, - roomserverProducer, *keyRing, federation, accountsDB, deviceDB, + roomserverProducer, federationSenderAPI, *keyRing, federation, accountsDB, + deviceDB, ) } diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 781b8ea46..ed2d8b741 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" @@ -33,6 +34,7 @@ func RoomAliasToID( federation *gomatrixserverlib.FederationClient, cfg config.Dendrite, aliasAPI roomserverAPI.RoomserverAliasAPI, + senderAPI federationSenderAPI.FederationSenderQueryAPI, ) util.JSONResponse { roomAlias := httpReq.FormValue("room_alias") if roomAlias == "" { @@ -59,10 +61,15 @@ func RoomAliasToID( } if queryRes.RoomID != "" { - // TODO: List servers that are aware of this room alias + serverQueryReq := federationSenderAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: queryRes.RoomID} + var serverQueryRes federationSenderAPI.QueryJoinedHostServerNamesInRoomResponse + if err = senderAPI.QueryJoinedHostServerNamesInRoom(httpReq.Context(), &serverQueryReq, &serverQueryRes); err != nil { + return httputil.LogThenError(httpReq, err) + } + resp = gomatrixserverlib.RespDirectory{ RoomID: queryRes.RoomID, - Servers: []gomatrixserverlib.ServerName{}, + Servers: serverQueryRes.ServerNames, } } else { // If no alias was found, return an error diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 9f576790b..eae1fabdb 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -46,6 +47,7 @@ func Setup( aliasAPI roomserverAPI.RoomserverAliasAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, + federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, accountDB *accounts.Database, @@ -156,7 +158,7 @@ func Setup( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return RoomAliasToID( - httpReq, federation, cfg, aliasAPI, + httpReq, federation, cfg, aliasAPI, federationSenderAPI, ) }, )).Methods(http.MethodGet) diff --git a/federationsender/query/query.go b/federationsender/query/query.go index 088244826..8c35bb29e 100644 --- a/federationsender/query/query.go +++ b/federationsender/query/query.go @@ -45,15 +45,12 @@ func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( return } - serverNamesSet := make(map[gomatrixserverlib.ServerName]bool, len(joinedHosts)) + response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(joinedHosts)) for _, host := range joinedHosts { - serverNamesSet[host.ServerName] = true + response.ServerNames = append(response.ServerNames, host.ServerName) } - response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(serverNamesSet)) - for name := range serverNamesSet { - response.ServerNames = append(response.ServerNames, name) - } + // TODO: remove duplicates? return } From af9568ba4468eb106ec305df496fbe1319fd74eb Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 20 Dec 2019 14:41:32 +0000 Subject: [PATCH 14/19] Fix /send_join and /send_leave (#821) Fix the /send_join and /send_leave endpoints, so that they use the v2 endpoints as mandated by MSC1802. Also comment out the SyTest tests that are failing because of lack of support for the v1 endpoints. --- federationapi/routing/routing.go | 6 ++++-- testfile | 25 ++++++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index eae1fabdb..302f7ed0b 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -33,6 +33,7 @@ import ( const ( pathPrefixV2Keys = "/_matrix/key/v2" pathPrefixV1Federation = "/_matrix/federation/v1" + pathPrefixV2Federation = "/_matrix/federation/v2" ) // Setup registers HTTP handlers with the given ServeMux. @@ -55,6 +56,7 @@ func Setup( ) { v2keysmux := apiMux.PathPrefix(pathPrefixV2Keys).Subrouter() v1fedmux := apiMux.PathPrefix(pathPrefixV1Federation).Subrouter() + v2fedmux := apiMux.PathPrefix(pathPrefixV2Federation).Subrouter() localKeys := common.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { return LocalKeys(cfg) @@ -200,7 +202,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/send_join/{roomID}/{userID}", common.MakeFedAPI( + v2fedmux.Handle("/send_join/{roomID}/{userID}", common.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) @@ -230,7 +232,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/send_leave/{roomID}/{userID}", common.MakeFedAPI( + v2fedmux.Handle("/send_leave/{roomID}/{userID}", common.MakeFedAPI( "federation_send_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) diff --git a/testfile b/testfile index 5ad3426e9..8a4e9de12 100644 --- a/testfile +++ b/testfile @@ -126,8 +126,14 @@ Checking local federation server Inbound federation can query profile data Outbound federation can send room-join requests Outbound federation can send events -Inbound federation can backfill events -Backfill checks the events requested belong to the room +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#Inbound federation can backfill events +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#Backfill checks the events requested belong to the room Can upload without a file name Can download without a file name locally Can upload with ASCII file name @@ -143,7 +149,10 @@ Trying to get push rules with unknown rule_id fails with 404 Events come down the correct room local user can join room with version 5 User can invite local user to room with version 5 -Inbound federation can receive v1 room-join requests +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#Inbound federation can receive v1 room-join requests Typing events appear in initial sync Typing events appear in incremental sync Typing events appear in gapped sync @@ -156,8 +165,14 @@ User can create and send/receive messages in a room with version 1 POST /createRoom ignores attempts to set the room version via creation_content Inbound federation rejects remote attempts to join local users to rooms Inbound federation rejects remote attempts to kick local users to rooms -An event which redacts itself should be ignored -A pair of events which redact each other should be ignored +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#An event which redacts itself should be ignored +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#A pair of events which redact each other should be ignored Full state sync includes joined rooms A message sent after an initial sync appears in the timeline of an incremental sync. Can add tag From e2d73855ebdb236aaa4388685872792ba3907925 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar <31231064+abhishekkumar2718@users.noreply.github.com> Date: Fri, 20 Dec 2019 20:12:57 +0530 Subject: [PATCH 15/19] Refuse /send_join without m.room.create (#824) Signed-off-by: Abhishek Kumar --- federationapi/routing/join.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 6f6574dd7..e2885dd99 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -154,16 +154,23 @@ func SendJoin( // Fetch the state and auth chain. We do this before we send the events // on, in case this fails. - var stateAndAuthChainRepsonse api.QueryStateAndAuthChainResponse + var stateAndAuthChainResponse api.QueryStateAndAuthChainResponse err = query.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ PrevEventIDs: event.PrevEventIDs(), AuthEventIDs: event.AuthEventIDs(), RoomID: roomID, - }, &stateAndAuthChainRepsonse) + }, &stateAndAuthChainResponse) if err != nil { return httputil.LogThenError(httpReq, err) } + if !stateAndAuthChainResponse.RoomExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Room does not exist"), + } + } + // Send the events to the room server. // We are responsible for notifying other servers that the user has joined // the room, so set SendAsServer to cfg.Matrix.ServerName @@ -177,8 +184,8 @@ func SendJoin( return util.JSONResponse{ Code: http.StatusOK, JSON: map[string]interface{}{ - "state": stateAndAuthChainRepsonse.StateEvents, - "auth_chain": stateAndAuthChainRepsonse.AuthChainEvents, + "state": stateAndAuthChainResponse.StateEvents, + "auth_chain": stateAndAuthChainResponse.AuthChainEvents, }, } } From f1e229e9d4a6c7f6ad7eda146ec0e3a8a6603af1 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 20 Dec 2019 14:44:34 +0000 Subject: [PATCH 16/19] AS should use the v1 endpoint, rather than r0 (#827) --- appservice/routing/routing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index 3c19c8401..0e4bd6bab 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -27,7 +27,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixApp = "/_matrix/app/r0" +const pathPrefixApp = "/_matrix/app/v1" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. From 52dfa40471e1ba220960ed8f14033d7e03a96385 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 20 Dec 2019 14:46:59 +0000 Subject: [PATCH 17/19] docker: Passthrough parameters to dendrite-monolith-server --- docker/services/monolith.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/services/monolith.sh b/docker/services/monolith.sh index a038728b4..2287555cd 100644 --- a/docker/services/monolith.sh +++ b/docker/services/monolith.sh @@ -2,4 +2,4 @@ bash ./docker/build.sh -./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key +./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key $@ From 4f75e4febe5ec3ed8666fc723e0f2350f372bf2d Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 20 Dec 2019 16:00:25 +0100 Subject: [PATCH 18/19] Fix copy & paste error (#812) --- roomserver/api/query.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roomserver/api/query.go b/roomserver/api/query.go index a544f8aa2..e52c74ac3 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -332,7 +332,7 @@ const RoomserverQueryMissingEventsPath = "/api/roomserver/queryMissingEvents" // RoomserverQueryStateAndAuthChainPath is the HTTP path for the QueryStateAndAuthChain API const RoomserverQueryStateAndAuthChainPath = "/api/roomserver/queryStateAndAuthChain" -// RoomserverQueryBackfillPath is the HTTP path for the QueryMissingEvents API +// RoomserverQueryBackfillPath is the HTTP path for the QueryBackfill API const RoomserverQueryBackfillPath = "/api/roomserver/QueryBackfill" // NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. @@ -475,6 +475,6 @@ func (h *httpRoomserverQueryAPI) QueryBackfill( span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBackfill") defer span.Finish() - apiURL := h.roomserverURL + RoomserverQueryMissingEventsPath + apiURL := h.roomserverURL + RoomserverQueryBackfillPath return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } From b34fce0d85be4626fcc9d7ffaa110aca148cdb4b Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 20 Dec 2019 16:02:09 +0100 Subject: [PATCH 19/19] Use gomatrixserverlib.Transaction instead of local type (#590) (#811) --- federationapi/routing/backfill.go | 10 ++++--- federationapi/types/types.go | 43 ------------------------------- 2 files changed, 6 insertions(+), 47 deletions(-) delete mode 100644 federationapi/types/types.go diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index d996db6a3..5c6b0087f 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -17,11 +17,11 @@ package routing import ( "net/http" "strconv" + "time" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -90,9 +90,11 @@ func Backfill( } } - txn := types.NewTransaction() - txn.Origin = cfg.Matrix.ServerName - txn.PDUs = evs + txn := gomatrixserverlib.Transaction{ + Origin: cfg.Matrix.ServerName, + PDUs: evs, + OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), + } // Send the events to the client. return util.JSONResponse{ diff --git a/federationapi/types/types.go b/federationapi/types/types.go deleted file mode 100644 index 24838d547..000000000 --- a/federationapi/types/types.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "time" - - "github.com/matrix-org/gomatrixserverlib" -) - -// Transaction is the representation of a transaction from the federation API -// See https://matrix.org/docs/spec/server_server/unstable.html for more info. -type Transaction struct { - // The server_name of the homeserver sending this transaction. - Origin gomatrixserverlib.ServerName `json:"origin"` - // POSIX timestamp in milliseconds on originating homeserver when this - // transaction started. - OriginServerTS int64 `json:"origin_server_ts"` - // List of persistent updates to rooms. - PDUs []gomatrixserverlib.Event `json:"pdus"` -} - -// NewTransaction sets the timestamp of a new transaction instance and then -// returns the said instance. -func NewTransaction() Transaction { - // Retrieve the current timestamp in nanoseconds and make it a milliseconds - // one. - ts := time.Now().UnixNano() / int64(time.Millisecond) - - return Transaction{OriginServerTS: ts} -}