From b1d5360335ef9b26f62504282669403f6f3e6df5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 11:55:17 +0100 Subject: [PATCH 01/10] Update HTTP clients (#1457) * Update gomatrixserverlib * Use separate HTTP client for API calls, set User-Agent for outbound HTTP requests --- cmd/dendrite-media-api-server/main.go | 3 +- cmd/dendrite-monolith-server/main.go | 3 +- go.mod | 2 +- go.sum | 10 ++----- internal/setup/base.go | 41 ++++++++++++++++++--------- 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index f442abfa7..2c2fe3b36 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -17,7 +17,6 @@ package main import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" - "github.com/matrix-org/gomatrixserverlib" ) func main() { @@ -26,7 +25,7 @@ func main() { defer base.Close() // nolint: errcheck userAPI := base.UserAPIClient() - client := gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation) + client := base.CreateClient() mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, userAPI, client) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 759f1c9ff..28a349a76 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -29,7 +29,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/dendrite/userapi" - "github.com/matrix-org/gomatrixserverlib" ) var ( @@ -125,7 +124,7 @@ func main() { monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, - Client: gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation), + Client: base.CreateClient(), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.KafkaConsumer, diff --git a/go.mod b/go.mod index c0dda07cd..b4e16c0c4 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a + github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 648807285..461b3d3c0 100644 --- a/go.sum +++ b/go.sum @@ -569,14 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad h1:n0P/Oy8ZqqTPzum6FEayAjamsmvJTuIcA10WQ8GcK70= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553 h1:tiel2c3I9xr0SRS05g3UvOjj6Sgg1I3Yn2/oGA1GgLk= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2 h1:a07U5eFT521mFiUtA/A8NwiZp5vfRU59/QKs+pa3Fkc= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a h1:kIwbS7eY7P/MX0oN4wRHGkoc4eTTnwOcdCawBZ3SrJI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae h1:9vxYG3+pXD3G74lZ8s67/afs2RkvyrJy6HEcqfHEVgw= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/internal/setup/base.go b/internal/setup/base.go index ef956dd2a..f0009d19d 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -69,6 +69,7 @@ type BaseDendrite struct { PublicMediaAPIMux *mux.Router InternalAPIMux *mux.Router UseHTTPAPIs bool + apiHttpClient *http.Client httpClient *http.Client Cfg *config.Dendrite Caches *caching.Caches @@ -118,6 +119,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Warnf("Failed to create cache") } + apiClient := http.Client{} client := http.Client{Timeout: HTTPClientTimeout} if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ @@ -148,6 +150,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo PublicKeyAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(), PublicMediaAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(), InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), + apiHttpClient: &apiClient, httpClient: &client, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, @@ -161,7 +164,7 @@ func (b *BaseDendrite) Close() error { // AppserviceHTTPClient returns the AppServiceQueryAPI for hitting the appservice component over HTTP. func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { - a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.httpClient) + a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.apiHttpClient) if err != nil { logrus.WithError(err).Panic("CreateHTTPAppServiceAPIs failed") } @@ -170,27 +173,27 @@ func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { // RoomserverHTTPClient returns RoomserverInternalAPI for hitting the roomserver over HTTP. func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAPI { - rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.httpClient, b.Caches) + rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.apiHttpClient, b.Caches) if err != nil { - logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.apiHttpClient) } return rsAPI } // UserAPIClient returns UserInternalAPI for hitting the userapi over HTTP. func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI { - userAPI, err := userapiinthttp.NewUserAPIClient(b.Cfg.UserAPIURL(), b.httpClient) + userAPI, err := userapiinthttp.NewUserAPIClient(b.Cfg.UserAPIURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("UserAPIClient failed", b.httpClient) + logrus.WithError(err).Panic("UserAPIClient failed", b.apiHttpClient) } return userAPI } // EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { - e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient) + e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("EDUServerClient failed", b.httpClient) + logrus.WithError(err).Panic("EDUServerClient failed", b.apiHttpClient) } return e } @@ -198,9 +201,9 @@ func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { // FederationSenderHTTPClient returns FederationSenderInternalAPI for hitting // the federation sender over HTTP func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.FederationSenderInternalAPI { - f, err := fsinthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.httpClient) + f, err := fsinthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("FederationSenderHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("FederationSenderHTTPClient failed", b.apiHttpClient) } return f } @@ -209,7 +212,7 @@ func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.Federati func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { f, err := skinthttp.NewServerKeyClient( b.Cfg.ServerKeyAPIURL(), - b.httpClient, + b.apiHttpClient, b.Caches, ) if err != nil { @@ -220,9 +223,9 @@ func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { // KeyServerHTTPClient returns KeyInternalAPI for hitting the key server over HTTP func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI { - f, err := keyinthttp.NewKeyServerClient(b.Cfg.KeyServerURL(), b.httpClient) + f, err := keyinthttp.NewKeyServerClient(b.Cfg.KeyServerURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("KeyServerHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("KeyServerHTTPClient failed", b.apiHttpClient) } return f } @@ -238,13 +241,25 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { return db } +// CreateClient creates a new client (normally used for media fetch requests). +// Should only be called once per component. +func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client { + client := gomatrixserverlib.NewClient( + b.Cfg.FederationSender.DisableTLSValidation, + ) + client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) + return client +} + // CreateFederationClient creates a new federation client. Should only be called // once per component. func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { - return gomatrixserverlib.NewFederationClient( + client := gomatrixserverlib.NewFederationClient( b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey, b.Cfg.FederationSender.DisableTLSValidation, ) + client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) + return client } // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on From cb4b93b16ca60099699bfcde0e2dcec30b4153d8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 14:33:48 +0100 Subject: [PATCH 02/10] Add nginx sample reverse proxy configs (#1458) * Add nginx sample reverse proxy configs * Add line breaks at end of files --- docs/nginx/monolith-sample.conf | 24 ++++++++++++++++++++++ docs/nginx/polylith-sample.conf | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 docs/nginx/monolith-sample.conf create mode 100644 docs/nginx/polylith-sample.conf diff --git a/docs/nginx/monolith-sample.conf b/docs/nginx/monolith-sample.conf new file mode 100644 index 000000000..4129729f4 --- /dev/null +++ b/docs/nginx/monolith-sample.conf @@ -0,0 +1,24 @@ +server { + listen 443 ssl; + server_name my.hostname.com; + + ssl_certificate /path/to/fullchain.pem; + ssl_certificate_key /path/to/privkey.pem; + ssl_dhparam /path/to/ssl-dhparams.pem; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_read_timeout 600; + + location = /.well-known/matrix/server { + return 200 '{ "m.server": "my.hostname.com:443" }'; + } + + location = /.well-known/matrix/client { + return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; + } + + location = /_matrix { + proxy_pass http://monolith:8008; + } +} diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf new file mode 100644 index 000000000..b2a91b0c5 --- /dev/null +++ b/docs/nginx/polylith-sample.conf @@ -0,0 +1,36 @@ +server { + listen 443 ssl; + server_name my.hostname.com; + + ssl_certificate /path/to/fullchain.pem; + ssl_certificate_key /path/to/privkey.pem; + ssl_dhparam /path/to/ssl-dhparams.pem; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_read_timeout 600; + + location = /.well-known/matrix/server { + return 200 '{ "m.server": "my.hostname.com:443" }'; + } + + location = /.well-known/matrix/client { + return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; + } + + location = /_matrix/client { + proxy_pass http://client_api:8071; + } + + location = /_matrix/federation { + proxy_pass http://federation_api:8072; + } + + location = /_matrix/key { + proxy_pass http://federation_api:8072; + } + + location = /_matrix/media { + proxy_pass http://media_api:8074; + } +} From 43b3c4a2fc493fa6cacf0174f8c9f31f6b9c88c1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 14:41:08 +0100 Subject: [PATCH 03/10] Fix bugs in nginx sample configs --- docs/nginx/monolith-sample.conf | 6 +++--- docs/nginx/polylith-sample.conf | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/nginx/monolith-sample.conf b/docs/nginx/monolith-sample.conf index 4129729f4..9ee5e1ac1 100644 --- a/docs/nginx/monolith-sample.conf +++ b/docs/nginx/monolith-sample.conf @@ -10,15 +10,15 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_read_timeout 600; - location = /.well-known/matrix/server { + location /.well-known/matrix/server { return 200 '{ "m.server": "my.hostname.com:443" }'; } - location = /.well-known/matrix/client { + location /.well-known/matrix/client { return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; } - location = /_matrix { + location /_matrix { proxy_pass http://monolith:8008; } } diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index b2a91b0c5..658e0e4a4 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -10,27 +10,27 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_read_timeout 600; - location = /.well-known/matrix/server { + location /.well-known/matrix/server { return 200 '{ "m.server": "my.hostname.com:443" }'; } - location = /.well-known/matrix/client { + location /.well-known/matrix/client { return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; } - location = /_matrix/client { + location /_matrix/client { proxy_pass http://client_api:8071; } - location = /_matrix/federation { + location /_matrix/federation { proxy_pass http://federation_api:8072; } - location = /_matrix/key { + location /_matrix/key { proxy_pass http://federation_api:8072; } - location = /_matrix/media { + location /_matrix/media { proxy_pass http://media_api:8074; } } From 378a0520be8a03ee4c0bab6de8a1fff24bf4dcdf Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 1 Oct 2020 15:00:16 +0100 Subject: [PATCH 04/10] Add custom issue templates; fixes #1421 (#1460) --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 49 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/FEATURE_REQUEST.md | 14 +++++++ .github/PULL_REQUEST_TEMPLATE.md | 2 +- 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/BUG_REPORT.md create mode 100644 .github/ISSUE_TEMPLATE/FEATURE_REQUEST.md diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md new file mode 100644 index 000000000..68ae922a3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -0,0 +1,49 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- + + + +### Background information + +- **Dendrite version or git SHA**: +- **Monolith or Polylith?**: +- **SQLite3 or Postgres?**: +- **Running in Docker?**: +- **`go version`**: + + + +### Description + + + +### Steps to reproduce + +- list the steps +- that reproduce the bug +- using hyphens as bullet points + + diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md new file mode 100644 index 000000000..084f683bd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md @@ -0,0 +1,14 @@ +--- +name: Feature request +about: Suggest an idea for this project + +--- + + + +**Description:** + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8d22f7f72..c1cb8d7cc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,5 +2,5 @@ -* [ ] I have added any new tests that need to pass to `testfile` as specified in [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md) +* [ ] I have added any new tests that need to pass to `sytest-whitelist` as specified in [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md) * [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/master/docs/CONTRIBUTING.md#sign-off) From b471d4d77ac884bc232a6a09d01f3fd4c3aa3a38 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 15:00:35 +0100 Subject: [PATCH 05/10] Set internal HTTP API timeout to 10 minutes (#1459) --- internal/setup/base.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/setup/base.go b/internal/setup/base.go index f0009d19d..f9ddfdf7d 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -119,7 +119,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Warnf("Failed to create cache") } - apiClient := http.Client{} + apiClient := http.Client{Timeout: time.Minute * 10} client := http.Client{Timeout: HTTPClientTimeout} if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ From 4f87df198acfcd804753df37a81d6b5a0fa735ba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 15:27:16 +0100 Subject: [PATCH 06/10] Update README.md (#1461) * Update README.md * Maybe bullet points * Tweaks * Put the shields back, use text instead, less OCD problems * One more tweak --- README.md | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ecb3cdb9b..15e0d6b53 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ -# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) +# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite Alerts](https://img.shields.io/matrix/dendrite-alerts:matrix.org.svg?label=%23dendrite-alerts%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-alerts:matrix.org) -Dendrite is a second-generation Matrix homeserver written in Go. It is not recommended to use Dendrite as -a production homeserver at this time as there is no stable release. +Dendrite is a second-generation Matrix homeserver written in Go! -Dendrite will start to receive versioned releases stable enough to run [once we enter beta](https://github.com/matrix-org/dendrite/milestone/8). +Join us in: -# Quick start +- **[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org)** - General chat about the Dendrite project, for users and server admins alike +- **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** - The place for developers, where all Dendrite development discussion happens +- **[#dendrite-alerts:matrix.org](https://matrix.to/#/#dendrite-alerts:matrix.org)** - Release notifications and important info, highly recommended for all Dendrite server admins + +## Quick start Requires Go 1.13+ and SQLite3 (Postgres is also supported): @@ -30,7 +33,7 @@ $ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config Then point your favourite Matrix client at `http://localhost:8008`. For full installation information, see [INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker). -# Progress +## Progress We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it @@ -59,7 +62,7 @@ This means Dendrite supports amongst others: - E2E keys and device lists -# Contributing +## Contributing We would be grateful for any help on issues marked as [Are We Synapse Yet](https://github.com/matrix-org/dendrite/labels/are-we-synapse-yet). These issues @@ -101,7 +104,7 @@ look for [Good First Issues](https://github.com/matrix-org/dendrite/labels/good% familiar with the project, look for [Help Wanted](https://github.com/matrix-org/dendrite/labels/help-wanted) issues. -# Hardware requirements +## Hardware requirements Dendrite in Monolith + SQLite works in a range of environments including iOS and in-browser via WASM. @@ -112,12 +115,3 @@ encrypted rooms: - CPU: Brief spikes when processing events, typically idles at 1% CPU. This means Dendrite should comfortably work on things like Raspberry Pis. - -# Discussion - -For questions about Dendrite we have a dedicated room on Matrix -[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org). Development -discussion should happen in -[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org). - - From 9b2d8f69aa2bab5dfc1ae9e52a3cc20c7201a1d7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 15:28:01 +0100 Subject: [PATCH 07/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15e0d6b53..72c0df07d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite Alerts](https://img.shields.io/matrix/dendrite-alerts:matrix.org.svg?label=%23dendrite-alerts%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-alerts:matrix.org) +# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) Dendrite is a second-generation Matrix homeserver written in Go! From dbae85283f3927444aa89221523405ada07a0f4a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 15:37:45 +0100 Subject: [PATCH 08/10] Bring docker sample config into sync with normal one --- build/docker/config/dendrite-config.yaml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index 7ebeeb6e7..2bf8dd85f 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -38,8 +38,13 @@ global: # The path to the signing private key file, used to sign requests and events. private_key: matrix_key.pem - # A unique identifier for this private key. Must start with the prefix "ed25519:". - key_id: ed25519:auto + # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) + # to old signing private keys that were formerly in use on this domain. These + # keys will not be used for federation request or event signing, but will be + # provided to any other homeserver that asks when trying to verify old events. + # old_private_keys: + # - private_key: old_matrix_key.pem + # expired_at: 1601024554498 # How long a remote server can cache our server signing key before requesting it # again. Increasing this number will reduce the number of requests made by other @@ -133,6 +138,14 @@ client_api: turn_username: "" turn_password: "" + # Settings for rate-limited endpoints. Rate limiting will kick in after the + # threshold number of "slots" have been taken by requests from a specific + # host. Each "slot" will be released after the cooloff time in milliseconds. + rate_limiting: + enabled: true + threshold: 5 + cooloff_ms: 500 + # Configuration for the EDU server. edu_server: internal_api: @@ -260,6 +273,11 @@ server_key_api: public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw - key_id: ed25519:a_RXGa public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ + + # This option will control whether Dendrite will prefer to look up keys directly + # or whether it should try perspective servers first, using direct fetches as a + # last resort. + prefer_direct_fetch: false # Configuration for the Sync API. sync_api: @@ -291,6 +309,8 @@ user_api: conn_max_lifetime: -1 # Configuration for Opentracing. +# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on +# how this works and how to set it up. tracing: enabled: false jaeger: From 7048532bc4adcc017739fae358fd6b6c44b63ebd Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 16:09:11 +0100 Subject: [PATCH 09/10] Update version imprinting (#1462) * Add version tag constant * Update build imprinting --- build.sh | 12 +++++++++--- internal/version.go | 3 ++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index 34e4b1153..f790077e3 100755 --- a/build.sh +++ b/build.sh @@ -3,10 +3,16 @@ # Put installed packages into ./bin export GOBIN=$PWD/`dirname $0`/bin -export BRANCH=`(git symbolic-ref --short HEAD | cut -d'/' -f 3 )|| ""` -export BUILD=`git rev-parse --short HEAD || ""` +if [ -d ".git" ] +then + export BUILD=`git rev-parse --short HEAD || ""` + export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""` + [[ $BRANCH == "master" ]] && export BRANCH="" -export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD" + export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD" +else + export FLAGS="" +fi go install -trimpath -ldflags "$FLAGS" -v $PWD/`dirname $0`/cmd/... diff --git a/internal/version.go b/internal/version.go index 851a09384..718273e72 100644 --- a/internal/version.go +++ b/internal/version.go @@ -12,10 +12,11 @@ const ( VersionMajor = 0 VersionMinor = 0 VersionPatch = 0 + VersionTag = "" // example: "rc1" ) func VersionString() string { - version := fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch) + version := fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionTag) if branch != "" { version += fmt.Sprintf("-%s", branch) } From 3e01db0049a839eb163523990b383156b69527c2 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Thu, 1 Oct 2020 21:00:56 +0200 Subject: [PATCH 10/10] Fix golangci-lint issues (#1464) * Fix S1039: unnecessary use of fmt.Sprintf * Fix S1036: unnecessary guard around map access Signed-off-by: Till Faelligen --- federationapi/routing/join.go | 4 +--- roomserver/state/state.go | 6 +----- syncapi/storage/storage_test.go | 16 ++++++++-------- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 9fa0794ef..c637116f7 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -192,9 +192,7 @@ func SendJoin( if event.StateKey() == nil || event.StateKeyEquals("") { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON( - fmt.Sprintf("No state key was provided in the join event."), - ), + JSON: jsonerror.BadJSON("No state key was provided in the join event."), } } diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 9ee6f40d4..0663499e7 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -717,11 +717,7 @@ func ResolveConflictsAdhoc( // Append the events if there is already a conflicted list for // this tuple key, create it if not. tuple := stateKeyTuple{event.Type(), *event.StateKey()} - if _, ok := eventMap[tuple]; ok { - eventMap[tuple] = append(eventMap[tuple], event) - } else { - eventMap[tuple] = []gomatrixserverlib.Event{event} - } + eventMap[tuple] = append(eventMap[tuple], event) } // Split out the events in the map into conflicted and unconflicted diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 0e827c95d..8f16642f0 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -81,7 +81,7 @@ func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []gomatrixserve })) state = append(state, events[len(events)-1]) events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &userA, Sender: userA, @@ -97,7 +97,7 @@ func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []gomatrixserve })) } events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &userB, Sender: userB, @@ -348,7 +348,7 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { Depth: int64(len(events) + 1), })) events = append(events, MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &testUserIDA, Sender: testUserIDA, @@ -367,7 +367,7 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { } // merge the fork, prev_events are all 3 messages, depth is increased by 1. events = append(events, MustCreateEvent(t, testRoomID, events[len(events)-3:], &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"body":"Message merge"}`)), + Content: []byte(`{"body":"Message merge"}`), Type: "m.room.message", Sender: testUserIDA, Depth: depth + 1, @@ -438,7 +438,7 @@ func TestGetEventsInTopologicalRangeMultiRoom(t *testing.T) { Depth: int64(len(events) + 1), })) events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &testUserIDA, Sender: testUserIDA, @@ -484,7 +484,7 @@ func TestGetEventsInRangeWithEventsInsertedLikeBackfill(t *testing.T) { // "federation" join userC := fmt.Sprintf("@radiance:%s", testOrigin) joinEvent := MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &userC, Sender: userC, @@ -615,14 +615,14 @@ func TestInviteBehaviour(t *testing.T) { db := MustCreateDatabase(t) inviteRoom1 := "!inviteRoom1:somewhere" inviteEvent1 := MustCreateEvent(t, inviteRoom1, nil, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"invite"}`)), + Content: []byte(`{"membership":"invite"}`), Type: "m.room.member", StateKey: &testUserIDA, Sender: "@inviteUser1:somewhere", }) inviteRoom2 := "!inviteRoom2:somewhere" inviteEvent2 := MustCreateEvent(t, inviteRoom2, nil, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"invite"}`)), + Content: []byte(`{"membership":"invite"}`), Type: "m.room.member", StateKey: &testUserIDA, Sender: "@inviteUser2:somewhere",