From 8035c50c066209108f9ccefebb9e12fdc1df3cab Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 8 Oct 2020 14:46:25 +0100 Subject: [PATCH 1/5] Don't get into situations where we have no forward extremities --- roomserver/internal/input/input_latest_events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index b6666cc03..229665a0b 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -296,7 +296,7 @@ func (u *latestEventsUpdater) calculateLatest( referenced, err := u.updater.IsReferenced(newEvent.EventReference) if err != nil { logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", newEvent.EventReference.EventID) - } else if !referenced { + } else if !referenced || len(newLatest) == 0 { newLatest = append(newLatest, newEvent) } From b12b7abcc0a61c3b422ca0f337aec7533ba7371f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 8 Oct 2020 14:53:46 +0100 Subject: [PATCH 2/5] v0.1.0 Squashed commit of the following: commit 570528e5f19d76fe2fd7ea646246eea9650b1b60 Author: Kegan Dougal Date: Thu Oct 8 14:52:10 2020 +0100 v0.1.0 commit 02c020bced3be6d1a63e1001ab9c0ef6231ba087 Merge: db840f02 8035c50c Author: Neil Alexander Date: Thu Oct 8 14:46:32 2020 +0100 Merge branch 'master' into v0.1.0 commit db840f025b5d7b7c7e1ba905646571cb03dd7b22 Merge: adc19a3d 78f6e1a3 Author: Neil Alexander Date: Thu Oct 8 13:31:36 2020 +0100 Merge branch 'master' into v0.1.0 commit adc19a3d5f0c9d5a85b3c1dd7d1772236fb8cc9b Merge: c8fc6855 3e12f6e9 Author: Neil Alexander Date: Thu Oct 8 10:31:58 2020 +0100 Merge branch 'master' into v0.1.0 commit c8fc68555c3607b0153d10df91d357d9603ccf90 Author: Neil Alexander Date: Wed Oct 7 18:41:04 2020 +0100 Version 0.1.0rc3 commit 15bf3851415dc21ebcfa98e0f2a5ec725034d6dd Merge: e7d9eea4 8bca7a83 Author: Neil Alexander Date: Wed Oct 7 18:39:25 2020 +0100 Merge branch 'master' into v0.1.0 commit e7d9eea4a09be7b05a87b1df2a9e87d3109e8fcc Author: Neil Alexander Date: Mon Oct 5 17:56:45 2020 +0100 v0.1.0rc2 commit 3fa76370f214e2ba3ec04f4c6f0f63d3baa273e7 Merge: f7cecdd9 52ddded7 Author: Neil Alexander Date: Mon Oct 5 17:56:28 2020 +0100 Merge branch 'master' into v0.1.0 commit f7cecdd9a85fe2806a99e426b806832e7036da1e Author: Kegan Dougal Date: Fri Oct 2 17:25:59 2020 +0100 Bump to 0.1.0rc1 --- internal/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/version.go b/internal/version.go index 718273e72..2ffd7c90e 100644 --- a/internal/version.go +++ b/internal/version.go @@ -10,7 +10,7 @@ var build string const ( VersionMajor = 0 - VersionMinor = 0 + VersionMinor = 1 VersionPatch = 0 VersionTag = "" // example: "rc1" ) From 009401ad4df2e8f5c0c2f08b8c73f807e99b2c13 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 8 Oct 2020 17:45:55 +0100 Subject: [PATCH 3/5] Version 0.1.0 Beta README (#1466) * Beta docs * More tweaks * More docs * Update README.md (#1497) * Call out missing features * Add CHANGES * Call out CHANGES * Update INSTALL.md * Update README.md Co-authored-by: Neil Alexander --- CHANGES.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 50 +++++++++++++++++++++++---- docs/INSTALL.md | 9 +++-- 3 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 CHANGES.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 000000000..17fb75bed --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,89 @@ +# Dendrite 0.1.0 (2020-10-08) + +First versioned release of Dendrite. + +## Client-Server API Features + +### Account registration and management +- Registration: By password only. +- Login: By password only. No fallback. +- Logout: Yes. +- Change password: Yes. +- Link email/msisdn to account: No. +- Deactivate account: Yes. +- Check if username is available: Yes. +- Account data: Yes. +- OpenID: No. + +### Rooms +- Room creation: Yes, including presets. +- Joining rooms: Yes, including by alias or `?server_name=`. +- Event sending: Yes, including transaction IDs. +- Aliases: Yes. +- Published room directory: Yes. +- Kicking users: Yes. +- Banning users: Yes. +- Inviting users: Yes, but not third-party invites. +- Forgetting rooms: No. +- Room versions: All (v1 - v6) +- Tagging: Yes. + +### User management +- User directory: Basic support. +- Ignoring users: No. +- Groups/Communities: No. + +### Device management +- Creating devices: Yes. +- Deleting devices: Yes. +- Send-to-device messaging: Yes. + +### Sync +- Filters: Timeline limit only. Rest unimplemented. +- Deprecated `/events` and `/initialSync`: No. + +### Room events +- Typing: Yes. +- Receipts: No. +- Read Markers: No. +- Presence: No. +- Content repository (attachments): Yes. +- History visibility: No, defaults to `joined`. +- Push notifications: No. +- Event context: No. +- Reporting content: No. + +### End-to-End Encryption +- Uploading device keys: Yes. +- Downloading device keys: Yes. +- Claiming one-time keys: Yes. +- Querying key changes: Yes. +- Cross-Signing: No. + +### Misc +- Server-side search: No. +- Guest access: Partial. +- Room previews: No, partial support for Peeking via MSC2753. +- Third-Party networks: No. +- Server notices: No. +- Policy lists: No. + +## Federation Features +- Querying keys (incl. notary): Yes. +- Server ACLs: Yes. +- Sending transactions: Yes. +- Joining rooms: Yes. +- Inviting to rooms: Yes, but not third-party invites. +- Leaving rooms: Yes. +- Content repository: Yes. +- Backfilling / get_missing_events: Yes. +- Retrieving state of the room (`/state` and `/state_ids`): Yes. +- Public rooms: Yes. +- Querying profile data: Yes. +- Device management: Yes. +- Send-to-Device messaging: Yes. +- Querying/Claiming E2E Keys: Yes. +- Typing: Yes. +- Presence: No. +- Receipts: No. +- OpenID: No. \ No newline at end of file diff --git a/README.md b/README.md index 72c0df07d..f27cb4029 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,28 @@ # Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) -Dendrite is a second-generation Matrix homeserver written in Go! +Dendrite is a second-generation Matrix homeserver written in Go. +It intends to provide an **efficient**, **reliable** and **scalable** alternative to Synapse: + - Efficient: A small memory footprint with better baseline performance than an out-of-the-box Synapse. + - Reliable: Implements the Matrix specification as written, using the + [same test suite](https://github.com/matrix-org/sytest) as Synapse as well as + a [brand new Go test suite](https://github.com/matrix-org/complement). + - Scalable: can run on multiple machines and eventually scale to massive homeserver deployments. + + +As of October 2020, Dendrite has now entered **beta** which means: +- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database. +- Dendrite has periodic semver releases. We intend to release new versions as we land significant features. +- Dendrite supports database schema upgrades between releases. This means you should never lose your messages when upgrading Dendrite. +- Breaking changes will not occur on minor releases. This means you can safely upgrade Dendrite without modifying your database or config file. + +This does not mean: + - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially. + - All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular, + read receipts, presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates. + - Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine. + +Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices. +In the future, we will be able to scale up to gigantic servers (equivalent to matrix.org) via polylith mode. Join us in: @@ -8,9 +30,26 @@ Join us in: - **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** - The place for developers, where all Dendrite development discussion happens - **[#dendrite-alerts:matrix.org](https://matrix.to/#/#dendrite-alerts:matrix.org)** - Release notifications and important info, highly recommended for all Dendrite server admins -## Quick start +## Requirements -Requires Go 1.13+ and SQLite3 (Postgres is also supported): +To build Dendrite, you will need Go 1.13 or later. + +For a usable federating Dendrite deployment, you will also need: +- A domain name (or subdomain) +- A valid TLS certificate issued by a trusted authority for that domain +- SRV records or a well-known file pointing to your deployment + +Also recommended are: +- A PostgreSQL database engine, which will perform better than SQLite with many users and/or larger rooms +- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/master/docs/nginx/monolith-sample.conf) + +The [Federation Tester](https://federationtester.matrix.org) can be used to verify your deployment. + +## Get started + +If you wish to build a fully-federating Dendrite instance, see [INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker). + +The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases: ```bash $ git clone https://github.com/matrix-org/dendrite @@ -30,14 +69,13 @@ $ go build ./cmd/dendrite-monolith-server $ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml ``` -Then point your favourite Matrix client at `http://localhost:8008`. For full installation information, see -[INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker). +Then point your favourite Matrix client at `http://localhost:8008`. ## Progress We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it -updates with CI. As of August 2020 we're at around 52% CS API coverage and 65% Federation coverage, though check +updates with CI. As of October 2020 we're at around 56% CS API coverage and 77% Federation coverage, though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably: - Receipts diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 913bc5832..be12d7b86 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -120,7 +120,10 @@ Assuming that Postgres 9.5 (or later) is installed: Each Dendrite server requires unique server keys. -Generate the self-signed SSL certificate for federation and the server signing key: +In order for an instance to federate correctly, you should have a valid +certificate issued by a trusted authority, and private key to match. If you +don't and just want to test locally, generate the self-signed SSL certificate +for federation and the server signing key: ```bash ./bin/generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key @@ -267,12 +270,12 @@ This manages end-to-end encryption keys for users. ./bin/dendrite-key-server --config dendrite.yaml ``` -#### Server Key server +#### Signing key server This manages signing keys for servers. ```bash -./bin/dendrite-server-key-api-server --config dendrite.yaml +./bin/dendrite-signing-key-server --config dendrite.yaml ``` #### EDU server From f3e8ae01efb0abd0904509ddaa2ae85017ca4aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFck=20Bonniot?= Date: Fri, 9 Oct 2020 10:15:35 +0200 Subject: [PATCH 4/5] Implement fully read markers (#1475) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See #653 Signed-off-by: Loïck Bonniot Co-authored-by: Kegsay --- clientapi/routing/account_data.go | 75 ++++++++++++++++++++++++++++++- clientapi/routing/routing.go | 9 ++-- sytest-whitelist | 3 ++ 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index d5fafedb1..48303c97f 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -20,8 +20,10 @@ import ( "io/ioutil" "net/http" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -91,6 +93,13 @@ func SaveAccountData( } } + if dataType == "m.fully_read" { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Unable to set read marker"), + } + } + body, err := ioutil.ReadAll(req.Body) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed") @@ -112,7 +121,7 @@ func SaveAccountData( } dataRes := api.InputAccountDataResponse{} if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed") + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") return util.ErrorResponse(err) } @@ -127,3 +136,67 @@ func SaveAccountData( JSON: struct{}{}, } } + +type readMarkerJSON struct { + FullyRead string `json:"m.fully_read"` + Read string `json:"m.read"` +} + +type fullyReadEvent struct { + EventID string `json:"event_id"` +} + +// SaveReadMarker implements POST /rooms/{roomId}/read_markers +func SaveReadMarker( + req *http.Request, userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, + syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string, +) util.JSONResponse { + // Verify that the user is a member of this room + resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + if resErr != nil { + return *resErr + } + + var r readMarkerJSON + resErr = httputil.UnmarshalJSONRequest(req, &r) + if resErr != nil { + return *resErr + } + + if r.FullyRead == "" { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"), + } + } + + data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) + } + + if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read"); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed") + return jsonerror.InternalServerError() + } + + // TODO handle the read receipt that may be included in the read marker + // See https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-rooms-roomid-read-markers + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 8606f69c3..b547efb4b 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -695,12 +695,15 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/read_markers", - httputil.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { + httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { if r := rateLimits.rateLimit(req); r != nil { return *r } - // TODO: return the read_markers. - return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}} + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SaveReadMarker(req, userAPI, rsAPI, syncProducer, device, vars["roomID"]) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/sytest-whitelist b/sytest-whitelist index e0f1f311e..420acb22c 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -456,6 +456,9 @@ After changing password, can log in with new password After changing password, existing session still works After changing password, different sessions can optionally be kept After changing password, a different session no longer works by default +Read markers appear in incremental v2 /sync +Read markers appear in initial v2 /sync +Read markers can be updated Local users can peek into world_readable rooms by room ID We can't peek into rooms with shared history_visibility We can't peek into rooms with invited history_visibility From c4c8bfd0270f3d7009f0fb7c953a26e2cb65442d Mon Sep 17 00:00:00 2001 From: Pestdoktor Date: Fri, 9 Oct 2020 10:15:51 +0200 Subject: [PATCH 5/5] reject invalid UTF-8 (#1472) * reject invalid UTF-8 Signed-off-by: Jonas Fentker * update sytest-whitelist Signed-off-by: Jonas Fentker Co-authored-by: Kegsay --- clientapi/httputil/httputil.go | 20 +++++++++++++++++++- clientapi/routing/device.go | 12 +++++------- clientapi/routing/routing.go | 6 ++++-- sytest-whitelist | 1 + 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/clientapi/httputil/httputil.go b/clientapi/httputil/httputil.go index b0fe6a6cb..29d7b0b37 100644 --- a/clientapi/httputil/httputil.go +++ b/clientapi/httputil/httputil.go @@ -16,7 +16,9 @@ package httputil import ( "encoding/json" + "io/ioutil" "net/http" + "unicode/utf8" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/util" @@ -25,7 +27,23 @@ import ( // UnmarshalJSONRequest into the given interface pointer. Returns an error JSON response if // there was a problem unmarshalling. Calling this function consumes the request body. func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONResponse { - if err := json.NewDecoder(req.Body).Decode(iface); err != nil { + // encoding/json allows invalid utf-8, matrix does not + // https://matrix.org/docs/spec/client_server/r0.6.1#api-standards + body, err := ioutil.ReadAll(req.Body) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed") + resp := jsonerror.InternalServerError() + return &resp + } + + if !utf8.Valid(body) { + return &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotJSON("Body contains invalid UTF-8"), + } + } + + if err := json.Unmarshal(body, iface); err != nil { // TODO: We may want to suppress the Error() return in production? It's useful when // debugging because an error will be produced for both invalid/malformed JSON AND // valid JSON with incorrect types for values. diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 56886d57f..d50c73b35 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -15,11 +15,11 @@ package routing import ( - "encoding/json" "io/ioutil" "net/http" "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -121,9 +121,8 @@ func UpdateDeviceByID( payload := deviceUpdateJSON{} - if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("json.NewDecoder.Decode failed") - return jsonerror.InternalServerError() + if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil { + return *resErr } var performRes api.PerformDeviceUpdateResponse @@ -211,9 +210,8 @@ func DeleteDevices( ctx := req.Context() payload := devicesDeleteJSON{} - if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { - util.GetLogger(ctx).WithError(err).Error("json.NewDecoder.Decode failed") - return jsonerror.InternalServerError() + if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil { + return *resErr } defer req.Body.Close() // nolint: errcheck diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index b547efb4b..4f99237f5 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -23,6 +23,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/auth" + clientutil "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" @@ -659,8 +660,9 @@ func Setup( SearchString string `json:"search_term"` Limit int `json:"limit"` }{} - if err := json.NewDecoder(req.Body).Decode(&postContent); err != nil { - return util.ErrorResponse(err) + + if resErr := clientutil.UnmarshalJSONRequest(req, &postContent); resErr != nil { + return *resErr } return *SearchUserDirectory( req.Context(), diff --git a/sytest-whitelist b/sytest-whitelist index 420acb22c..099fc6cbd 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -477,5 +477,6 @@ Inbound federation rejects invite rejections which include invalid JSON for room GET /capabilities is present and well formed for registered user m.room.history_visibility == "joined" allows/forbids appropriately for Guest users m.room.history_visibility == "joined" allows/forbids appropriately for Real users +POST rejects invalid utf-8 in JSON Users cannot kick users who have already left a room A prev_batch token from incremental sync can be used in the v1 messages API