From 8035c50c066209108f9ccefebb9e12fdc1df3cab Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 8 Oct 2020 14:46:25 +0100 Subject: [PATCH 1/6] Don't get into situations where we have no forward extremities --- roomserver/internal/input/input_latest_events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index b6666cc03..229665a0b 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -296,7 +296,7 @@ func (u *latestEventsUpdater) calculateLatest( referenced, err := u.updater.IsReferenced(newEvent.EventReference) if err != nil { logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", newEvent.EventReference.EventID) - } else if !referenced { + } else if !referenced || len(newLatest) == 0 { newLatest = append(newLatest, newEvent) } From b12b7abcc0a61c3b422ca0f337aec7533ba7371f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 8 Oct 2020 14:53:46 +0100 Subject: [PATCH 2/6] v0.1.0 Squashed commit of the following: commit 570528e5f19d76fe2fd7ea646246eea9650b1b60 Author: Kegan Dougal Date: Thu Oct 8 14:52:10 2020 +0100 v0.1.0 commit 02c020bced3be6d1a63e1001ab9c0ef6231ba087 Merge: db840f02 8035c50c Author: Neil Alexander Date: Thu Oct 8 14:46:32 2020 +0100 Merge branch 'master' into v0.1.0 commit db840f025b5d7b7c7e1ba905646571cb03dd7b22 Merge: adc19a3d 78f6e1a3 Author: Neil Alexander Date: Thu Oct 8 13:31:36 2020 +0100 Merge branch 'master' into v0.1.0 commit adc19a3d5f0c9d5a85b3c1dd7d1772236fb8cc9b Merge: c8fc6855 3e12f6e9 Author: Neil Alexander Date: Thu Oct 8 10:31:58 2020 +0100 Merge branch 'master' into v0.1.0 commit c8fc68555c3607b0153d10df91d357d9603ccf90 Author: Neil Alexander Date: Wed Oct 7 18:41:04 2020 +0100 Version 0.1.0rc3 commit 15bf3851415dc21ebcfa98e0f2a5ec725034d6dd Merge: e7d9eea4 8bca7a83 Author: Neil Alexander Date: Wed Oct 7 18:39:25 2020 +0100 Merge branch 'master' into v0.1.0 commit e7d9eea4a09be7b05a87b1df2a9e87d3109e8fcc Author: Neil Alexander Date: Mon Oct 5 17:56:45 2020 +0100 v0.1.0rc2 commit 3fa76370f214e2ba3ec04f4c6f0f63d3baa273e7 Merge: f7cecdd9 52ddded7 Author: Neil Alexander Date: Mon Oct 5 17:56:28 2020 +0100 Merge branch 'master' into v0.1.0 commit f7cecdd9a85fe2806a99e426b806832e7036da1e Author: Kegan Dougal Date: Fri Oct 2 17:25:59 2020 +0100 Bump to 0.1.0rc1 --- internal/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/version.go b/internal/version.go index 718273e72..2ffd7c90e 100644 --- a/internal/version.go +++ b/internal/version.go @@ -10,7 +10,7 @@ var build string const ( VersionMajor = 0 - VersionMinor = 0 + VersionMinor = 1 VersionPatch = 0 VersionTag = "" // example: "rc1" ) From 009401ad4df2e8f5c0c2f08b8c73f807e99b2c13 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 8 Oct 2020 17:45:55 +0100 Subject: [PATCH 3/6] Version 0.1.0 Beta README (#1466) * Beta docs * More tweaks * More docs * Update README.md (#1497) * Call out missing features * Add CHANGES * Call out CHANGES * Update INSTALL.md * Update README.md Co-authored-by: Neil Alexander --- CHANGES.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 50 +++++++++++++++++++++++---- docs/INSTALL.md | 9 +++-- 3 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 CHANGES.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 000000000..17fb75bed --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,89 @@ +# Dendrite 0.1.0 (2020-10-08) + +First versioned release of Dendrite. + +## Client-Server API Features + +### Account registration and management +- Registration: By password only. +- Login: By password only. No fallback. +- Logout: Yes. +- Change password: Yes. +- Link email/msisdn to account: No. +- Deactivate account: Yes. +- Check if username is available: Yes. +- Account data: Yes. +- OpenID: No. + +### Rooms +- Room creation: Yes, including presets. +- Joining rooms: Yes, including by alias or `?server_name=`. +- Event sending: Yes, including transaction IDs. +- Aliases: Yes. +- Published room directory: Yes. +- Kicking users: Yes. +- Banning users: Yes. +- Inviting users: Yes, but not third-party invites. +- Forgetting rooms: No. +- Room versions: All (v1 - v6) +- Tagging: Yes. + +### User management +- User directory: Basic support. +- Ignoring users: No. +- Groups/Communities: No. + +### Device management +- Creating devices: Yes. +- Deleting devices: Yes. +- Send-to-device messaging: Yes. + +### Sync +- Filters: Timeline limit only. Rest unimplemented. +- Deprecated `/events` and `/initialSync`: No. + +### Room events +- Typing: Yes. +- Receipts: No. +- Read Markers: No. +- Presence: No. +- Content repository (attachments): Yes. +- History visibility: No, defaults to `joined`. +- Push notifications: No. +- Event context: No. +- Reporting content: No. + +### End-to-End Encryption +- Uploading device keys: Yes. +- Downloading device keys: Yes. +- Claiming one-time keys: Yes. +- Querying key changes: Yes. +- Cross-Signing: No. + +### Misc +- Server-side search: No. +- Guest access: Partial. +- Room previews: No, partial support for Peeking via MSC2753. +- Third-Party networks: No. +- Server notices: No. +- Policy lists: No. + +## Federation Features +- Querying keys (incl. notary): Yes. +- Server ACLs: Yes. +- Sending transactions: Yes. +- Joining rooms: Yes. +- Inviting to rooms: Yes, but not third-party invites. +- Leaving rooms: Yes. +- Content repository: Yes. +- Backfilling / get_missing_events: Yes. +- Retrieving state of the room (`/state` and `/state_ids`): Yes. +- Public rooms: Yes. +- Querying profile data: Yes. +- Device management: Yes. +- Send-to-Device messaging: Yes. +- Querying/Claiming E2E Keys: Yes. +- Typing: Yes. +- Presence: No. +- Receipts: No. +- OpenID: No. \ No newline at end of file diff --git a/README.md b/README.md index 72c0df07d..f27cb4029 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,28 @@ # Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) -Dendrite is a second-generation Matrix homeserver written in Go! +Dendrite is a second-generation Matrix homeserver written in Go. +It intends to provide an **efficient**, **reliable** and **scalable** alternative to Synapse: + - Efficient: A small memory footprint with better baseline performance than an out-of-the-box Synapse. + - Reliable: Implements the Matrix specification as written, using the + [same test suite](https://github.com/matrix-org/sytest) as Synapse as well as + a [brand new Go test suite](https://github.com/matrix-org/complement). + - Scalable: can run on multiple machines and eventually scale to massive homeserver deployments. + + +As of October 2020, Dendrite has now entered **beta** which means: +- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database. +- Dendrite has periodic semver releases. We intend to release new versions as we land significant features. +- Dendrite supports database schema upgrades between releases. This means you should never lose your messages when upgrading Dendrite. +- Breaking changes will not occur on minor releases. This means you can safely upgrade Dendrite without modifying your database or config file. + +This does not mean: + - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially. + - All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular, + read receipts, presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates. + - Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine. + +Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices. +In the future, we will be able to scale up to gigantic servers (equivalent to matrix.org) via polylith mode. Join us in: @@ -8,9 +30,26 @@ Join us in: - **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** - The place for developers, where all Dendrite development discussion happens - **[#dendrite-alerts:matrix.org](https://matrix.to/#/#dendrite-alerts:matrix.org)** - Release notifications and important info, highly recommended for all Dendrite server admins -## Quick start +## Requirements -Requires Go 1.13+ and SQLite3 (Postgres is also supported): +To build Dendrite, you will need Go 1.13 or later. + +For a usable federating Dendrite deployment, you will also need: +- A domain name (or subdomain) +- A valid TLS certificate issued by a trusted authority for that domain +- SRV records or a well-known file pointing to your deployment + +Also recommended are: +- A PostgreSQL database engine, which will perform better than SQLite with many users and/or larger rooms +- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/master/docs/nginx/monolith-sample.conf) + +The [Federation Tester](https://federationtester.matrix.org) can be used to verify your deployment. + +## Get started + +If you wish to build a fully-federating Dendrite instance, see [INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker). + +The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases: ```bash $ git clone https://github.com/matrix-org/dendrite @@ -30,14 +69,13 @@ $ go build ./cmd/dendrite-monolith-server $ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml ``` -Then point your favourite Matrix client at `http://localhost:8008`. For full installation information, see -[INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker). +Then point your favourite Matrix client at `http://localhost:8008`. ## Progress We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it -updates with CI. As of August 2020 we're at around 52% CS API coverage and 65% Federation coverage, though check +updates with CI. As of October 2020 we're at around 56% CS API coverage and 77% Federation coverage, though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably: - Receipts diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 913bc5832..be12d7b86 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -120,7 +120,10 @@ Assuming that Postgres 9.5 (or later) is installed: Each Dendrite server requires unique server keys. -Generate the self-signed SSL certificate for federation and the server signing key: +In order for an instance to federate correctly, you should have a valid +certificate issued by a trusted authority, and private key to match. If you +don't and just want to test locally, generate the self-signed SSL certificate +for federation and the server signing key: ```bash ./bin/generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key @@ -267,12 +270,12 @@ This manages end-to-end encryption keys for users. ./bin/dendrite-key-server --config dendrite.yaml ``` -#### Server Key server +#### Signing key server This manages signing keys for servers. ```bash -./bin/dendrite-server-key-api-server --config dendrite.yaml +./bin/dendrite-signing-key-server --config dendrite.yaml ``` #### EDU server From f3e8ae01efb0abd0904509ddaa2ae85017ca4aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFck=20Bonniot?= Date: Fri, 9 Oct 2020 10:15:35 +0200 Subject: [PATCH 4/6] Implement fully read markers (#1475) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See #653 Signed-off-by: Loïck Bonniot Co-authored-by: Kegsay --- clientapi/routing/account_data.go | 75 ++++++++++++++++++++++++++++++- clientapi/routing/routing.go | 9 ++-- sytest-whitelist | 3 ++ 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index d5fafedb1..48303c97f 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -20,8 +20,10 @@ import ( "io/ioutil" "net/http" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -91,6 +93,13 @@ func SaveAccountData( } } + if dataType == "m.fully_read" { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Unable to set read marker"), + } + } + body, err := ioutil.ReadAll(req.Body) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed") @@ -112,7 +121,7 @@ func SaveAccountData( } dataRes := api.InputAccountDataResponse{} if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed") + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") return util.ErrorResponse(err) } @@ -127,3 +136,67 @@ func SaveAccountData( JSON: struct{}{}, } } + +type readMarkerJSON struct { + FullyRead string `json:"m.fully_read"` + Read string `json:"m.read"` +} + +type fullyReadEvent struct { + EventID string `json:"event_id"` +} + +// SaveReadMarker implements POST /rooms/{roomId}/read_markers +func SaveReadMarker( + req *http.Request, userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, + syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string, +) util.JSONResponse { + // Verify that the user is a member of this room + resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + if resErr != nil { + return *resErr + } + + var r readMarkerJSON + resErr = httputil.UnmarshalJSONRequest(req, &r) + if resErr != nil { + return *resErr + } + + if r.FullyRead == "" { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"), + } + } + + data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) + } + + if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read"); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed") + return jsonerror.InternalServerError() + } + + // TODO handle the read receipt that may be included in the read marker + // See https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-rooms-roomid-read-markers + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 8606f69c3..b547efb4b 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -695,12 +695,15 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/read_markers", - httputil.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { + httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { if r := rateLimits.rateLimit(req); r != nil { return *r } - // TODO: return the read_markers. - return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}} + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SaveReadMarker(req, userAPI, rsAPI, syncProducer, device, vars["roomID"]) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/sytest-whitelist b/sytest-whitelist index e0f1f311e..420acb22c 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -456,6 +456,9 @@ After changing password, can log in with new password After changing password, existing session still works After changing password, different sessions can optionally be kept After changing password, a different session no longer works by default +Read markers appear in incremental v2 /sync +Read markers appear in initial v2 /sync +Read markers can be updated Local users can peek into world_readable rooms by room ID We can't peek into rooms with shared history_visibility We can't peek into rooms with invited history_visibility From c4c8bfd0270f3d7009f0fb7c953a26e2cb65442d Mon Sep 17 00:00:00 2001 From: Pestdoktor Date: Fri, 9 Oct 2020 10:15:51 +0200 Subject: [PATCH 5/6] reject invalid UTF-8 (#1472) * reject invalid UTF-8 Signed-off-by: Jonas Fentker * update sytest-whitelist Signed-off-by: Jonas Fentker Co-authored-by: Kegsay --- clientapi/httputil/httputil.go | 20 +++++++++++++++++++- clientapi/routing/device.go | 12 +++++------- clientapi/routing/routing.go | 6 ++++-- sytest-whitelist | 1 + 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/clientapi/httputil/httputil.go b/clientapi/httputil/httputil.go index b0fe6a6cb..29d7b0b37 100644 --- a/clientapi/httputil/httputil.go +++ b/clientapi/httputil/httputil.go @@ -16,7 +16,9 @@ package httputil import ( "encoding/json" + "io/ioutil" "net/http" + "unicode/utf8" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/util" @@ -25,7 +27,23 @@ import ( // UnmarshalJSONRequest into the given interface pointer. Returns an error JSON response if // there was a problem unmarshalling. Calling this function consumes the request body. func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONResponse { - if err := json.NewDecoder(req.Body).Decode(iface); err != nil { + // encoding/json allows invalid utf-8, matrix does not + // https://matrix.org/docs/spec/client_server/r0.6.1#api-standards + body, err := ioutil.ReadAll(req.Body) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed") + resp := jsonerror.InternalServerError() + return &resp + } + + if !utf8.Valid(body) { + return &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotJSON("Body contains invalid UTF-8"), + } + } + + if err := json.Unmarshal(body, iface); err != nil { // TODO: We may want to suppress the Error() return in production? It's useful when // debugging because an error will be produced for both invalid/malformed JSON AND // valid JSON with incorrect types for values. diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 56886d57f..d50c73b35 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -15,11 +15,11 @@ package routing import ( - "encoding/json" "io/ioutil" "net/http" "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -121,9 +121,8 @@ func UpdateDeviceByID( payload := deviceUpdateJSON{} - if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("json.NewDecoder.Decode failed") - return jsonerror.InternalServerError() + if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil { + return *resErr } var performRes api.PerformDeviceUpdateResponse @@ -211,9 +210,8 @@ func DeleteDevices( ctx := req.Context() payload := devicesDeleteJSON{} - if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { - util.GetLogger(ctx).WithError(err).Error("json.NewDecoder.Decode failed") - return jsonerror.InternalServerError() + if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil { + return *resErr } defer req.Body.Close() // nolint: errcheck diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index b547efb4b..4f99237f5 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -23,6 +23,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/auth" + clientutil "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" @@ -659,8 +660,9 @@ func Setup( SearchString string `json:"search_term"` Limit int `json:"limit"` }{} - if err := json.NewDecoder(req.Body).Decode(&postContent); err != nil { - return util.ErrorResponse(err) + + if resErr := clientutil.UnmarshalJSONRequest(req, &postContent); resErr != nil { + return *resErr } return *SearchUserDirectory( req.Context(), diff --git a/sytest-whitelist b/sytest-whitelist index 420acb22c..099fc6cbd 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -477,5 +477,6 @@ Inbound federation rejects invite rejections which include invalid JSON for room GET /capabilities is present and well formed for registered user m.room.history_visibility == "joined" allows/forbids appropriately for Guest users m.room.history_visibility == "joined" allows/forbids appropriately for Real users +POST rejects invalid utf-8 in JSON Users cannot kick users who have already left a room A prev_batch token from incremental sync can be used in the v1 messages API From 1cd525ef0d7b6209232f93c4d0527a0fccfcdb4f Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 9 Oct 2020 10:17:23 +0200 Subject: [PATCH 6/6] Extend device_devices table (#1471) * Add last_used_ts and IP to database * Add migrations * Rename column Prepare statements * Add interface method and implement it Signed-off-by: Till Faelligen * Rename struct fields * Add user_agent to database * Add userAgent to registration calls * Add missing "IF NOT EXISTS" * use txn writer * Add UserAgent to Device Co-authored-by: Kegsay --- clientapi/routing/login.go | 5 ++- clientapi/routing/register.go | 14 +++--- cmd/create-account/main.go | 2 +- userapi/api/api.go | 7 +++ userapi/internal/api.go | 2 +- userapi/storage/devices/interface.go | 3 +- .../deltas/20201001204705_last_seen_ts_ip.sql | 13 ++++++ .../storage/devices/postgres/devices_table.go | 34 +++++++++++--- userapi/storage/devices/postgres/storage.go | 13 ++++-- .../deltas/20201001204705_last_seen_ts_ip.sql | 44 +++++++++++++++++++ .../storage/devices/sqlite3/devices_table.go | 28 ++++++++++-- userapi/storage/devices/sqlite3/storage.go | 13 ++++-- 12 files changed, 154 insertions(+), 24 deletions(-) create mode 100644 userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql create mode 100644 userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 772775aa0..f84f078dd 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -79,7 +79,7 @@ func Login( return *authErr } // make a device/access token - return completeAuth(req.Context(), cfg.Matrix.ServerName, userAPI, login) + return completeAuth(req.Context(), cfg.Matrix.ServerName, userAPI, login, req.RemoteAddr, req.UserAgent()) } return util.JSONResponse{ Code: http.StatusMethodNotAllowed, @@ -89,6 +89,7 @@ func Login( func completeAuth( ctx context.Context, serverName gomatrixserverlib.ServerName, userAPI userapi.UserInternalAPI, login *auth.Login, + ipAddr, userAgent string, ) util.JSONResponse { token, err := auth.GenerateAccessToken() if err != nil { @@ -108,6 +109,8 @@ func completeAuth( DeviceID: login.DeviceID, AccessToken: token, Localpart: localpart, + IPAddr: ipAddr, + UserAgent: userAgent, }, &performRes) if err != nil { return util.JSONResponse{ diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 937abc83d..756eafe2f 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -543,6 +543,8 @@ func handleGuestRegistration( Localpart: res.Account.Localpart, DeviceDisplayName: r.InitialDisplayName, AccessToken: token, + IPAddr: req.RemoteAddr, + UserAgent: req.UserAgent(), }, &devRes) if err != nil { return util.JSONResponse{ @@ -691,7 +693,7 @@ func handleApplicationServiceRegistration( // Don't need to worry about appending to registration stages as // application service registration is entirely separate. return completeRegistration( - req.Context(), userAPI, r.Username, "", appserviceID, + req.Context(), userAPI, r.Username, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.InhibitLogin, r.InitialDisplayName, r.DeviceID, ) } @@ -710,7 +712,7 @@ func checkAndCompleteFlow( if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { // This flow was completed, registration can continue return completeRegistration( - req.Context(), userAPI, r.Username, r.Password, "", + req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), r.InhibitLogin, r.InitialDisplayName, r.DeviceID, ) } @@ -762,10 +764,10 @@ func LegacyRegister( return util.MessageResponse(http.StatusForbidden, "HMAC incorrect") } - return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil) + return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), false, nil, nil) case authtypes.LoginTypeDummy: // there is nothing to do - return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil) + return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), false, nil, nil) default: return util.JSONResponse{ Code: http.StatusNotImplemented, @@ -812,7 +814,7 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u func completeRegistration( ctx context.Context, userAPI userapi.UserInternalAPI, - username, password, appserviceID string, + username, password, appserviceID, ipAddr, userAgent string, inhibitLogin eventutil.WeakBoolean, displayName, deviceID *string, ) util.JSONResponse { @@ -880,6 +882,8 @@ func completeRegistration( AccessToken: token, DeviceDisplayName: displayName, DeviceID: deviceID, + IPAddr: ipAddr, + UserAgent: userAgent, }, &devRes) if err != nil { return util.JSONResponse{ diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index 73e223d61..a9bd92794 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -92,7 +92,7 @@ func main() { } device, err := deviceDB.CreateDevice( - context.Background(), *username, nil, *accessToken, nil, + context.Background(), *username, nil, *accessToken, nil, "127.0.0.1", "", ) if err != nil { fmt.Println(err.Error()) diff --git a/userapi/api/api.go b/userapi/api/api.go index d384c5b18..6c3f3c69c 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -192,6 +192,10 @@ type PerformDeviceCreationRequest struct { DeviceID *string // optional: if nil no display name will be associated with this device. DeviceDisplayName *string + // IP address of this device + IPAddr string + // Useragent for this device + UserAgent string } // PerformDeviceCreationResponse is the response for PerformDeviceCreation @@ -222,6 +226,9 @@ type Device struct { // associated with access tokens. SessionID int64 DisplayName string + LastSeenTS int64 + LastSeenIP string + UserAgent string } // Account represents a Matrix account on this home server. diff --git a/userapi/internal/api.go b/userapi/internal/api.go index ec8284397..81d002414 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -113,7 +113,7 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe "device_id": req.DeviceID, "display_name": req.DeviceDisplayName, }).Info("PerformDeviceCreation") - dev, err := a.DeviceDB.CreateDevice(ctx, req.Localpart, req.DeviceID, req.AccessToken, req.DeviceDisplayName) + dev, err := a.DeviceDB.CreateDevice(ctx, req.Localpart, req.DeviceID, req.AccessToken, req.DeviceDisplayName, req.IPAddr, req.UserAgent) if err != nil { return err } diff --git a/userapi/storage/devices/interface.go b/userapi/storage/devices/interface.go index 168c84c5c..9953ba062 100644 --- a/userapi/storage/devices/interface.go +++ b/userapi/storage/devices/interface.go @@ -31,10 +31,11 @@ type Database interface { // an error will be returned. // If no device ID is given one is generated. // Returns the device on success. - CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *api.Device, returnErr error) + CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string) (dev *api.Device, returnErr error) UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error RemoveDevice(ctx context.Context, deviceID, localpart string) error RemoveDevices(ctx context.Context, localpart string, devices []string) error // RemoveAllDevices deleted all devices for this user. Returns the devices deleted. RemoveAllDevices(ctx context.Context, localpart, exceptDeviceID string) (devices []api.Device, err error) + UpdateDeviceLastSeen(ctx context.Context, deviceID, ipAddr string) error } diff --git a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql new file mode 100644 index 000000000..4f5f2b172 --- /dev/null +++ b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql @@ -0,0 +1,13 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL; +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT; +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE device_devices DROP COLUMN last_seen_ts; +ALTER TABLE device_devices DROP COLUMN ip; +ALTER TABLE device_devices DROP COLUMN user_agent; +-- +goose StatementEnd diff --git a/userapi/storage/devices/postgres/devices_table.go b/userapi/storage/devices/postgres/devices_table.go index c06af7549..2a4d337c7 100644 --- a/userapi/storage/devices/postgres/devices_table.go +++ b/userapi/storage/devices/postgres/devices_table.go @@ -51,8 +51,15 @@ CREATE TABLE IF NOT EXISTS device_devices ( -- When this devices was first recognised on the network, as a unix timestamp (ms resolution). created_ts BIGINT NOT NULL, -- The display name, human friendlier than device_id and updatable - display_name TEXT - -- TODO: device keys, device display names, last used ts and IP address?, token restrictions (if 3rd-party OAuth app) + display_name TEXT, + -- The time the device was last used, as a unix timestamp (ms resolution). + last_seen_ts BIGINT NOT NULL, + -- The last seen IP address of this device + ip TEXT, + -- User agent of this device + user_agent TEXT + + -- TODO: device keys, device display names, token restrictions (if 3rd-party OAuth app) ); -- Device IDs must be unique for a given user. @@ -60,7 +67,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(loca ` const insertDeviceSQL = "" + - "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" + + "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name, last_seen_ts, ip, user_agent) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" + " RETURNING session_id" const selectDeviceByTokenSQL = "" + @@ -87,6 +94,9 @@ const deleteDevicesSQL = "" + const selectDevicesByIDSQL = "" + "SELECT device_id, localpart, display_name FROM device_devices WHERE device_id = ANY($1)" +const updateDeviceLastSeen = "" + + "UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE device_id = $3" + type devicesStatements struct { insertDeviceStmt *sql.Stmt selectDeviceByTokenStmt *sql.Stmt @@ -94,6 +104,7 @@ type devicesStatements struct { selectDevicesByLocalpartStmt *sql.Stmt selectDevicesByIDStmt *sql.Stmt updateDeviceNameStmt *sql.Stmt + updateDeviceLastSeenStmt *sql.Stmt deleteDeviceStmt *sql.Stmt deleteDevicesByLocalpartStmt *sql.Stmt deleteDevicesStmt *sql.Stmt @@ -132,6 +143,9 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN if s.selectDevicesByIDStmt, err = db.Prepare(selectDevicesByIDSQL); err != nil { return } + if s.updateDeviceLastSeenStmt, err = db.Prepare(updateDeviceLastSeen); err != nil { + return + } s.serverName = server return } @@ -141,12 +155,12 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN // Returns the device on success. func (s *devicesStatements) insertDevice( ctx context.Context, txn *sql.Tx, id, localpart, accessToken string, - displayName *string, + displayName *string, ipAddr, userAgent string, ) (*api.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 stmt := sqlutil.TxStmt(txn, s.insertDeviceStmt) - if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName).Scan(&sessionID); err != nil { + if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName, createdTimeMS, ipAddr, userAgent).Scan(&sessionID); err != nil { return nil, err } return &api.Device{ @@ -154,6 +168,9 @@ func (s *devicesStatements) insertDevice( UserID: userutil.MakeUserID(localpart, s.serverName), AccessToken: accessToken, SessionID: sessionID, + LastSeenTS: createdTimeMS, + LastSeenIP: ipAddr, + UserAgent: userAgent, }, nil } @@ -280,3 +297,10 @@ func (s *devicesStatements) selectDevicesByLocalpart( return devices, rows.Err() } + +func (s *devicesStatements) updateDeviceLastSeen(ctx context.Context, txn *sql.Tx, deviceID, ipAddr string) error { + lastSeenTs := time.Now().UnixNano() / 1000000 + stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt) + _, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, deviceID) + return err +} diff --git a/userapi/storage/devices/postgres/storage.go b/userapi/storage/devices/postgres/storage.go index c5bd5b6cf..faa5796b0 100644 --- a/userapi/storage/devices/postgres/storage.go +++ b/userapi/storage/devices/postgres/storage.go @@ -83,7 +83,7 @@ func (d *Database) GetDevicesByID(ctx context.Context, deviceIDs []string) ([]ap // Returns the device on success. func (d *Database) CreateDevice( ctx context.Context, localpart string, deviceID *string, accessToken string, - displayName *string, + displayName *string, ipAddr, userAgent string, ) (dev *api.Device, returnErr error) { if deviceID != nil { returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { @@ -93,7 +93,7 @@ func (d *Database) CreateDevice( return err } - dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName) + dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName, ipAddr, userAgent) return err }) } else { @@ -108,7 +108,7 @@ func (d *Database) CreateDevice( returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error - dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) + dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName, ipAddr, userAgent) return err }) if returnErr == nil { @@ -189,3 +189,10 @@ func (d *Database) RemoveAllDevices( }) return } + +// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address +func (d *Database) UpdateDeviceLastSeen(ctx context.Context, deviceID, ipAddr string) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.devices.updateDeviceLastSeen(ctx, txn, deviceID, ipAddr) + }) +} diff --git a/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql new file mode 100644 index 000000000..887f90e0d --- /dev/null +++ b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql @@ -0,0 +1,44 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE device_devices RENAME TO device_devices_tmp; +CREATE TABLE device_devices ( + access_token TEXT PRIMARY KEY, + session_id INTEGER, + device_id TEXT , + localpart TEXT , + created_ts BIGINT, + display_name TEXT, + last_seen_ts BIGINT, + ip TEXT, + user_agent TEXT, + UNIQUE (localpart, device_id) +); +INSERT +INTO device_devices ( + access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent +) SELECT + access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', '' +FROM device_devices_tmp; +DROP TABLE device_devices_tmp; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE device_devices RENAME TO device_devices_tmp; +CREATE TABLE IF NOT EXISTS device_devices ( + access_token TEXT PRIMARY KEY, + session_id INTEGER, + device_id TEXT , + localpart TEXT , + created_ts BIGINT, + display_name TEXT, + UNIQUE (localpart, device_id) +); +INSERT +INTO device_devices ( + access_token, session_id, device_id, localpart, created_ts, display_name +) SELECT + access_token, session_id, device_id, localpart, created_ts, display_name +FROM device_devices_tmp; +DROP TABLE device_devices_tmp; +-- +goose StatementEnd \ No newline at end of file diff --git a/userapi/storage/devices/sqlite3/devices_table.go b/userapi/storage/devices/sqlite3/devices_table.go index c75e19825..6b0de10ee 100644 --- a/userapi/storage/devices/sqlite3/devices_table.go +++ b/userapi/storage/devices/sqlite3/devices_table.go @@ -40,14 +40,17 @@ CREATE TABLE IF NOT EXISTS device_devices ( localpart TEXT , created_ts BIGINT, display_name TEXT, + last_seen_ts BIGINT, + ip TEXT, + user_agent TEXT, UNIQUE (localpart, device_id) ); ` const insertDeviceSQL = "" + - "INSERT INTO device_devices (device_id, localpart, access_token, created_ts, display_name, session_id)" + - " VALUES ($1, $2, $3, $4, $5, $6)" + "INSERT INTO device_devices (device_id, localpart, access_token, created_ts, display_name, session_id, last_seen_ts, ip, user_agent)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" const selectDevicesCountSQL = "" + "SELECT COUNT(access_token) FROM device_devices" @@ -76,6 +79,9 @@ const deleteDevicesSQL = "" + const selectDevicesByIDSQL = "" + "SELECT device_id, localpart, display_name FROM device_devices WHERE device_id IN ($1)" +const updateDeviceLastSeen = "" + + "UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE device_id = $3" + type devicesStatements struct { db *sql.DB writer sqlutil.Writer @@ -86,6 +92,7 @@ type devicesStatements struct { selectDevicesByIDStmt *sql.Stmt selectDevicesByLocalpartStmt *sql.Stmt updateDeviceNameStmt *sql.Stmt + updateDeviceLastSeenStmt *sql.Stmt deleteDeviceStmt *sql.Stmt deleteDevicesByLocalpartStmt *sql.Stmt serverName gomatrixserverlib.ServerName @@ -125,6 +132,9 @@ func (s *devicesStatements) prepare(db *sql.DB, writer sqlutil.Writer, server go if s.selectDevicesByIDStmt, err = db.Prepare(selectDevicesByIDSQL); err != nil { return } + if s.updateDeviceLastSeenStmt, err = db.Prepare(updateDeviceLastSeen); err != nil { + return + } s.serverName = server return } @@ -134,7 +144,7 @@ func (s *devicesStatements) prepare(db *sql.DB, writer sqlutil.Writer, server go // Returns the device on success. func (s *devicesStatements) insertDevice( ctx context.Context, txn *sql.Tx, id, localpart, accessToken string, - displayName *string, + displayName *string, ipAddr, userAgent string, ) (*api.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 @@ -144,7 +154,7 @@ func (s *devicesStatements) insertDevice( return nil, err } sessionID++ - if _, err := insertStmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName, sessionID); err != nil { + if _, err := insertStmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName, sessionID, createdTimeMS, ipAddr, userAgent); err != nil { return nil, err } return &api.Device{ @@ -152,6 +162,9 @@ func (s *devicesStatements) insertDevice( UserID: userutil.MakeUserID(localpart, s.serverName), AccessToken: accessToken, SessionID: sessionID, + LastSeenTS: createdTimeMS, + LastSeenIP: ipAddr, + UserAgent: userAgent, }, nil } @@ -288,3 +301,10 @@ func (s *devicesStatements) selectDevicesByID(ctx context.Context, deviceIDs []s } return devices, rows.Err() } + +func (s *devicesStatements) updateDeviceLastSeen(ctx context.Context, txn *sql.Tx, deviceID, ipAddr string) error { + lastSeenTs := time.Now().UnixNano() / 1000000 + stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt) + _, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, deviceID) + return err +} diff --git a/userapi/storage/devices/sqlite3/storage.go b/userapi/storage/devices/sqlite3/storage.go index 7c6645dd6..cfaf4fd99 100644 --- a/userapi/storage/devices/sqlite3/storage.go +++ b/userapi/storage/devices/sqlite3/storage.go @@ -87,7 +87,7 @@ func (d *Database) GetDevicesByID(ctx context.Context, deviceIDs []string) ([]ap // Returns the device on success. func (d *Database) CreateDevice( ctx context.Context, localpart string, deviceID *string, accessToken string, - displayName *string, + displayName *string, ipAddr, userAgent string, ) (dev *api.Device, returnErr error) { if deviceID != nil { returnErr = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { @@ -97,7 +97,7 @@ func (d *Database) CreateDevice( return err } - dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName) + dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName, ipAddr, userAgent) return err }) } else { @@ -112,7 +112,7 @@ func (d *Database) CreateDevice( returnErr = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { var err error - dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) + dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName, ipAddr, userAgent) return err }) if returnErr == nil { @@ -193,3 +193,10 @@ func (d *Database) RemoveAllDevices( }) return } + +// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address +func (d *Database) UpdateDeviceLastSeen(ctx context.Context, deviceID, ipAddr string) error { + return d.writer.Do(d.db, nil, func(txn *sql.Tx) error { + return d.devices.updateDeviceLastSeen(ctx, txn, deviceID, ipAddr) + }) +}