From 980fa55846811eeff89f116c49b38b085143c64e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 10:39:29 +0100 Subject: [PATCH 01/14] Stronger passwordless account checks (fixes #2780) --- userapi/internal/api.go | 2 ++ userapi/storage/shared/storage.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 591faffd6..2f7795dfe 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -838,6 +838,8 @@ func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.Q return nil case bcrypt.ErrMismatchedHashAndPassword: // user exists, but password doesn't match return nil + case bcrypt.ErrHashTooShort: // user exists, but probably a passwordless account + return nil default: res.Exists = true res.Account = acc diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 3ff299f1b..09eeedc9f 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -75,6 +75,9 @@ func (d *Database) GetAccountByPassword( if err != nil { return nil, err } + if hash == "" { + return nil, bcrypt.ErrHashTooShort + } if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil { return nil, err } From 04bab142901b9dc4479b2cf4e5504fbdd9dd3cf1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 10:45:15 +0100 Subject: [PATCH 02/14] Add regression test for 980fa55846811eeff89f116c49b38b085143c64e --- userapi/userapi_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 31a69793b..984fe8854 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -151,6 +151,33 @@ func TestQueryProfile(t *testing.T) { }) } +// TestPasswordlessLoginFails ensures that a passwordless account cannot +// be logged into using an arbitrary password (effectively a regression test +// for https://github.com/matrix-org/dendrite/issues/2780). +func TestPasswordlessLoginFails(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType) + defer close() + _, err := accountDB.CreateAccount(ctx, "auser", "", "", api.AccountTypeAppService) + if err != nil { + t.Fatalf("failed to make account: %s", err) + } + + userReq := &api.QueryAccountByPasswordRequest{ + Localpart: "auser", + PlaintextPassword: "apassword", + } + userRes := &api.QueryAccountByPasswordResponse{} + if err := userAPI.QueryAccountByPassword(ctx, userReq, userRes); err != nil { + t.Fatal(err) + } + if userRes.Exists || userRes.Account != nil { + t.Fatal("QueryAccountByPassword should not return correctly for a passwordless account") + } + }) +} + func TestLoginToken(t *testing.T) { ctx := context.Background() From b32b6d6e8eb5d0a6c22e9215e2fccf4bdd1db8c3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 11:03:52 +0100 Subject: [PATCH 03/14] Update issue and pull request templates --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 22 ++++++++++++---------- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index 206713e04..49c9a37ef 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -7,24 +7,27 @@ about: Create a report to help us improve ### Background information -- **Dendrite version or git SHA**: -- **Monolith or Polylith?**: -- **SQLite3 or Postgres?**: -- **Running in Docker?**: +- **Dendrite version or git SHA**: +- **Monolith or Polylith?**: +- **SQLite3 or Postgres?**: +- **Running in Docker?**: - **`go version`**: - **Client used (if applicable)**: - ### Description - - **What** is the problem: - - **Who** is affected: - - **How** is this bug manifesting: - - **When** did this first appear: +- **What** is the problem: +- **Who** is affected: +- **How** is this bug manifesting: +- **When** did this first appear: + * [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests. -* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off) +* [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Your Name ` From 80a0ab6246aa095f428430c38b13861406dd5c78 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 11:09:40 +0100 Subject: [PATCH 04/14] Further tweak to the issue template --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index 49c9a37ef..f40c56609 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -8,9 +8,10 @@ about: Create a report to help us improve All bug reports must provide the following background information Text between ### Background information From fb6cb2dbcbeb7cd7546ca4d126394720d215c310 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 11:14:16 +0100 Subject: [PATCH 05/14] Tweak `GetAccountByPassword` more --- clientapi/auth/password.go | 6 ++++++ userapi/storage/shared/storage.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index bcb4ca97b..890b18183 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -68,6 +68,12 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.BadJSON("A username must be supplied."), } } + if len(r.Password) == 0 { + return nil, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.BadJSON("A password must be supplied."), + } + } localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName) if err != nil { return nil, &util.JSONResponse{ diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 09eeedc9f..4e28f7b5a 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -75,7 +75,7 @@ func (d *Database) GetAccountByPassword( if err != nil { return nil, err } - if hash == "" { + if len(hash) == 0 && len(plaintextPassword) > 0 { return nil, bcrypt.ErrHashTooShort } if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil { From 0f09e9d196d3375e38a490881e06668a82fb6c40 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 12:19:16 +0200 Subject: [PATCH 06/14] Move /event to the SyncAPI (#2782) This allows us to apply history visibility without having to recalculate it in the roomserver. Unblocks https://github.com/matrix-org/complement/pull/495, fix missing part of https://github.com/matrix-org/dendrite/issues/617 --- clientapi/routing/getevent.go | 138 ------------------------- clientapi/routing/routing.go | 9 -- docs/caddy/polylith/Caddyfile | 2 +- docs/hiawatha/polylith-sample.conf | 4 +- docs/nginx/polylith-sample.conf | 4 +- syncapi/internal/history_visibility.go | 8 +- syncapi/routing/getevent.go | 102 ++++++++++++++++++ syncapi/routing/routing.go | 10 ++ 8 files changed, 124 insertions(+), 153 deletions(-) delete mode 100644 clientapi/routing/getevent.go create mode 100644 syncapi/routing/getevent.go diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go deleted file mode 100644 index 7f5842800..000000000 --- a/clientapi/routing/getevent.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2019 Alex Chen -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "net/http" - - "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/config" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -type getEventRequest struct { - req *http.Request - device *userapi.Device - roomID string - eventID string - cfg *config.ClientAPI - requestedEvent *gomatrixserverlib.Event -} - -// GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId} -// https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid -func GetEvent( - req *http.Request, - device *userapi.Device, - roomID string, - eventID string, - cfg *config.ClientAPI, - rsAPI api.ClientRoomserverAPI, -) util.JSONResponse { - eventsReq := api.QueryEventsByIDRequest{ - EventIDs: []string{eventID}, - } - var eventsResp api.QueryEventsByIDResponse - err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed") - return jsonerror.InternalServerError() - } - - if len(eventsResp.Events) == 0 { - // Event not found locally - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), - } - } - - requestedEvent := eventsResp.Events[0].Event - - r := getEventRequest{ - req: req, - device: device, - roomID: roomID, - eventID: eventID, - cfg: cfg, - requestedEvent: requestedEvent, - } - - stateReq := api.QueryStateAfterEventsRequest{ - RoomID: r.requestedEvent.RoomID(), - PrevEventIDs: r.requestedEvent.PrevEventIDs(), - StateToFetch: []gomatrixserverlib.StateKeyTuple{{ - EventType: gomatrixserverlib.MRoomMember, - StateKey: device.UserID, - }}, - } - var stateResp api.QueryStateAfterEventsResponse - if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed") - return jsonerror.InternalServerError() - } - - if !stateResp.RoomExists { - util.GetLogger(req.Context()).Errorf("Expected to find room for event %s but failed", r.requestedEvent.EventID()) - return jsonerror.InternalServerError() - } - - if !stateResp.PrevEventsExist { - // Missing some events locally; stateResp.StateEvents unavailable. - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), - } - } - - var appService *config.ApplicationService - if device.AppserviceID != "" { - for _, as := range cfg.Derived.ApplicationServices { - if as.ID == device.AppserviceID { - appService = &as - break - } - } - } - - for _, stateEvent := range stateResp.StateEvents { - if appService != nil { - if !appService.IsInterestedInUserID(*stateEvent.StateKey()) { - continue - } - } else if !stateEvent.StateKeyEquals(device.UserID) { - continue - } - membership, err := stateEvent.Membership() - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("stateEvent.Membership failed") - return jsonerror.InternalServerError() - } - if membership == gomatrixserverlib.Join { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: gomatrixserverlib.ToClientEvent(r.requestedEvent, gomatrixserverlib.FormatAll), - } - } - } - - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), - } -} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 7d1c434c4..f1fa66ca6 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -367,15 +367,6 @@ func Setup( nil, cfg, rsAPI, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) - v3mux.Handle("/rooms/{roomID}/event/{eventID}", - httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI) - }), - ).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile index 244e50e7e..906097e4e 100644 --- a/docs/caddy/polylith/Caddyfile +++ b/docs/caddy/polylith/Caddyfile @@ -55,7 +55,7 @@ matrix.example.com { # Change the end of each reverse_proxy line to the correct # address for your various services. @sync_api { - path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ + path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ } reverse_proxy @sync_api sync_api:8073 diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf index 5ed0cb5ae..036140643 100644 --- a/docs/hiawatha/polylith-sample.conf +++ b/docs/hiawatha/polylith-sample.conf @@ -18,8 +18,10 @@ VirtualHost { # /_matrix/client/.*/user/{userId}/filter/{filterID} # /_matrix/client/.*/keys/changes # /_matrix/client/.*/rooms/{roomId}/messages + # /_matrix/client/.*/rooms/{roomId}/context/{eventID} + # /_matrix/client/.*/rooms/{roomId}/event/{eventID} # to sync_api - ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 600 + ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ http://localhost:8073 600 ReverseProxy = /_matrix/client http://localhost:8071 600 ReverseProxy = /_matrix/federation http://localhost:8072 600 ReverseProxy = /_matrix/key http://localhost:8072 600 diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index 274d75658..345d8a6b4 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -28,8 +28,10 @@ server { # /_matrix/client/.*/user/{userId}/filter/{filterID} # /_matrix/client/.*/keys/changes # /_matrix/client/.*/rooms/{roomId}/messages + # /_matrix/client/.*/rooms/{roomId}/context/{eventID} + # /_matrix/client/.*/rooms/{roomId}/event/{eventID} # to sync_api - location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ { + location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ { proxy_pass http://sync_api:8073; } diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go index bbfe19f4c..71d7ddd15 100644 --- a/syncapi/internal/history_visibility.go +++ b/syncapi/internal/history_visibility.go @@ -19,11 +19,13 @@ import ( "math" "time" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage" ) func init() { @@ -189,7 +191,7 @@ func visibilityForEvents( UserID: userID, }, membershipResp) if err != nil { - return result, err + logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'") } // Create a map from eventID -> eventVisibility diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go new file mode 100644 index 000000000..d2cdc1b5f --- /dev/null +++ b/syncapi/routing/getevent.go @@ -0,0 +1,102 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/internal" + "github.com/matrix-org/dendrite/syncapi/storage" + userapi "github.com/matrix-org/dendrite/userapi/api" +) + +// GetEvent implements +// +// GET /_matrix/client/r0/rooms/{roomId}/event/{eventId} +// +// https://spec.matrix.org/v1.4/client-server-api/#get_matrixclientv3roomsroomideventeventid +func GetEvent( + req *http.Request, + device *userapi.Device, + roomID string, + eventID string, + cfg *config.SyncAPI, + syncDB storage.Database, + rsAPI api.SyncRoomserverAPI, +) util.JSONResponse { + ctx := req.Context() + db, err := syncDB.NewDatabaseTransaction(ctx) + logger := util.GetLogger(ctx).WithFields(logrus.Fields{ + "event_id": eventID, + "room_id": roomID, + }) + if err != nil { + logger.WithError(err).Error("GetEvent: syncDB.NewDatabaseTransaction failed") + return jsonerror.InternalServerError() + } + + events, err := db.Events(ctx, []string{eventID}) + if err != nil { + logger.WithError(err).Error("GetEvent: syncDB.Events failed") + return jsonerror.InternalServerError() + } + + // The requested event does not exist in our database + if len(events) == 0 { + logger.Debugf("GetEvent: requested event doesn't exist locally") + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + // If the request is coming from an appservice, get the user from the request + userID := device.UserID + if asUserID := req.FormValue("user_id"); device.AppserviceID != "" && asUserID != "" { + userID = asUserID + } + + // Apply history visibility to determine if the user is allowed to view the event + events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, userID, "event") + if err != nil { + logger.WithError(err).Error("GetEvent: internal.ApplyHistoryVisibilityFilter failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + + // We only ever expect there to be one event + if len(events) != 1 { + // 0 events -> not allowed to view event; > 1 events -> something that shouldn't happen + logger.WithField("event_count", len(events)).Debug("GetEvent: can't return the requested event") + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: gomatrixserverlib.HeaderedToClientEvent(events[0], gomatrixserverlib.FormatAll), + } +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 8f84a1341..069dee81f 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -60,6 +60,16 @@ func Setup( return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, rsAPI, cfg, srp, lazyLoadCache) })).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/rooms/{roomID}/event/{eventID}", + httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, syncDB, rsAPI) + }), + ).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/user/{userId}/filter", httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) From dcc01162874d2634738080614f02cbbeb6bc598f Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 15:38:00 +0200 Subject: [PATCH 07/14] SyTest List Maintenance --- sytest-blacklist | 1 - sytest-whitelist | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/sytest-blacklist b/sytest-blacklist index 5b2e973a6..634c07cf3 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -47,7 +47,6 @@ Notifications can be viewed with GET /notifications # More flakey -If remote user leaves room we no longer receive device updates Guest users can join guest_access rooms # This will fail in HTTP API mode, so blacklisted for now diff --git a/sytest-whitelist b/sytest-whitelist index 31940b884..9ba9df75d 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -742,3 +742,4 @@ User in private room doesn't appear in user directory User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave +Existing members see new member's presence \ No newline at end of file From 39581af3ba657032fbf66ba3719a1cb334c0519b Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 15:49:56 +0200 Subject: [PATCH 08/14] CI update --- .github/workflows/dendrite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index be3c7c173..f8019b3ea 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -342,7 +342,7 @@ jobs: # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest + go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - name: Run actions/checkout@v2 for dendrite uses: actions/checkout@v2 From b000db81ca889b350663493c27eb58654c75e295 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:36:26 +0200 Subject: [PATCH 09/14] Send E2EE related errors to sentry (#2784) Only sends errors if we're not retrying them in NATS. Not sure if those should be scoped/tagged with something like "E2EE". --- federationapi/consumers/keychange.go | 17 ++++++++++++++--- federationapi/consumers/sendtodevice.go | 14 ++++++++++---- federationapi/queue/queue.go | 3 +++ federationapi/routing/send.go | 4 ++++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index f3314bc98..67dfdc1d3 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -18,6 +18,11 @@ import ( "context" "encoding/json" + "github.com/getsentry/sentry-go" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/types" @@ -26,9 +31,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/gomatrixserverlib" - "github.com/nats-io/nats.go" - "github.com/sirupsen/logrus" ) // KeyChangeConsumer consumes events that originate in key server. @@ -78,6 +80,7 @@ func (t *KeyChangeConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) boo msg := msgs[0] // Guaranteed to exist if onMessage is called var m api.DeviceMessage if err := json.Unmarshal(msg.Data, &m); err != nil { + sentry.CaptureException(err) logrus.WithError(err).Errorf("failed to read device message from key change topic") return true } @@ -105,6 +108,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { // only send key change events which originated from us _, originServerName, err := gomatrixserverlib.SplitID('@', m.UserID) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("Failed to extract domain from key change event") return true } @@ -118,6 +122,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { WantMembership: "join", }, &queryRes) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined rooms for user") return true } @@ -125,6 +130,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { // send this key change to all servers who share rooms with this user. destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in") return true } @@ -147,6 +153,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { Keys: m.KeyJSON, } if edu.Content, err = json.Marshal(event); err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("failed to marshal EDU JSON") return true } @@ -160,6 +167,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { output := m.CrossSigningKeyUpdate _, host, err := gomatrixserverlib.SplitID('@', output.UserID) if err != nil { + sentry.CaptureException(err) logrus.WithError(err).Errorf("fedsender key change consumer: user ID parse failure") return true } @@ -176,12 +184,14 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { WantMembership: "join", }, &queryRes) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user") return true } // send this key change to all servers who share rooms with this user. destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in") return true } @@ -196,6 +206,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { Origin: string(t.serverName), } if edu.Content, err = json.Marshal(output); err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to marshal output, dropping") return true } diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go index ffc1d8894..9aec22a3e 100644 --- a/federationapi/consumers/sendtodevice.go +++ b/federationapi/consumers/sendtodevice.go @@ -18,16 +18,18 @@ import ( "context" "encoding/json" + "github.com/getsentry/sentry-go" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" syncTypes "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/nats-io/nats.go" - log "github.com/sirupsen/logrus" ) // OutputSendToDeviceConsumer consumes events that originate in the clientapi. @@ -76,6 +78,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats sender := msg.Header.Get("sender") _, originServerName, err := gomatrixserverlib.SplitID('@', sender) if err != nil { + sentry.CaptureException(err) log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender") return true } @@ -85,12 +88,14 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats // Extract the send-to-device event from msg. var ote syncTypes.OutputSendToDeviceEvent if err = json.Unmarshal(msg.Data, &ote); err != nil { + sentry.CaptureException(err) log.WithError(err).Errorf("output log: message parse failed (expected send-to-device)") return true } _, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID) if err != nil { + sentry.CaptureException(err) log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination") return true } @@ -116,6 +121,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats }, } if edu.Content, err = json.Marshal(tdm); err != nil { + sentry.CaptureException(err) log.WithError(err).Error("failed to marshal EDU JSON") return true } diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 88664fcf9..8245aa5bd 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -307,11 +308,13 @@ func (oqs *OutgoingQueues) SendEDU( ephemeralJSON, err := json.Marshal(e) if err != nil { + sentry.CaptureException(err) return fmt.Errorf("json.Marshal: %w", err) } nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(ephemeralJSON)) if err != nil { + sentry.CaptureException(err) return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 060af676d..b3bbaa394 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -350,6 +351,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { for deviceID, message := range byUser { // TODO: check that the user and the device actually exist here if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { + sentry.CaptureException(err) util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ "sender": directPayload.Sender, "user_id": userID, @@ -360,6 +362,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { } case gomatrixserverlib.MDeviceListUpdate: if err := t.producer.SendDeviceListUpdate(ctx, e.Content, t.Origin); err != nil { + sentry.CaptureException(err) util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate") } case gomatrixserverlib.MReceipt: @@ -395,6 +398,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { } case types.MSigningKeyUpdate: if err := t.producer.SendSigningKeyUpdate(ctx, e.Content, t.Origin); err != nil { + sentry.CaptureException(err) logrus.WithError(err).Errorf("Failed to process signing key update") } case gomatrixserverlib.MPresence: From 6bf1912525724baa1a8bf5e0bf51524d7cee59ba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 16:54:04 +0100 Subject: [PATCH 10/14] Fix joined hosts with `RewritesState` (#2785) This ensures that the joined hosts in the federation API are correct after the state is rewritten. This might fix some races around the time of joining federated rooms. --- federationapi/storage/shared/storage.go | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/federationapi/storage/shared/storage.go b/federationapi/storage/shared/storage.go index a00d782f1..9e40f311c 100644 --- a/federationapi/storage/shared/storage.go +++ b/federationapi/storage/shared/storage.go @@ -70,27 +70,27 @@ func (d *Database) UpdateRoom( ) (joinedHosts []types.JoinedHost, err error) { err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { if purgeRoomFirst { - // If the event is a create event then we'll delete all of the existing - // data for the room. The only reason that a create event would be replayed - // to us in this way is if we're about to receive the entire room state. if err = d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil { return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err) } - } - - joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID) - if err != nil { - return err - } - - for _, add := range addHosts { - err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName) - if err != nil { + for _, add := range addHosts { + if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil { + return err + } + joinedHosts = append(joinedHosts, add) + } + } else { + if joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID); err != nil { + return err + } + for _, add := range addHosts { + if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil { + return err + } + } + if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil { return err } - } - if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil { - return err } return nil }) From 9ed8ff6b938e0452d122c68b38cf4fedcd74c0bf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 11 Oct 2022 10:48:36 +0100 Subject: [PATCH 11/14] Tweak federation `M_NOT_FOUND` errors --- federationapi/routing/events.go | 6 +++++- federationapi/routing/state.go | 15 ++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index 23796edfa..6168912bd 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -20,6 +20,7 @@ import ( "net/http" "time" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -95,7 +96,10 @@ func fetchEvent(ctx context.Context, rsAPI api.FederationRoomserverAPI, eventID } if len(eventsResponse.Events) == 0 { - return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} + return nil, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Event not found"), + } } return eventsResponse.Events[0].Event, nil diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 5377eb88f..1d08d0a82 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -135,23 +135,24 @@ func getState( return nil, nil, &resErr } - if !response.StateKnown { + switch { + case !response.RoomExists: + return nil, nil, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Room not found"), + } + case !response.StateKnown: return nil, nil, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("State not known"), } - } - if response.IsRejected { + case response.IsRejected: return nil, nil, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Event not found"), } } - if !response.RoomExists { - return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} - } - return response.StateEvents, response.AuthChainEvents, nil } From 3920b9f9b6155db69822d0dbcd36acb1eaa51c34 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 11 Oct 2022 10:58:34 +0100 Subject: [PATCH 12/14] Tweak `GetStateDeltas` behaviour (#2788) Improves the control flow of `GetStateDeltas` for clarity and possibly also fixes a bug where duplicate state delta entries could be inserted with different memberships instead of being correctly overridden by `join`. --- syncapi/storage/shared/storage_sync.go | 44 ++++++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index 0e19d97d2..d5b5b3121 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -360,34 +360,50 @@ func (d *DatabaseTransaction) GetStateDeltas( newlyJoinedRooms := make(map[string]bool, len(state)) for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { - if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" { - if membership == gomatrixserverlib.Join && prevMembership != membership { - // send full room state down instead of a delta + // Look for our membership in the state events and skip over any + // membership events that are not related to us. + membership, prevMembership := getMembershipFromEvent(ev.Event, userID) + if membership == "" { + continue + } + + if membership == gomatrixserverlib.Join { + // If our membership is now join but the previous membership wasn't + // then this is a "join transition", so we'll insert this room. + if prevMembership != membership { + // Get the full room state, as we'll send that down for a newly + // joined room instead of a delta. var s []types.StreamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter) - if err != nil { + if s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter); err != nil { if err == sql.ErrNoRows { continue } return nil, nil, err } + + // Add the information for this room into the state so that + // it will get added with all of the rest of the joined rooms. state[roomID] = s newlyJoinedRooms[roomID] = true - continue // we'll add this room in when we do joined rooms } - deltas = append(deltas, types.StateDelta{ - Membership: membership, - MembershipPos: ev.StreamPosition, - StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), - RoomID: roomID, - }) - break + // We won't add joined rooms into the delta at this point as they + // are added later on. + continue } + + deltas = append(deltas, types.StateDelta{ + Membership: membership, + MembershipPos: ev.StreamPosition, + StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + RoomID: roomID, + }) + break } } - // Add in currently joined rooms + // Finally, add in currently joined rooms, including those from the + // join transitions above. for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, types.StateDelta{ Membership: gomatrixserverlib.Join, From 0a9aebdf011921680e5b0646bf50d7900423aa69 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 11 Oct 2022 12:27:21 +0100 Subject: [PATCH 13/14] Private read receipts (#2789) Implement behaviours for `m.read.private` receipts. --- clientapi/routing/account_data.go | 44 ++++++++++++++--------------- clientapi/routing/receipt.go | 36 ++++++++++++++++++----- clientapi/routing/routing.go | 2 +- federationapi/consumers/receipts.go | 8 ++++++ internal/eventutil/types.go | 5 ++-- syncapi/streams/stream_receipt.go | 4 +++ 6 files changed, 66 insertions(+), 33 deletions(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index b28f0bb1f..4742b1240 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -154,33 +154,31 @@ func SaveReadMarker( return *resErr } - if r.FullyRead == "" { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"), + if r.FullyRead != "" { + data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) } } - data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) - if err != nil { - return jsonerror.InternalServerError() - } - - dataReq := api.InputAccountDataRequest{ - UserID: device.UserID, - DataType: "m.fully_read", - RoomID: roomID, - AccountData: data, - } - dataRes := api.InputAccountDataResponse{} - if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") - return util.ErrorResponse(err) - } - - // Handle the read receipt that may be included in the read marker + // Handle the read receipts that may be included in the read marker. if r.Read != "" { - return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read) + return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read", r.Read) + } + if r.ReadPrivate != "" { + return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read.private", r.ReadPrivate) } return util.JSONResponse{ diff --git a/clientapi/routing/receipt.go b/clientapi/routing/receipt.go index 0f9b1b4ff..99217a780 100644 --- a/clientapi/routing/receipt.go +++ b/clientapi/routing/receipt.go @@ -15,19 +15,22 @@ package routing import ( + "encoding/json" "fmt" "net/http" "time" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) -func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { +func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { timestamp := gomatrixserverlib.AsTimestamp(time.Now()) logrus.WithFields(logrus.Fields{ "roomID": roomID, @@ -37,13 +40,32 @@ func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, devi "timestamp": timestamp, }).Debug("Setting receipt") - // currently only m.read is accepted - if receiptType != "m.read" { - return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType)) - } + switch receiptType { + case "m.read", "m.read.private": + if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil { + return util.ErrorResponse(err) + } - if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil { - return util.ErrorResponse(err) + case "m.fully_read": + data, err := json.Marshal(fullyReadEvent{EventID: eventID}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) + } + + default: + return util.MessageResponse(400, fmt.Sprintf("Receipt type '%s' not known", receiptType)) } return util.JSONResponse{ diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f1fa66ca6..e72880ec5 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -1343,7 +1343,7 @@ func Setup( return util.ErrorResponse(err) } - return SetReceipt(req, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"]) + return SetReceipt(req, userAPI, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"]) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/presence/{userId}/status", diff --git a/federationapi/consumers/receipts.go b/federationapi/consumers/receipts.go index 366cb264e..75827cb68 100644 --- a/federationapi/consumers/receipts.go +++ b/federationapi/consumers/receipts.go @@ -81,6 +81,14 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) Type: msg.Header.Get("type"), } + switch receipt.Type { + case "m.read": + // These are allowed to be sent over federation + case "m.read.private", "m.fully_read": + // These must not be sent over federation + return true + } + // only send receipt events which originated from us _, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID) if err != nil { diff --git a/internal/eventutil/types.go b/internal/eventutil/types.go index afc62d8c2..18175d6a0 100644 --- a/internal/eventutil/types.go +++ b/internal/eventutil/types.go @@ -35,8 +35,9 @@ type AccountData struct { } type ReadMarkerJSON struct { - FullyRead string `json:"m.fully_read"` - Read string `json:"m.read"` + FullyRead string `json:"m.fully_read"` + Read string `json:"m.read"` + ReadPrivate string `json:"m.read.private"` } // NotificationData contains statistics about notifications, sent from diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index bba911022..977815078 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -67,6 +67,10 @@ func (p *ReceiptStreamProvider) IncrementalSync( if _, ok := req.IgnoredUsers.List[receipt.UserID]; ok { continue } + // Don't send private read receipts to other users + if receipt.Type == "m.read.private" && req.Device.UserID != receipt.UserID { + continue + } receiptsByRoom[receipt.RoomID] = append(receiptsByRoom[receipt.RoomID], receipt) } From 3c1474f68f2ac5e564d2bd4bd15f01d2a73f2845 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 11 Oct 2022 16:04:02 +0200 Subject: [PATCH 14/14] Fix `/get_missing_events` for rooms with `joined`/`invited` history_visibility (#2787) Sytest was using a wrong `history_visibility` for `invited` (https://github.com/matrix-org/sytest/pull/1303), so `invited` was passing for the wrong reason (-> defaulted to `shared`, as `invite` wasn't understood). This change now handles missing events like Synapse, if a server isn't allowed to see the event, it gets a redacted version of it, making the `get_missing_events` tests pass. --- are-we-synapse-yet.list | 2 +- roomserver/auth/auth.go | 30 +++++++------------ roomserver/internal/helpers/helpers.go | 15 +++++----- .../internal/perform/perform_backfill.go | 5 +++- roomserver/internal/query/query.go | 6 ++-- .../storage/postgres/state_snapshot_table.go | 4 ++- sytest-whitelist | 5 ++-- 7 files changed, 33 insertions(+), 34 deletions(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index c776a7400..81c0f8049 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -643,7 +643,7 @@ fed Inbound federation redacts events from erased users fme Outbound federation can request missing events fme Inbound federation can return missing events for world_readable visibility fme Inbound federation can return missing events for shared visibility -fme Inbound federation can return missing events for invite visibility +fme Inbound federation can return missing events for invited visibility fme Inbound federation can return missing events for joined visibility fme outliers whose auth_events are in a different room are correctly rejected fbk Outbound federation can backfill events diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index aa1d5bc25..31a856e8e 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -13,8 +13,6 @@ package auth import ( - "encoding/json" - "github.com/matrix-org/gomatrixserverlib" ) @@ -30,7 +28,7 @@ func IsServerAllowed( historyVisibility := HistoryVisibilityForRoom(authEvents) // 1. If the history_visibility was set to world_readable, allow. - if historyVisibility == "world_readable" { + if historyVisibility == gomatrixserverlib.HistoryVisibilityWorldReadable { return true } // 2. If the user's membership was join, allow. @@ -39,12 +37,12 @@ func IsServerAllowed( return true } // 3. If history_visibility was set to shared, and the user joined the room at any point after the event was sent, allow. - if historyVisibility == "shared" && serverCurrentlyInRoom { + if historyVisibility == gomatrixserverlib.HistoryVisibilityShared && serverCurrentlyInRoom { return true } // 4. If the user's membership was invite, and the history_visibility was set to invited, allow. invitedUserExists := IsAnyUserOnServerWithMembership(serverName, authEvents, gomatrixserverlib.Invite) - if invitedUserExists && historyVisibility == "invited" { + if invitedUserExists && historyVisibility == gomatrixserverlib.HistoryVisibilityInvited { return true } @@ -52,27 +50,16 @@ func IsServerAllowed( return false } -func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) string { +func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) gomatrixserverlib.HistoryVisibility { // https://matrix.org/docs/spec/client_server/r0.6.0#id87 // By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared. - visibility := "shared" - knownStates := []string{"invited", "joined", "shared", "world_readable"} + visibility := gomatrixserverlib.HistoryVisibilityShared for _, ev := range authEvents { if ev.Type() != gomatrixserverlib.MRoomHistoryVisibility { continue } - // TODO: This should be HistoryVisibilityContent to match things like 'MemberContent'. Do this when moving to GMSL - content := struct { - HistoryVisibility string `json:"history_visibility"` - }{} - if err := json.Unmarshal(ev.Content(), &content); err != nil { - break // value is not understood - } - for _, s := range knownStates { - if s == content.HistoryVisibility { - visibility = s - break - } + if vis, err := ev.HistoryVisibility(); err == nil { + visibility = vis } } return visibility @@ -80,6 +67,9 @@ func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) string { func IsAnyUserOnServerWithMembership(serverName gomatrixserverlib.ServerName, authEvents []*gomatrixserverlib.Event, wantMembership string) bool { for _, ev := range authEvents { + if ev.Type() != gomatrixserverlib.MRoomMember { + continue + } membership, err := ev.Membership() if err != nil || membership != wantMembership { continue diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index 3b83a0a6d..a6de8ac84 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -324,7 +324,7 @@ func slowGetHistoryVisibilityState( func ScanEventTree( ctx context.Context, db storage.Database, info *types.RoomInfo, front []string, visited map[string]bool, limit int, serverName gomatrixserverlib.ServerName, -) ([]types.EventNID, error) { +) ([]types.EventNID, map[string]struct{}, error) { var resultNIDs []types.EventNID var err error var allowed bool @@ -345,6 +345,7 @@ func ScanEventTree( var checkedServerInRoom bool var isServerInRoom bool + redactEventIDs := make(map[string]struct{}) // Loop through the event IDs to retrieve the requested events and go // through the whole tree (up to the provided limit) using the events' @@ -358,7 +359,7 @@ BFSLoop: // Retrieve the events to process from the database. events, err = db.EventsFromIDs(ctx, front) if err != nil { - return resultNIDs, err + return resultNIDs, redactEventIDs, err } if !checkedServerInRoom && len(events) > 0 { @@ -395,16 +396,16 @@ BFSLoop: ) // drop the error, as we will often error at the DB level if we don't have the prev_event itself. Let's // just return what we have. - return resultNIDs, nil + return resultNIDs, redactEventIDs, nil } // If the event hasn't been seen before and the HS // requesting to retrieve it is allowed to do so, add it to // the list of events to retrieve. - if allowed { - next = append(next, pre) - } else { + next = append(next, pre) + if !allowed { util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).Info("Not allowed to see event") + redactEventIDs[pre] = struct{}{} } } } @@ -413,7 +414,7 @@ BFSLoop: front = next } - return resultNIDs, err + return resultNIDs, redactEventIDs, err } func QueryLatestEventsAndState( diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index 69a075733..57e121ea2 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -78,7 +78,7 @@ func (r *Backfiller) PerformBackfill( } // Scan the event tree for events to send back. - resultNIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) + resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) if err != nil { return err } @@ -95,6 +95,9 @@ func (r *Backfiller) PerformBackfill( } for _, event := range loadedEvents { + if _, ok := redactEventIDs[event.EventID()]; ok { + event.Redact() + } response.Events = append(response.Events, event.Headered(info.RoomVersion)) } diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 7a424a334..f41132403 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -453,7 +453,7 @@ func (r *Queryer) QueryMissingEvents( return fmt.Errorf("missing RoomInfo for room %s", events[0].RoomID()) } - resultNIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) + resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) if err != nil { return err } @@ -470,7 +470,9 @@ func (r *Queryer) QueryMissingEvents( if verr != nil { return verr } - + if _, ok := redactEventIDs[event.EventID()]; ok { + event.Redact() + } response.Events = append(response.Events, event.Headered(roomVersion)) } } diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index 99c76befe..a00c026f4 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -21,10 +21,11 @@ import ( "fmt" "github.com/lib/pq" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/util" ) const stateSnapshotSchema = ` @@ -91,6 +92,7 @@ const bulkSelectStateForHistoryVisibilitySQL = ` WHERE state_snapshot_nid = $1 ) ) + ORDER BY depth ASC ) AS roomserver_events INNER JOIN roomserver_event_state_keys ON roomserver_events.event_state_key_nid = roomserver_event_state_keys.event_state_key_nid diff --git a/sytest-whitelist b/sytest-whitelist index 9ba9df75d..a3218ed70 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -306,7 +306,7 @@ Alternative server names do not cause a routing loop Events whose auth_events are in the wrong room do not mess up the room state Inbound federation can return events Inbound federation can return missing events for world_readable visibility -Inbound federation can return missing events for invite visibility +Inbound federation can return missing events for invited visibility Inbound federation can get public room list PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message @@ -742,4 +742,5 @@ User in private room doesn't appear in user directory User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave -Existing members see new member's presence \ No newline at end of file +Existing members see new member's presence +Inbound federation can return missing events for joined visibility \ No newline at end of file