From d065219de1c2860c9cf28e8d8169fabfb71dcc31 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 26 Sep 2023 15:47:37 +0200 Subject: [PATCH 01/40] Fix invitations not sending push notifications (#3207) The tests added in https://github.com/matrix-org/sytest/pull/1356 uncovered that we don't consider invitations as events the userapi should handle and thus just don't notify the client about any new invitations received over federation. --- sytest-whitelist | 6 +++- userapi/consumers/roomserver.go | 60 +++++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/sytest-whitelist b/sytest-whitelist index c61e0bc3c..60ba02302 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -783,4 +783,8 @@ Invited user can reject invite for empty room Invited user can reject local invite after originator leaves Guest users can join guest_access rooms Forgotten room messages cannot be paginated -Local device key changes get to remote servers with correct prev_id \ No newline at end of file +Local device key changes get to remote servers with correct prev_id +HS provides query metadata +HS can provide query metadata on a single protocol +Invites over federation are correctly pushed +Invites over federation are correctly pushed with name \ No newline at end of file diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 047fe9216..d5baa074c 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -92,18 +92,36 @@ func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called // Only handle events we care about - if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent { - return true - } - var output rsapi.OutputEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("roomserver output log: message parse failure") - return true - } - event := output.NewRoomEvent.Event - if event == nil { - log.Errorf("userapi consumer: expected event") + + var event *rstypes.HeaderedEvent + var isNewRoomEvent bool + switch rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) { + case rsapi.OutputTypeNewRoomEvent: + isNewRoomEvent = true + fallthrough + case rsapi.OutputTypeNewInviteEvent: + var output rsapi.OutputEvent + if err := json.Unmarshal(msg.Data, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("roomserver output log: message parse failure") + return true + } + if isNewRoomEvent { + event = output.NewRoomEvent.Event + } else { + event = output.NewInviteEvent.Event + } + + if event == nil { + log.Errorf("userapi consumer: expected event") + return true + } + + log.WithFields(log.Fields{ + "event_id": event.EventID(), + "event_type": event.Type(), + }).Tracef("Received message from roomserver: %#v", output) + default: return true } @@ -111,11 +129,6 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms go s.storeMessageStats(ctx, event.Type(), string(event.SenderID()), event.RoomID().String()) } - log.WithFields(log.Fields{ - "event_id": event.EventID(), - "event_type": event.Type(), - }).Tracef("Received message from roomserver: %#v", output) - metadata, err := msg.Metadata() if err != nil { return true @@ -448,6 +461,19 @@ func (s *OutputRoomEventConsumer) roomName(ctx context.Context, event *rstypes.H } } + // Special case for invites, as we don't store any "current state" for these events, + // we need to make sure that, if present, the m.room.name is sent as well. + if event.Type() == spec.MRoomMember && + gjson.GetBytes(event.Content(), "membership").Str == "invite" { + invState := gjson.GetBytes(event.JSON(), "unsigned.invite_room_state") + for _, ev := range invState.Array() { + if ev.Get("type").Str == spec.MRoomName { + name := ev.Get("content.name").Str + return name, nil + } + } + } + req := &rsapi.QueryCurrentStateRequest{ RoomID: event.RoomID().String(), StateTuples: []gomatrixserverlib.StateKeyTuple{roomNameTuple, canonicalAliasTuple}, From 16d922de701ca28966127b2e1160d65527f65e8e Mon Sep 17 00:00:00 2001 From: devonh Date: Tue, 26 Sep 2023 17:44:49 +0000 Subject: [PATCH 02/40] Complement fixes for pseudoIDs (#3206) --- clientapi/routing/sendevent.go | 9 +- go.mod | 4 +- go.sum | 8 +- roomserver/internal/perform/perform_leave.go | 22 +++- syncapi/synctypes/clientevent.go | 114 ++++++++++--------- 5 files changed, 91 insertions(+), 66 deletions(-) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index fc9c05ba9..69131966b 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -263,7 +263,11 @@ func SendEvent( } func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID string, rsAPI api.ClientRoomserverAPI) error { - userMap := r["users"].(map[string]interface{}) + users, ok := r["users"] + if !ok { + return nil + } + userMap := users.(map[string]interface{}) validRoomID, err := spec.NewRoomID(roomID) if err != nil { return err @@ -277,7 +281,8 @@ func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID strin if err != nil { return err } else if senderID == nil { - return fmt.Errorf("sender ID not found for %s in %s", uID, *validRoomID) + util.GetLogger(req.Context()).Warnf("sender ID not found for %s in %s", uID, *validRoomID) + continue } userMap[string(*senderID)] = level delete(userMap, user) diff --git a/go.mod b/go.mod index 564b4250a..78c1058e7 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a + github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 @@ -36,7 +36,7 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.2 - github.com/tidwall/gjson v1.16.0 + github.com/tidwall/gjson v1.17.0 github.com/tidwall/sjson v1.2.5 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible diff --git a/go.sum b/go.sum index f7d21d96b..3cf569497 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a h1:+RC9Ddmt5v4y58qmdz5WuEEWCJ9gBWuYLyndnWkGfXU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49 h1:o4mdKYYIYCi/QplAjBAJ5kvu3NXXkutZF88gTTpZjj4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= @@ -318,8 +318,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= -github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index 5c63a6684..5bea00445 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -93,11 +93,21 @@ func (r *Leaver) performLeaveRoomByID( isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, *leaver) if err == nil && isInvitePending { sender, serr := r.RSAPI.QueryUserIDForSender(ctx, *roomID, senderUser) - if serr != nil || sender == nil { - return nil, fmt.Errorf("sender %q has no matching userID", senderUser) + if serr != nil { + return nil, fmt.Errorf("failed looking up userID for sender %q: %w", senderUser, serr) } - if !r.Cfg.Matrix.IsLocalServerName(sender.Domain()) { - return r.performFederatedRejectInvite(ctx, req, res, *sender, eventID, *leaver) + + var domain spec.ServerName + if sender == nil { + // TODO: Currently a federated invite has no way of knowing the mxid_mapping of the inviter. + // Should we add the inviter's m.room.member event (with mxid_mapping) to invite_room_state to allow + // the invited user to leave via the inviter's server? + domain = roomID.Domain() + } else { + domain = sender.Domain() + } + if !r.Cfg.Matrix.IsLocalServerName(domain) { + return r.performFederatedRejectInvite(ctx, req, res, domain, eventID, *leaver) } // check that this is not a "server notice room" accData := &userapi.QueryAccountDataResponse{} @@ -219,14 +229,14 @@ func (r *Leaver) performFederatedRejectInvite( ctx context.Context, req *api.PerformLeaveRequest, res *api.PerformLeaveResponse, // nolint:unparam - inviteSender spec.UserID, eventID string, + inviteDomain spec.ServerName, eventID string, leaver spec.SenderID, ) ([]api.OutputEvent, error) { // Ask the federation sender to perform a federated leave for us. leaveReq := fsAPI.PerformLeaveRequest{ RoomID: req.RoomID, UserID: req.Leaver.String(), - ServerNames: []spec.ServerName{inviteSender.Domain()}, + ServerNames: []spec.ServerName{inviteDomain}, } leaveRes := fsAPI.PerformLeaveResponse{} if err := r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil { diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go index 6812f8332..fe4f6c07f 100644 --- a/syncapi/synctypes/clientevent.go +++ b/syncapi/synctypes/clientevent.go @@ -347,71 +347,81 @@ func updatePowerLevelEvent(userIDForSender spec.UserIDForSender, se gomatrixserv return se, nil } - pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(se) - if err != nil { - return nil, err - } - newPls := make(map[string]int64) - var userID *spec.UserID - for user, level := range pls.Users { - if eventFormat != FormatSyncFederation { - userID, err = userIDForSender(se.RoomID(), spec.SenderID(user)) - if err != nil { - return nil, err - } - user = userID.String() - } - newPls[user] = level - } - var newPlBytes, newEv []byte - newPlBytes, err = json.Marshal(newPls) - if err != nil { - return nil, err - } - newEv, err = sjson.SetRawBytes(se.JSON(), "content.users", newPlBytes) - if err != nil { - return nil, err - } + newEv := se.JSON() - // do the same for prev content - prevContent := gjson.GetBytes(se.JSON(), "unsigned.prev_content") - if !prevContent.Exists() { - var evNew gomatrixserverlib.PDU - evNew, err = gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSON(newEv, false) + usersField := gjson.GetBytes(se.JSON(), "content.users") + if usersField.Exists() { + pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(se) if err != nil { return nil, err } - return evNew, err - } - pls = gomatrixserverlib.PowerLevelContent{} - err = json.Unmarshal([]byte(prevContent.Raw), &pls) - if err != nil { - return nil, err + newPls := make(map[string]int64) + var userID *spec.UserID + for user, level := range pls.Users { + if eventFormat != FormatSyncFederation { + userID, err = userIDForSender(se.RoomID(), spec.SenderID(user)) + if err != nil { + return nil, err + } + user = userID.String() + } + newPls[user] = level + } + + var newPlBytes []byte + newPlBytes, err = json.Marshal(newPls) + if err != nil { + return nil, err + } + newEv, err = sjson.SetRawBytes(se.JSON(), "content.users", newPlBytes) + if err != nil { + return nil, err + } } - newPls = make(map[string]int64) - for user, level := range pls.Users { - if eventFormat != FormatSyncFederation { - userID, err = userIDForSender(se.RoomID(), spec.SenderID(user)) + // do the same for prev content + prevUsersField := gjson.GetBytes(se.JSON(), "unsigned.prev_content.users") + if prevUsersField.Exists() { + prevContent := gjson.GetBytes(se.JSON(), "unsigned.prev_content") + if !prevContent.Exists() { + evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSON(newEv, false) if err != nil { return nil, err } - user = userID.String() + + return evNew, err + } + pls := gomatrixserverlib.PowerLevelContent{} + err := json.Unmarshal([]byte(prevContent.Raw), &pls) + if err != nil { + return nil, err + } + + newPls := make(map[string]int64) + for user, level := range pls.Users { + if eventFormat != FormatSyncFederation { + userID, userErr := userIDForSender(se.RoomID(), spec.SenderID(user)) + if userErr != nil { + return nil, userErr + } + user = userID.String() + } + newPls[user] = level + } + + var newPlBytes []byte + newPlBytes, err = json.Marshal(newPls) + if err != nil { + return nil, err + } + newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes) + if err != nil { + return nil, err } - newPls[user] = level - } - newPlBytes, err = json.Marshal(newPls) - if err != nil { - return nil, err - } - newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes) - if err != nil { - return nil, err } - var evNew gomatrixserverlib.PDU - evNew, err = gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSONWithEventID(se.EventID(), newEv, false) + evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSONWithEventID(se.EventID(), newEv, false) if err != nil { return nil, err } From 05a8f1ede3ca0ccd0f9a7d327ef3cb30b6fd2798 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 27 Sep 2023 08:27:08 +0200 Subject: [PATCH 03/40] Support for room version v11 (#3204) Fixes #3203 --- are-we-synapse-yet.list | 10 +++++++++- clientapi/routing/redaction.go | 8 +++++++- go.mod | 2 +- go.sum | 4 ++-- roomserver/internal/perform/perform_create_room.go | 11 ++++++++++- sytest-whitelist | 10 +++++++++- 6 files changed, 38 insertions(+), 7 deletions(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 585374738..80c0dbff6 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -944,4 +944,12 @@ rmv remote user can join room with version 10 rmv User can invite remote user to room with version 10 rmv Remote user can backfill in a room with version 10 rmv Can reject invites over federation for rooms with version 10 -rmv Can receive redactions from regular users over federation in room version 10 \ No newline at end of file +rmv Can receive redactions from regular users over federation in room version 10 +rmv User can create and send/receive messages in a room with version 11 +rmv local user can join room with version 11 +rmv User can invite local user to room with version 11 +rmv remote user can join room with version 11 +rmv User can invite remote user to room with version 11 +rmv Remote user can backfill in a room with version 11 +rmv Can reject invites over federation for rooms with version 11 +rmv Can receive redactions from regular users over federation in room version 11 \ No newline at end of file diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index aa579db64..f331a73c7 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -34,7 +34,8 @@ import ( ) type redactionContent struct { - Reason string `json:"reason"` + Reason string `json:"reason"` + Redacts string `json:"redacts"` } type redactionResponse struct { @@ -151,6 +152,11 @@ func SendRedaction( Type: spec.MRoomRedaction, Redacts: eventID, } + + // Room version 11 expects the "redacts" field on the + // content field, so add it here as well + r.Redacts = eventID + err = proto.SetContent(r) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("proto.SetContent failed") diff --git a/go.mod b/go.mod index 78c1058e7..3176a8b7b 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49 + github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 diff --git a/go.sum b/go.sum index 3cf569497..7f0e98838 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49 h1:o4mdKYYIYCi/QplAjBAJ5kvu3NXXkutZF88gTTpZjj4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4 h1:UuXfC7b29RBDfMdLmggeF3opu3XuGi8bNT9SKZtZc3I= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go index cd6629d28..eb8de7811 100644 --- a/roomserver/internal/perform/perform_create_room.go +++ b/roomserver/internal/perform/perform_create_room.go @@ -90,7 +90,16 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo } else { senderID = spec.SenderID(userID.String()) } - createContent["creator"] = senderID + + // TODO: Maybe, at some point, GMSL should return the events to create, so we can define the version + // entirely there. + switch createRequest.RoomVersion { + case gomatrixserverlib.RoomVersionV11: + // RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175 + default: + createContent["creator"] = senderID + } + createContent["room_version"] = createRequest.RoomVersion powerLevelContent := eventutil.InitialPowerLevelsContent(string(senderID)) joinRuleContent := gomatrixserverlib.JoinRuleContent{ diff --git a/sytest-whitelist b/sytest-whitelist index 60ba02302..492c756ba 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -787,4 +787,12 @@ Local device key changes get to remote servers with correct prev_id HS provides query metadata HS can provide query metadata on a single protocol Invites over federation are correctly pushed -Invites over federation are correctly pushed with name \ No newline at end of file +Invites over federation are correctly pushed with name +User can create and send/receive messages in a room with version 11 +local user can join room with version 11 +User can invite local user to room with version 11 +remote user can join room with version 11 +User can invite remote user to room with version 11 +Remote user can backfill in a room with version 11 +Can reject invites over federation for rooms with version 11 +Can receive redactions from regular users over federation in room version 11 \ No newline at end of file From 10b4fbc66d4dca20faa6cc56a272d5303871094b Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 28 Sep 2023 07:36:34 +0200 Subject: [PATCH 04/40] Fix m.direct only being partially upgraded (#3209) Previously we would update `m.direct` once we found the old room ID. If the roomID is found somewhere in the middle, we would never add the rest of the users, resulting in only partially upgraded `m.direct` and chats loosing their 1:1 flag. --- userapi/consumers/roomserver.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index d5baa074c..6da41f8a1 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -266,8 +266,8 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID, directChats := gjson.ParseBytes(directChatsRaw) newDirectChats := make(map[string][]string) // iterate over all userID -> roomIDs + var found bool directChats.ForEach(func(userID, roomIDs gjson.Result) bool { - var found bool for _, roomID := range roomIDs.Array() { newDirectChats[userID.Str] = append(newDirectChats[userID.Str], roomID.Str) // add the new roomID to m.direct @@ -276,22 +276,21 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID, newDirectChats[userID.Str] = append(newDirectChats[userID.Str], newRoomID) } } - // Only hit the database if we found the old room as a DM for this user - if found { - var data []byte - data, err = json.Marshal(newDirectChats) - if err != nil { - return true - } - if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil { - return true - } - } return true }) - if err != nil { - return fmt.Errorf("failed to update m.direct state") + + // Only hit the database if we found the old room as a DM for this user + if found { + var data []byte + data, err = json.Marshal(newDirectChats) + if err != nil { + return err + } + if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil { + return fmt.Errorf("failed to update m.direct state: %w", err) + } } + return nil } From f02d998253a6fb2b4e4563f7bd66ac5f00aa97f9 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 28 Sep 2023 07:36:57 +0200 Subject: [PATCH 05/40] Remove the creator field when upgrading to v11 (#3210) Minor oversight --- roomserver/internal/perform/perform_upgrade.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go index c32e10d53..9d7c7b567 100644 --- a/roomserver/internal/perform/perform_upgrade.go +++ b/roomserver/internal/perform/perform_upgrade.go @@ -368,7 +368,16 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // in the create event (such as for the room types MSC). newCreateContent := map[string]interface{}{} _ = json.Unmarshal(oldCreateEvent.Content(), &newCreateContent) - newCreateContent["creator"] = string(senderID) + + switch newVersion { + case gomatrixserverlib.RoomVersionV11: + // RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175 + // So if we are upgrading from pre v11, we need to remove the field. + delete(newCreateContent, "creator") + default: + newCreateContent["creator"] = senderID + } + newCreateContent["room_version"] = newVersion newCreateContent["predecessor"] = gomatrixserverlib.PreviousRoom{ EventID: tombstoneEvent.EventID(), From f1db57c7f80093d4b9727d72dff93f1fe6d9f829 Mon Sep 17 00:00:00 2001 From: jahway603 <64485701+jahway603@users.noreply.github.com> Date: Thu, 28 Sep 2023 01:38:29 -0400 Subject: [PATCH 06/40] Updated minimum required go version in README.md (#3194) Updated minimum required go version in README.md ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `jahway603 ` Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 34604eff9..bde19b07e 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for more information on requirements. -To build Dendrite, you will need Go 1.18 or later. +To build Dendrite, you will need Go 1.20 or later. For a usable federating Dendrite deployment, you will also need: From 4d344b65b2132b1c4ce7fe4f142295b9716c9ca5 Mon Sep 17 00:00:00 2001 From: Tracker-Friendly <109815155+Tracker-Friendly@users.noreply.github.com> Date: Thu, 28 Sep 2023 05:40:12 +0000 Subject: [PATCH 07/40] Fixed typo in documentation (#3212) ### Pull Request Checklist * [ ] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests This PR doesn't need tests because it's a documentation update * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed off privately Co-authored-by: Tracker-Friendly --- docs/administration/5_optimisation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration/5_optimisation.md b/docs/administration/5_optimisation.md index b327171eb..57b7924d3 100644 --- a/docs/administration/5_optimisation.md +++ b/docs/administration/5_optimisation.md @@ -95,7 +95,7 @@ Consider enabling the DNS cache by modifying the `global` section of your config ## Time synchronisation Matrix relies heavily on TLS which requires the system time to be correct. If the clock -drifts then you may find that federation no works reliably (or at all) and clients may +drifts then you may find that federation will not work reliably (or at all) and clients may struggle to connect to your Dendrite server. Ensure that the time is synchronised on your system by enabling NTP sync. From b341a66152269ea8c031955a6654dfb4b07e56e3 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 28 Sep 2023 12:06:21 +0200 Subject: [PATCH 08/40] Version 0.13.3 (#3213) --- .github/workflows/helm.yml | 2 +- CHANGES.md | 19 +++++++++++++++++++ cmd/dendrite-demo-yggdrasil/README.md | 2 +- cmd/dendrite-upgrade-tests/main.go | 3 +-- docs/installation/1_planning.md | 2 +- go.mod | 2 +- helm/dendrite/Chart.yaml | 4 ++-- helm/dendrite/README.md | 5 +---- internal/version.go | 2 +- 9 files changed, 28 insertions(+), 13 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index bf62a1c19..9a5eb2b62 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -32,7 +32,7 @@ jobs: version: v3.10.0 - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.4.1 + uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" with: diff --git a/CHANGES.md b/CHANGES.md index f4a814566..bdb6a796e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,24 @@ # Changelog +## Dendrite 0.13.3 (2023-09-28) + +### Fixes: + +- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir)) +- Invitations are now correctly pushed to devices +- A bug which could result in the corruption of `m.direct` account data has been fixed + +### Features + +- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response +- Room version 11 is now supported +- Clients can request the `federation` `event_format` when creating filters +- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md) + +### Other + +- Dendrite now requires Go 1.20 if building from source + ## Dendrite 0.13.2 (2023-08-23) ### Fixes: diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md index 14fc3a2db..23304c214 100644 --- a/cmd/dendrite-demo-yggdrasil/README.md +++ b/cmd/dendrite-demo-yggdrasil/README.md @@ -1,6 +1,6 @@ # Yggdrasil Demo -This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later. +This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.20 or later. To run the homeserver, start at the root of the Dendrite repository and run: diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index 68919e525..b78c5f605 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "net/http" "os" @@ -515,7 +514,7 @@ func testCreateAccount(dockerClient *client.Client, version *semver.Version, con } defer response.Close() - data, err := ioutil.ReadAll(response.Reader) + data, err := io.ReadAll(response.Reader) if err != nil { return err } diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md index 354003aef..37ca5702a 100644 --- a/docs/installation/1_planning.md +++ b/docs/installation/1_planning.md @@ -59,7 +59,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie ### Go -At this time, Dendrite supports being built with Go 1.18 or later. We do not support building +At this time, Dendrite supports being built with Go 1.20 or later. We do not support building Dendrite with older versions of Go than this. If you are installing Go using a package manager, you should check (by running `go version`) that you are using a suitable version before you start. diff --git a/go.mod b/go.mod index 3176a8b7b..c2ab105b1 100644 --- a/go.mod +++ b/go.mod @@ -143,4 +143,4 @@ require ( modernc.org/token v1.0.1 // indirect ) -go 1.18 +go 1.20 diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 5590a39b1..674152051 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: dendrite -version: "0.13.3" -appVersion: "0.13.2" +version: "0.13.4" +appVersion: "0.13.3" description: Dendrite Matrix Homeserver type: application keywords: diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index 7f7ea484a..cb78a54f2 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,7 +1,7 @@ # dendrite -![Version: 0.13.2](https://img.shields.io/badge/Version-0.13.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.2](https://img.shields.io/badge/AppVersion-0.13.2-informational?style=flat-square) +![Version: 0.13.4](https://img.shields.io/badge/Version-0.13.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.3](https://img.shields.io/badge/AppVersion-0.13.3-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -63,9 +63,6 @@ Create a folder `appservices` and place your configurations in there. The confi | strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | | strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | | strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | -| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | -| strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | -| strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | | dendrite_config.version | int | `2` | | | dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. | | dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) | diff --git a/internal/version.go b/internal/version.go index 81e0fc529..1f8a62bc2 100644 --- a/internal/version.go +++ b/internal/version.go @@ -18,7 +18,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 13 - VersionPatch = 2 + VersionPatch = 3 VersionTag = "" // example: "rc1" gitRevLen = 7 // 7 matches the displayed characters on github.com From 1853f58cb492c423641636562c6aa6eb701426a2 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Thu, 28 Sep 2023 12:38:53 +0200 Subject: [PATCH 09/40] Add missing sliding sync config --- dendrite-sample.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml index 8abc23011..7affc2599 100644 --- a/dendrite-sample.yaml +++ b/dendrite-sample.yaml @@ -72,6 +72,10 @@ global: # The base URL to delegate client-server communications to e.g. https://localhost well_known_client_name: "" + # The server name to delegate sliding sync communications to, with optional port. + # Requires `well_known_client_name` to also be configured. + well_known_sliding_sync_proxy: "" + # Lists of domains that the server will trust as identity servers to verify third # party identifiers such as phone numbers and email addresses. trusted_third_party_id_servers: From 3d02c8103152fbb9a5401a091ef0f681a32bfeee Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:50:31 +0200 Subject: [PATCH 10/40] Fix tests for x86 (#3214) --- syncapi/internal/history_visibility_test.go | 2 +- syncapi/storage/interface.go | 2 +- syncapi/storage/postgres/memberships_table.go | 2 +- syncapi/storage/shared/storage_consumer.go | 2 +- syncapi/storage/sqlite3/memberships_table.go | 2 +- syncapi/storage/tables/interface.go | 2 +- syncapi/storage/tables/memberships_test.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/syncapi/internal/history_visibility_test.go b/syncapi/internal/history_visibility_test.go index 984f90edd..24515bbb2 100644 --- a/syncapi/internal/history_visibility_test.go +++ b/syncapi/internal/history_visibility_test.go @@ -58,7 +58,7 @@ type mockDB struct { roomID string } -func (s *mockDB) SelectMembershipForUser(ctx context.Context, roomID string, userID string, pos int64) (string, int, error) { +func (s *mockDB) SelectMembershipForUser(ctx context.Context, roomID string, userID string, pos int64) (string, int64, error) { if roomID == s.roomID { membership, ok := s.currentMembership[userID] if !ok { diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index dca5d1a14..97c781b9b 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -107,7 +107,7 @@ type DatabaseTransaction interface { // SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found // returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty // string as the membership. - SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) // getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error) GetPresences(ctx context.Context, userID []string) ([]*types.PresenceInternal, error) diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index fcbe14b16..4fe4260da 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -131,7 +131,7 @@ func (s *membershipsStatements) SelectMembershipCount( // string as the membership. func (s *membershipsStatements) SelectMembershipForUser( ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64, -) (membership string, topologyPos int, err error) { +) (membership string, topologyPos int64, err error) { stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt) err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos) if err != nil { diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index 0f4080d53..923ead9bd 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -583,7 +583,7 @@ func (d *Database) GetPresences(ctx context.Context, userIDs []string) ([]*types return d.Presence.GetPresenceForUsers(ctx, nil, userIDs) } -func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) { +func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) { return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos) } diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index 05f756fda..a1b16306c 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -134,7 +134,7 @@ func (s *membershipsStatements) SelectMembershipCount( // string as the membership. func (s *membershipsStatements) SelectMembershipForUser( ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64, -) (membership string, topologyPos int, err error) { +) (membership string, topologyPos int64, err error) { stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt) err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos) if err != nil { diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index f5c66c42d..45117d6d3 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -194,7 +194,7 @@ type Receipts interface { type Memberships interface { UpsertMembership(ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error) - SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) PurgeMemberships(ctx context.Context, txn *sql.Tx, roomID string) error SelectMemberships( ctx context.Context, txn *sql.Tx, diff --git a/syncapi/storage/tables/memberships_test.go b/syncapi/storage/tables/memberships_test.go index a421a9772..0a36f5887 100644 --- a/syncapi/storage/tables/memberships_test.go +++ b/syncapi/storage/tables/memberships_test.go @@ -124,7 +124,7 @@ func testUpsert(t *testing.T, ctx context.Context, table tables.Memberships, mem if err != nil { t.Fatalf("failed to select membership: %s", err) } - expectedPos := 1 + var expectedPos int64 = 1 if pos != expectedPos { t.Fatalf("expected pos to be %d, got %d", expectedPos, pos) } From 2259e71c0cc7d154eefd85a6466b08e7afb7f69a Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:33:04 +0200 Subject: [PATCH 11/40] Fix `resolve-state` (#3229) Previously we would "start" the roomserver API, which isn't the best idea, given it also starts processing Jetstream events. We now use a `dummyQuerier` to implement the needed interface for "converting" userID/senderIDs. As per the comment, this **DOES NOT** do any magic for pseudoID rooms. --- cmd/resolve-state/main.go | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index 3ffcac9e6..5be449097 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -11,13 +11,11 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" @@ -35,6 +33,19 @@ var roomVersion = flag.String("roomversion", "5", "the room version to parse eve var filterType = flag.String("filtertype", "", "the event types to filter on") var difference = flag.Bool("difference", false, "whether to calculate the difference between snapshots") +// dummyQuerier implements QuerySenderIDAPI. Does **NOT** do any "magic" for pseudoID rooms +// to avoid having to "start" a full roomserver API. +type dummyQuerier struct{} + +func (d dummyQuerier) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + s := spec.SenderIDFromUserID(userID) + return &s, nil +} + +func (d dummyQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return senderID.ToUserID(), nil +} + // nolint:gocyclo func main() { ctx := context.Background() @@ -56,27 +67,32 @@ func main() { } } - fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") - processCtx := process.NewProcessContext() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + + dbOpts := cfg.RoomServer.Database + if dbOpts.ConnectionString == "" { + dbOpts = cfg.Global.DatabaseOptions + } + + fmt.Println("Opening database") roomserverDB, err := storage.Open( - processCtx.Context(), cm, &cfg.RoomServer.Database, - caching.NewRistrettoCache(128*1024*1024, time.Hour, true), + processCtx.Context(), cm, &dbOpts, + caching.NewRistrettoCache(8*1024*1024, time.Minute*5, caching.DisableMetrics), ) if err != nil { panic(err) } - natsInstance := &jetstream.NATSInstance{} - rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, - natsInstance, caching.NewRistrettoCache(128*1024*1024, time.Hour, true), false) + rsAPI := dummyQuerier{} roomInfo := &types.RoomInfo{ RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion), } stateres := state.NewStateResolution(roomserverDB, roomInfo, rsAPI) + fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") + if *difference { if len(snapshotNIDs) != 2 { panic("need exactly two state snapshot NIDs to calculate difference") From 5888329b133468b23bb6d84ab9f244c667c9ac17 Mon Sep 17 00:00:00 2001 From: kegsay Date: Wed, 11 Oct 2023 17:41:12 +0100 Subject: [PATCH 12/40] Update Complement to match new public API shape (#3232) Sister PR to matrix-org/complement#666 Context: https://github.com/matrix-org/complement/issues/654#issuecomment-1746613495 --- .github/workflows/dendrite.yml | 2 +- build/scripts/complement.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 772b45cb2..ac40f06b0 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -440,7 +440,7 @@ jobs: # Run Complement - run: | set -o pipefail && - go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt -hide all + go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all shell: bash name: Run Complement Tests env: diff --git a/build/scripts/complement.sh b/build/scripts/complement.sh index 29feff304..8608d8fa5 100755 --- a/build/scripts/complement.sh +++ b/build/scripts/complement.sh @@ -15,5 +15,5 @@ tar -xzf master.tar.gz # Run the tests! cd complement-master -COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests +COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi From 933ae2db91c4c65dfc7b8a395567cd39f4c366c8 Mon Sep 17 00:00:00 2001 From: devonh Date: Thu, 12 Oct 2023 18:03:06 +0000 Subject: [PATCH 13/40] Update bug report to reflect current team members (#3234) --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index a3b8c0754..e18968baf 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -62,6 +62,6 @@ If you can identify any relevant log snippets from server logs, please include those (please be careful to remove any personal or private data). Please surround them with ``` (three backticks, on a line on their own), so that they are formatted legibly. -Alternatively, please send logs to @kegan:matrix.org or @neilalexander:matrix.org +Alternatively, please send logs to @kegan:matrix.org, @s7evink:matrix.org or @devonh:one.ems.host with a link to the respective Github issue, thanks! --> From fe2955a4dbf40562374d79bc991e0d7028f0f239 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 09:40:21 +0200 Subject: [PATCH 14/40] Bump golang.org/x/net from 0.14.0 to 0.17.0 (#3233) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.14.0 to 0.17.0.
Commits
  • b225e7c http2: limit maximum handler goroutines to MaxConcurrentStreams
  • 88194ad go.mod: update golang.org/x dependencies
  • 2b60a61 quic: fix several bugs in flow control accounting
  • 73d82ef quic: handle DATA_BLOCKED frames
  • 5d5a036 quic: handle streams moving from the data queue to the meta queue
  • 350aad2 quic: correctly extend peer's flow control window after MAX_DATA
  • 21814e7 quic: validate connection id transport parameters
  • a600b35 quic: avoid redundant MAX_DATA updates
  • ea63359 http2: check stream body is present on read timeout
  • ddd8598 quic: version negotiation
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/net&package-manager=go_modules&previous-version=0.14.0&new-version=0.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/matrix-org/dendrite/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index c2ab105b1..839e23f4c 100644 --- a/go.mod +++ b/go.mod @@ -42,12 +42,12 @@ require ( github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.6 go.uber.org/atomic v1.10.0 - golang.org/x/crypto v0.13.0 + golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 golang.org/x/image v0.5.0 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e golang.org/x/sync v0.3.0 - golang.org/x/term v0.12.0 + golang.org/x/term v0.13.0 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.4.0 @@ -123,8 +123,8 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.12.0 // indirect diff --git a/go.sum b/go.sum index 7f0e98838..e036f7b2b 100644 --- a/go.sum +++ b/go.sum @@ -354,8 +354,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -386,8 +386,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -418,12 +418,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 8c23c1150c37a88e078037b8c4b47f4efecab727 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:09:05 +0200 Subject: [PATCH 15/40] Tweaks around the device list updater (#3227) I hope the comments explain the changes. `notifyWorkers` notifies a worker which then calls `processServer`, which in turn gets all users and calls `processServerUser`. There is no need to call `processServer` for the same domain on startup. --- .golangci.yml | 2 +- userapi/internal/device_list_update.go | 45 ++++++++++++++++---- userapi/internal/device_list_update_test.go | 46 +++++++++++++++++++++ 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index bb8d38a8b..5bee0a885 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -180,7 +180,6 @@ linters-settings: linters: enable: - errcheck - - goconst - gocyclo - goimports # Does everything gofmt does - gosimple @@ -211,6 +210,7 @@ linters: - stylecheck - typecheck # Should turn back on soon - unconvert # Should turn back on soon + - goconst # Slightly annoying, as it reports "issues" in SQL statements disable-all: false presets: fast: false diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go index 3fccf56bb..2f33589fe 100644 --- a/userapi/internal/device_list_update.go +++ b/userapi/internal/device_list_update.go @@ -180,11 +180,13 @@ func (u *DeviceListUpdater) Start() error { if err != nil { return err } + + newStaleLists := dedupeStaleLists(staleLists) offset, step := time.Second*10, time.Second - if max := len(staleLists); max > 120 { + if max := len(newStaleLists); max > 120 { step = (time.Second * 120) / time.Duration(max) } - for _, userID := range staleLists { + for _, userID := range newStaleLists { userID := userID // otherwise we are only sending the last entry time.AfterFunc(offset, func() { u.notifyWorkers(userID) @@ -416,6 +418,12 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Duration, bool) { ctx := u.process.Context() + // If the process.Context is canceled, there is no need to go further. + // This avoids spamming the logs when shutting down + if errors.Is(ctx.Err(), context.Canceled) { + return defaultWaitTime, false + } + logger := util.GetLogger(ctx).WithField("server_name", serverName) deviceListUpdateCount.WithLabelValues(string(serverName)).Inc() @@ -428,13 +436,6 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura return waitTime, true } - defer func() { - for _, userID := range userIDs { - // always clear the channel to unblock Update calls regardless of success/failure - u.clearChannel(userID) - } - }() - for _, userID := range userIDs { userWait, err := u.processServerUser(ctx, serverName, userID) if err != nil { @@ -461,6 +462,11 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName spec.ServerName, userID string) (time.Duration, error) { ctx, cancel := context.WithTimeout(ctx, requestTimeout) defer cancel() + + // If we are processing more than one user per server, this unblocks further calls to Update + // immediately instead of just after **all** users have been processed. + defer u.clearChannel(userID) + logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "server_name": serverName, "user_id": userID, @@ -579,3 +585,24 @@ func (u *DeviceListUpdater) updateDeviceList(res *fclient.RespUserDevices) error } return nil } + +// dedupeStaleLists de-duplicates the stateList entries using the domain. +// This is used on startup, processServer is getting all users anyway, so +// there is no need to send every user to the workers. +func dedupeStaleLists(staleLists []string) []string { + seenDomains := make(map[spec.ServerName]struct{}) + newStaleLists := make([]string, 0, len(staleLists)) + for _, userID := range staleLists { + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + // non-fatal and should not block starting up + continue + } + if _, ok := seenDomains[domain]; ok { + continue + } + newStaleLists = append(newStaleLists, userID) + seenDomains[domain] = struct{}{} + } + return newStaleLists +} diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 10b9c6521..38fd8b583 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -428,3 +428,49 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) { } }) } + +func Test_dedupeStateList(t *testing.T) { + alice := "@alice:localhost" + bob := "@bob:localhost" + charlie := "@charlie:notlocalhost" + invalidUserID := "iaminvalid:localhost" + + tests := []struct { + name string + staleLists []string + want []string + }{ + { + name: "empty stateLists", + staleLists: []string{}, + want: []string{}, + }, + { + name: "single entry", + staleLists: []string{alice}, + want: []string{alice}, + }, + { + name: "multiple entries without dupe servers", + staleLists: []string{alice, charlie}, + want: []string{alice, charlie}, + }, + { + name: "multiple entries with dupe servers", + staleLists: []string{alice, bob, charlie}, + want: []string{alice, charlie}, + }, + { + name: "list with invalid userID", + staleLists: []string{alice, bob, invalidUserID}, + want: []string{alice}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dedupeStaleLists(tt.staleLists); !reflect.DeepEqual(got, tt.want) { + t.Errorf("dedupeStaleLists() = %v, want %v", got, tt.want) + } + }) + } +} From 8b3adaf244a699e81577d81f1e0a1a38d42e5f51 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 23 Oct 2023 15:17:21 +0200 Subject: [PATCH 16/40] Fix state resets (#3231) Needs https://github.com/matrix-org/gomatrixserverlib/pull/419 May fix: https://github.com/matrix-org/dendrite/issues/2508, https://github.com/matrix-org/dendrite/issues/1760 --- clientapi/routing/profile.go | 2 +- go.mod | 2 +- go.sum | 4 +- roomserver/roomserver_test.go | 112 +++++++++++++++++++++++++++++++++- 4 files changed, 114 insertions(+), 6 deletions(-) diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 564cd588a..9959144c8 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -300,7 +300,7 @@ func updateProfile( }, e } - if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil { + if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, false); err != nil { util.GetLogger(ctx).WithError(err).Error("SendEvents failed") return util.JSONResponse{ Code: http.StatusInternalServerError, diff --git a/go.mod b/go.mod index 839e23f4c..7d4c3677b 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4 + github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 diff --git a/go.sum b/go.sum index e036f7b2b..c1392cbd4 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4 h1:UuXfC7b29RBDfMdLmggeF3opu3XuGi8bNT9SKZtZc3I= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be h1:bZP16ydP8uRoRBo1p/7WHMexjg7JJGj81fKzZ1FULb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 47626b30a..90e67b699 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -407,7 +407,9 @@ type fledglingEvent struct { RoomID string Redacts string Depth int64 - PrevEvents []interface{} + PrevEvents []any + AuthEvents []any + Content map[string]any } func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEvent) { @@ -424,7 +426,13 @@ func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEve Depth: ev.Depth, PrevEvents: ev.PrevEvents, }) - err := eb.SetContent(map[string]interface{}{}) + if ev.Content == nil { + ev.Content = map[string]any{} + } + if ev.AuthEvents != nil { + eb.AuthEvents = ev.AuthEvents + } + err := eb.SetContent(ev.Content) if err != nil { t.Fatalf("mustCreateEvent: failed to marshal event content %v", err) } @@ -1077,3 +1085,103 @@ func TestUpgrade(t *testing.T) { } }) } + +func TestStateReset(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + charlie := test.NewUser(t) + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + // Prepare APIs + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + // create a new room + room := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat)) + + // join with Bob and Charlie + bobJoinEv := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]any{"membership": "join"}, test.WithStateKey(bob.ID)) + charlieJoinEv := room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]any{"membership": "join"}, test.WithStateKey(charlie.ID)) + + // Send and create the room + if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // send a message + bobMsg := room.CreateAndInsert(t, bob, "m.room.message", map[string]any{"body": "hello world"}) + charlieMsg := room.CreateAndInsert(t, charlie, "m.room.message", map[string]any{"body": "hello world"}) + + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobMsg, charlieMsg}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Bob changes his name + expectedDisplayname := "Bob!" + bobDisplayname := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]any{"membership": "join", "displayname": expectedDisplayname}, test.WithStateKey(bob.ID)) + + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobDisplayname}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Change another state event + jrEv := room.CreateAndInsert(t, alice, spec.MRoomJoinRules, gomatrixserverlib.JoinRuleContent{JoinRule: "invite"}, test.WithStateKey("")) + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{jrEv}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // send a message + bobMsg = room.CreateAndInsert(t, bob, "m.room.message", map[string]any{"body": "hello world"}) + charlieMsg = room.CreateAndInsert(t, charlie, "m.room.message", map[string]any{"body": "hello world"}) + + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobMsg, charlieMsg}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Craft the state reset message, which is using Bobs initial join event and the + // last message Charlie sent as the prev_events. This should trigger the recalculation + // of the "current" state, since the message event does not have state and no missing events in the DB. + stateResetMsg := mustCreateEvent(t, fledglingEvent{ + Type: "m.room.message", + SenderID: charlie.ID, + RoomID: room.ID, + Depth: charlieMsg.Depth() + 1, + PrevEvents: []any{ + bobJoinEv.EventID(), + charlieMsg.EventID(), + }, + AuthEvents: []any{ + room.Events()[0].EventID(), // create event + room.Events()[2].EventID(), // PL event + charlieJoinEv.EventID(), // Charlie join event + }, + }) + + // Send the state reset message + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{stateResetMsg}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Validate that there is a membership event for Bob + bobMembershipEv := api.GetStateEvent(ctx, rsAPI, room.ID, gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomMember, + StateKey: bob.ID, + }) + + if bobMembershipEv == nil { + t.Fatalf("Membership event for Bob does not exist. State reset?") + } else { + // Validate it's the correct membership event + if dn := gjson.GetBytes(bobMembershipEv.Content(), "displayname").Str; dn != expectedDisplayname { + t.Fatalf("Expected displayname to be %q, got %q", expectedDisplayname, dn) + } + } + }) +} From c1d6b9aa8ef29a76b61b1aaf60572d3493adf306 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 09:11:58 +0200 Subject: [PATCH 17/40] Bump github.com/nats-io/nats-server/v2 from 2.9.19 to 2.9.23 (#3238) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/nats-io/nats-server/v2](https://github.com/nats-io/nats-server) from 2.9.19 to 2.9.23.
Release notes

Sourced from github.com/nats-io/nats-server/v2's releases.

Release v2.9.23

Changelog

Go Version

  • 1.20.10

Fixed

Accounts

  • Prevent bypassing authorization block when enabling system account access in accounts block (#4605). Backport from v2.10.2

Leafnodes

  • Prevent a leafnode cluster from receiving a message multiple times in a queue subscription (#4578). Backport from v2.10.2

JetStream

  • Hold lock when calculating the first message for subject in a message block (#4531). Backport from v2.10.0
  • Add self-healing mechanism to detect and delete orphaned Raft groups (#4647). Backport from v2.10.0
  • Prevent forward proposals in consumers after scaling down a stream (#4647). Backport from v2.10.0
  • Fix race condition during leader failover scenarios resulting in potential duplicate messages being sourced (#4592). Backport from v2.10.2

Complete Changes

https://github.com/nats-io/nats-server/compare/v2.9.22...v2.9.23

Release v2.9.22

Changelog

Go Version

  • 1.20.8 (updated out-of-cycle since Go 1.19 is now EOL)

Dependencies

  • github.com/nats-io/jwt/v2 v2.5.0
  • golang.org/x/crypto v0.12.0
  • golang.org/x/sys v0.11.0

Improved

Monitoring

  • CORS Allow-Origin passthrough for monitoring server (#4423) Thanks to @​mdawar for the contribution!

JetStream

  • Improve consumer scaling reliability with filters and cluster restart (#4404)
  • Send event on lame duck mode (LDM) to avoid placing assets on shutting down nodes (#4405)
  • Skip filestore tombstones if downgrade from 2.10 occurs (#4452)
  • Adjust delivered and waiting count when consumer message delivery fails (#4472)

Fixed

Config

  • Allow empty configs and fix JSON compatibility (#4394, #4418)
  • Remove TLS OCSP debug log on reload (#4453)

... (truncated)

Commits
  • 45436e1 Release v2.9.23 (#4652)
  • 72ffa38 Release v2.9.23
  • 05fe77f Backport #4592 to 2.9 (#4651)
  • 6a73e68 [2.9.x] Bump Travis Go version to 1.20.10 (#4650)
  • 8b981a2 Backports from v2.10 for v2.9.23 release (#4647)
  • 28eb7c0 Only setup auto no-auth for $G account iff no authorization block was defined.
  • 9f16edd Make sure to not forward a message across a route for dq sub when we are a sp...
  • 0ac7895 Add in utility to detect and delete any NRG orphans.
  • 50722e9 When scaling a consumer down make sure to pop the loopAndForwardProposals go ...
  • 770cf2e Backport JetStream benchmarks improvements to 2.9.x (#4644)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/nats-io/nats-server/v2&package-manager=go_modules&previous-version=2.9.19&new-version=2.9.23)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/matrix-org/dendrite/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 7d4c3677b..e8fbbb81e 100644 --- a/go.mod +++ b/go.mod @@ -26,8 +26,8 @@ require ( github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 - github.com/nats-io/nats-server/v2 v2.9.19 - github.com/nats-io/nats.go v1.27.0 + github.com/nats-io/nats-server/v2 v2.9.23 + github.com/nats-io/nats.go v1.28.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/opentracing/opentracing-go v1.2.0 @@ -94,7 +94,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -104,7 +104,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mschoch/smat v0.2.0 // indirect - github.com/nats-io/jwt/v2 v2.4.1 // indirect + github.com/nats-io/jwt/v2 v2.5.0 // indirect github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/onsi/ginkgo/v2 v2.11.0 // indirect diff --git a/go.sum b/go.sum index c1392cbd4..102ff3aa0 100644 --- a/go.sum +++ b/go.sum @@ -190,8 +190,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -242,12 +242,12 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= -github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= -github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA= -github.com/nats-io/nats-server/v2 v2.9.19/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw= -github.com/nats-io/nats.go v1.27.0 h1:3o9fsPhmoKm+yK7rekH2GtWoE+D9jFbw8N3/ayI1C00= -github.com/nats-io/nats.go v1.27.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= +github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= +github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= +github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU= +github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= +github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c= +github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= From 1b124fe9cba8b4699a1f65382e4ea1a5674d12fa Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 24 Oct 2023 11:51:08 +0200 Subject: [PATCH 18/40] Implement MSC3987, fix setting Element Android notifications (#3242) Should fix https://github.com/matrix-org/dendrite/issues/3183, since Element Android already implements [MSC3987](https://github.com/vector-im/element-android/pull/8530) This is also part of https://github.com/matrix-org/dendrite/issues/3225 --- clientapi/clientapi_test.go | 2 +- internal/pushrules/action_test.go | 2 -- internal/pushrules/default_override.go | 11 +++++------ internal/pushrules/default_pushrules_test.go | 6 +++--- internal/pushrules/util.go | 5 +---- internal/pushrules/util_test.go | 5 ++--- internal/pushrules/validate.go | 2 +- userapi/consumers/roomserver.go | 4 ++-- userapi/consumers/roomserver_test.go | 10 +++------- 9 files changed, 18 insertions(+), 29 deletions(-) diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index 82ec9fea2..f2d617cb9 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -1418,7 +1418,7 @@ func TestPushRules(t *testing.T) { validateFunc: func(t *testing.T, respBody *bytes.Buffer) { actions := gjson.GetBytes(respBody.Bytes(), "actions").Array() // only a basic check - assert.Equal(t, 1, len(actions)) + assert.Equal(t, 0, len(actions)) }, }, { diff --git a/internal/pushrules/action_test.go b/internal/pushrules/action_test.go index 72db9c998..d1eb16ef1 100644 --- a/internal/pushrules/action_test.go +++ b/internal/pushrules/action_test.go @@ -13,8 +13,6 @@ func TestActionJSON(t *testing.T) { Want Action }{ {Action{Kind: NotifyAction}}, - {Action{Kind: DontNotifyAction}}, - {Action{Kind: CoalesceAction}}, {Action{Kind: SetTweakAction}}, {Action{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, diff --git a/internal/pushrules/default_override.go b/internal/pushrules/default_override.go index f97427b71..cb825af5f 100644 --- a/internal/pushrules/default_override.go +++ b/internal/pushrules/default_override.go @@ -10,6 +10,7 @@ func defaultOverrideRules(userID string) []*Rule { &mRuleRoomNotifDefinition, &mRuleTombstoneDefinition, &mRuleReactionDefinition, + &mRuleACLsDefinition, } } @@ -30,7 +31,7 @@ var ( RuleID: MRuleMaster, Default: true, Enabled: false, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleSuppressNoticesDefinition = Rule{ RuleID: MRuleSuppressNotices, @@ -43,7 +44,7 @@ var ( Pattern: pointer("m.notice"), }, }, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleMemberEventDefinition = Rule{ RuleID: MRuleMemberEvent, @@ -56,7 +57,7 @@ var ( Pattern: pointer("m.room.member"), }, }, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleContainsDisplayNameDefinition = Rule{ RuleID: MRuleContainsDisplayName, @@ -152,9 +153,7 @@ var ( Pattern: pointer("m.reaction"), }, }, - Actions: []*Action{ - {Kind: DontNotifyAction}, - }, + Actions: []*Action{}, } ) diff --git a/internal/pushrules/default_pushrules_test.go b/internal/pushrules/default_pushrules_test.go index dea829842..506fe3cc8 100644 --- a/internal/pushrules/default_pushrules_test.go +++ b/internal/pushrules/default_pushrules_test.go @@ -21,12 +21,12 @@ func TestDefaultRules(t *testing.T) { // Default override rules { name: ".m.rule.master", - inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":[]}`), want: mRuleMasterDefinition, }, { name: ".m.rule.suppress_notices", - inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":[]}`), want: mRuleSuppressNoticesDefinition, }, { @@ -36,7 +36,7 @@ func TestDefaultRules(t *testing.T) { }, { name: ".m.rule.member_event", - inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":[]}`), want: mRuleMemberEventDefinition, }, { diff --git a/internal/pushrules/util.go b/internal/pushrules/util.go index de8fe5cd0..e2821b57a 100644 --- a/internal/pushrules/util.go +++ b/internal/pushrules/util.go @@ -16,10 +16,7 @@ func ActionsToTweaks(as []*Action) (ActionKind, map[string]interface{}, error) { for _, a := range as { switch a.Kind { - case DontNotifyAction: - // Don't bother processing any further - return DontNotifyAction, nil, nil - + case DontNotifyAction: // Ignored case SetTweakAction: if tweaks == nil { tweaks = map[string]interface{}{} diff --git a/internal/pushrules/util_test.go b/internal/pushrules/util_test.go index 89f8243d9..83eee7ede 100644 --- a/internal/pushrules/util_test.go +++ b/internal/pushrules/util_test.go @@ -17,17 +17,16 @@ func TestActionsToTweaks(t *testing.T) { {"empty", nil, UnknownAction, nil}, {"zero", []*Action{{}}, UnknownAction, nil}, {"onlyPrimary", []*Action{{Kind: NotifyAction}}, NotifyAction, nil}, - {"onlyPrimaryDontNotify", []*Action{{Kind: DontNotifyAction}}, DontNotifyAction, nil}, + {"onlyPrimaryDontNotify", []*Action{}, UnknownAction, nil}, {"onlyTweak", []*Action{{Kind: SetTweakAction, Tweak: HighlightTweak}}, UnknownAction, map[string]interface{}{"highlight": nil}}, {"onlyTweakWithValue", []*Action{{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, UnknownAction, map[string]interface{}{"sound": "default"}}, { "all", []*Action{ - {Kind: CoalesceAction}, {Kind: SetTweakAction, Tweak: HighlightTweak}, {Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}, }, - CoalesceAction, + UnknownAction, map[string]interface{}{"highlight": nil, "sound": "default"}, }, } diff --git a/internal/pushrules/validate.go b/internal/pushrules/validate.go index b54ec3fb0..4cc479345 100644 --- a/internal/pushrules/validate.go +++ b/internal/pushrules/validate.go @@ -18,7 +18,7 @@ func ValidateRule(kind Kind, rule *Rule) []error { errs = append(errs, fmt.Errorf("invalid rule ID: %s", rule.RuleID)) } - if len(rule.Actions) == 0 { + if rule.Actions == nil { errs = append(errs, fmt.Errorf("missing actions")) } for _, action := range rule.Actions { diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 6da41f8a1..fca741298 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -538,8 +538,8 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype if err != nil { return fmt.Errorf("pushrules.ActionsToTweaks: %w", err) } - // TODO: support coalescing. - if a != pushrules.NotifyAction && a != pushrules.CoalesceAction { + + if a != pushrules.NotifyAction { log.WithFields(log.Fields{ "event_id": event.EventID(), "room_id": event.RoomID().String(), diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index 49dd5b238..7b7c08618 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -81,12 +81,8 @@ func Test_evaluatePushRules(t *testing.T) { { name: "m.reaction doesn't notify", eventContent: `{"type":"m.reaction","room_id":"!room:example.com"}`, - wantAction: pushrules.DontNotifyAction, - wantActions: []*pushrules.Action{ - { - Kind: pushrules.DontNotifyAction, - }, - }, + wantAction: pushrules.UnknownAction, + wantActions: []*pushrules.Action{}, }, { name: "m.room.message notifies", @@ -136,7 +132,7 @@ func Test_evaluatePushRules(t *testing.T) { t.Fatalf("expected action to be '%s', got '%s'", tc.wantAction, gotAction) } // this is taken from `notifyLocal` - if tc.wantNotify && gotAction != pushrules.NotifyAction && gotAction != pushrules.CoalesceAction { + if tc.wantNotify && gotAction != pushrules.NotifyAction { t.Fatalf("expected to notify but didn't") } }) From 4fa8512d576ca2a425e6cc546264934d959bced5 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 25 Oct 2023 09:47:21 +0200 Subject: [PATCH 19/40] Check event is not rejected (#3243) Companion PR to https://github.com/matrix-org/gomatrixserverlib/pull/421 --- cmd/resolve-state/main.go | 13 +++++++++++++ go.mod | 2 +- go.sum | 4 ++-- roomserver/internal/input/input_missing.go | 7 +++++++ roomserver/internal/query/query.go | 14 ++++++++++++++ roomserver/state/state.go | 8 ++++++++ roomserver/storage/shared/room_updater.go | 4 ++++ 7 files changed, 49 insertions(+), 3 deletions(-) diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index 5be449097..d6db72436 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -202,12 +202,25 @@ func main() { authEvents[i] = authEventEntries[i].PDU } + // Get the roomNID + roomInfo, err = roomserverDB.RoomInfo(ctx, authEvents[0].RoomID().String()) + if err != nil { + panic(err) + } + fmt.Println("Resolving state") var resolved Events resolved, err = gomatrixserverlib.ResolveConflicts( gomatrixserverlib.RoomVersion(*roomVersion), events, authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, rejectedErr := roomserverDB.IsEventRejected(ctx, roomInfo.RoomNID, eventID) + if rejectedErr != nil { + return true + } + return isRejected + }, ) if err != nil { panic(err) diff --git a/go.mod b/go.mod index e8fbbb81e..de60d9b4e 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be + github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 diff --git a/go.sum b/go.sum index 102ff3aa0..955cdc11c 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be h1:bZP16ydP8uRoRBo1p/7WHMexjg7JJGj81fKzZ1FULb4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca h1:JCP72vU4Vcmur2071RwYVOSoekR+ZjbC03wZD5lAAK0= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go index d9ab291e9..21493287e 100644 --- a/roomserver/internal/input/input_missing.go +++ b/roomserver/internal/input/input_missing.go @@ -498,6 +498,13 @@ func (t *missingStateReq) resolveStatesAndCheck(ctx context.Context, roomVersion roomVersion, gomatrixserverlib.ToPDUs(stateEventList), gomatrixserverlib.ToPDUs(authEventList), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return t.inputer.Queryer.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, err := t.db.IsEventRejected(ctx, t.roomInfo.RoomNID, eventID) + if err != nil { + return true + } + return isRejected + }, ) if err != nil { return nil, err diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index f87a3f7ed..74b010281 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -165,6 +165,13 @@ func (r *Queryer) QueryStateAfterEvents( info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return r.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, rejectedErr := r.DB.IsEventRejected(ctx, info.RoomNID, eventID) + if rejectedErr != nil { + return true + } + return isRejected + }, ) if err != nil { return fmt.Errorf("state.ResolveConflictsAdhoc: %w", err) @@ -676,6 +683,13 @@ func (r *Queryer) QueryStateAndAuthChain( info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return r.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, rejectedErr := r.DB.IsEventRejected(ctx, info.RoomNID, eventID) + if rejectedErr != nil { + return true + } + return isRejected + }, ) if err != nil { return err diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 1e776ff6c..dfd439a22 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -45,6 +45,7 @@ type StateResolutionStorage interface { AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error) Events(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, eventNIDs []types.EventNID) ([]types.Event, error) EventsFromIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]types.Event, error) + IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error) } type StateResolution struct { @@ -1066,6 +1067,13 @@ func (v *StateResolution) resolveConflictsV2( func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return v.Querier.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, err := v.db.IsEventRejected(ctx, v.roomInfo.RoomNID, eventID) + if err != nil { + return true + } + return isRejected + }, ) }() diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go index 70672a33e..06284d2e3 100644 --- a/roomserver/storage/shared/room_updater.go +++ b/roomserver/storage/shared/room_updater.go @@ -250,3 +250,7 @@ func (u *RoomUpdater) MarkEventAsSent(eventNID types.EventNID) error { func (u *RoomUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (*MembershipUpdater, error) { return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomInfo.RoomNID, targetUserNID, targetLocal) } + +func (u *RoomUpdater) IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error) { + return u.d.IsEventRejected(ctx, roomNID, eventID) +} From e02a7948d8556398ceb345a241c175b5ca1d011f Mon Sep 17 00:00:00 2001 From: WrenIX <133280015+wrenix@users.noreply.github.com> Date: Wed, 25 Oct 2023 10:08:54 +0200 Subject: [PATCH 20/40] fix(helm): empty storage class in pvcs (#3191) fix #3103 --- not yet tested [skip ci] --- helm/dendrite/templates/pvc.yaml | 27 ++++++++++++++++++++++++--- helm/dendrite/values.yaml | 32 ++++++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/helm/dendrite/templates/pvc.yaml b/helm/dendrite/templates/pvc.yaml index 88eff3bed..70b1ce563 100644 --- a/helm/dendrite/templates/pvc.yaml +++ b/helm/dendrite/templates/pvc.yaml @@ -12,7 +12,14 @@ spec: resources: requests: storage: {{ .Values.persistence.media.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.media.storageClass }} + {{ $storageClass := .Values.persistence.media.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} {{ if not .Values.persistence.jetstream.existingClaim }} --- @@ -28,7 +35,14 @@ spec: resources: requests: storage: {{ .Values.persistence.jetstream.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.jetstream.storageClass }} + {{ $storageClass := .Values.persistence.jetstream.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} {{ if not .Values.persistence.search.existingClaim }} --- @@ -44,5 +58,12 @@ spec: resources: requests: storage: {{ .Values.persistence.search.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.search.storageClass }} + {{ $storageClass := .Values.persistence.search.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index 8a72f6693..afce1d930 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -26,7 +26,13 @@ persistence: # -- The storage class to use for volume claims. # Used unless specified at the specific component. # Defaults to the cluster default storage class. - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" jetstream: # -- Use an existing volume claim for jetstream existingClaim: "" @@ -34,7 +40,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" media: # -- Use an existing volume claim for media files existingClaim: "" @@ -42,7 +54,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" search: # -- Use an existing volume claim for the fulltext search index existingClaim: "" @@ -50,7 +68,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" # -- Add additional volumes to the Dendrite Pod extraVolumes: [] From a0375d41fbbdabd98df743d2e7fa77b4d0c44d4b Mon Sep 17 00:00:00 2001 From: devonh Date: Wed, 25 Oct 2023 08:13:18 +0000 Subject: [PATCH 21/40] Add simple test for one time keys (#3239) --- userapi/storage/storage_test.go | 51 +++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index a46ee9ebb..5a789dfd7 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -1,6 +1,7 @@ package storage_test import ( + "bytes" "context" "encoding/json" "fmt" @@ -758,3 +759,53 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) { } }) } + +func TestOneTimeKeys(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, clean := mustCreateKeyDatabase(t, dbType) + defer clean() + userID := "@alice:localhost" + deviceID := "alice_device" + otk := api.OneTimeKeys{ + UserID: userID, + DeviceID: deviceID, + KeyJSON: map[string]json.RawMessage{"curve25519:KEY1": []byte(`{"key":"v1"}`)}, + } + + // Add a one time key to the DB + _, err := db.StoreOneTimeKeys(ctx, otk) + MustNotError(t, err) + + // Check the count of one time keys is correct + count, err := db.OneTimeKeysCount(ctx, userID, deviceID) + MustNotError(t, err) + if count.KeyCount["curve25519"] != 1 { + t.Fatalf("Expected 1 key, got %d", count.KeyCount["curve25519"]) + } + + // Check the actual key contents are correct + keysJSON, err := db.ExistingOneTimeKeys(ctx, userID, deviceID, []string{"curve25519:KEY1"}) + MustNotError(t, err) + keyJSON, err := keysJSON["curve25519:KEY1"].MarshalJSON() + MustNotError(t, err) + if !bytes.Equal(keyJSON, []byte(`{"key":"v1"}`)) { + t.Fatalf("Existing keys do not match expected. Got %v", keysJSON["curve25519:KEY1"]) + } + + // Claim a one time key from the database. This should remove it from the database. + claimedKeys, err := db.ClaimKeys(ctx, map[string]map[string]string{userID: {deviceID: "curve25519"}}) + MustNotError(t, err) + + // Check the claimed key contents are correct + if !reflect.DeepEqual(claimedKeys[0], otk) { + t.Fatalf("Expected to claim stored key %v. Got %v", otk, claimedKeys[0]) + } + + // Check the count of one time keys is now zero + count, err = db.OneTimeKeysCount(ctx, userID, deviceID) + MustNotError(t, err) + if count.KeyCount["curve25519"] != 0 { + t.Fatalf("Expected 0 keys, got %d", count.KeyCount["curve25519"]) + } + }) +} From 89482ad7902d6cacddc29f8eac62ab66ff4ebf24 Mon Sep 17 00:00:00 2001 From: CicadaCinema <52425971+CicadaCinema@users.noreply.github.com> Date: Wed, 25 Oct 2023 09:24:06 +0100 Subject: [PATCH 22/40] clean up dead links, fix typo (#3130) I fixed any dead links beginning https://matrix.org/speculator and some issues I found along the way. https://web.archive.org/web/20190329152312/https://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api is now found at https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api https://web.archive.org/web/20170620093435/https://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register is now found at https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register https://github.com/matrix-org/matrix-spec/blob/2a8d64fef7a40717ef9f5748ee0551b2117be037/specification/intro.rst?plain=1#L443 is now found at https://spec.matrix.org/v1.7/appendices/#user-identifiers --- clientapi/routing/register.go | 6 +++--- docs/development/CONTRIBUTING.md | 2 +- internal/validate.go | 2 +- setup/config/config_clientapi.go | 20 +++++++++----------- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 565c41533..090a2fc20 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -236,7 +236,7 @@ type authDict struct { // TODO: Lots of custom keys depending on the type } -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api +// https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api type userInteractiveResponse struct { Flows []authtypes.Flow `json:"flows"` Completed []authtypes.LoginType `json:"completed"` @@ -256,7 +256,7 @@ func newUserInteractiveResponse( } } -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register +// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register type registerResponse struct { UserID string `json:"user_id"` AccessToken string `json:"access_token,omitempty"` @@ -462,7 +462,7 @@ func validateApplicationService( } // Register processes a /register request. -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register +// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register func Register( req *http.Request, userAPI userapi.ClientUserAPI, diff --git a/docs/development/CONTRIBUTING.md b/docs/development/CONTRIBUTING.md index 71e7516a2..caab1e749 100644 --- a/docs/development/CONTRIBUTING.md +++ b/docs/development/CONTRIBUTING.md @@ -109,7 +109,7 @@ To configure the connection to a remote Postgres, you can use the following envi ```bash POSTGRES_USER=postgres -POSTGERS_PASSWORD=yourPostgresPassword +POSTGRES_PASSWORD=yourPostgresPassword POSTGRES_HOST=localhost POSTGRES_DB=postgres # the superuser database to use ``` diff --git a/internal/validate.go b/internal/validate.go index 99088f240..7f0d8b9e6 100644 --- a/internal/validate.go +++ b/internal/validate.go @@ -25,7 +25,7 @@ import ( ) const ( - maxUsernameLength = 254 // http://matrix.org/speculator/spec/HEAD/intro.html#user-identifiers TODO account for domain + maxUsernameLength = 254 // https://spec.matrix.org/v1.7/appendices/#user-identifiers TODO account for domain minPasswordLength = 8 // http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based maxPasswordLength = 512 // https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161 diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 44136e2a0..85dfe0beb 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -93,17 +93,15 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors) { checkNotEmpty(configErrs, "client_api.recaptcha_sitekey_class", c.RecaptchaSitekeyClass) } // Ensure there is any spam counter measure when enabling registration - if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled { - if !c.RecaptchaEnabled { - configErrs.Add( - "You have tried to enable open registration without any secondary verification methods " + - "(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " + - "increasing the risk that your server will be used to send spam or abuse, and may result in " + - "your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " + - "start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " + - "should set the registration_disabled option in your Dendrite config.", - ) - } + if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled && !c.RecaptchaEnabled { + configErrs.Add( + "You have tried to enable open registration without any secondary verification methods " + + "(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " + + "increasing the risk that your server will be used to send spam or abuse, and may result in " + + "your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " + + "start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " + + "should set the registration_disabled option in your Dendrite config.", + ) } } From 317b1018a38fbbc573f095b880beba5b1fab60be Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:53:40 +0200 Subject: [PATCH 23/40] Version 0.13.4 (#3244) If I didn't mess up the workflow, this should remove some ugliness from the version string (e.g. 0.13.2+57ddbe0.57ddbe0, dupe commit hash, as a result of https://github.com/matrix-org/dendrite/pull/3147) --- .github/workflows/docker.yml | 18 ------------------ CHANGES.md | 18 ++++++++++++++++++ Dockerfile | 5 ++--- build/docker/Dockerfile.demo-yggdrasil | 2 +- helm/dendrite/Chart.yaml | 4 ++-- helm/dendrite/README.md | 7 ++----- internal/version.go | 2 +- 7 files changed, 26 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8448d8e23..8d3a8d674 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -32,10 +32,6 @@ jobs: if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -60,7 +56,6 @@ jobs: cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -75,7 +70,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -109,10 +103,6 @@ jobs: if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -137,7 +127,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true @@ -153,7 +142,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true @@ -176,10 +164,6 @@ jobs: if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -204,7 +188,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-yggdrasil platforms: ${{ env.PLATFORMS }} push: true @@ -220,7 +203,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-yggdrasil platforms: ${{ env.PLATFORMS }} push: true diff --git a/CHANGES.md b/CHANGES.md index bdb6a796e..57e3a3d4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,23 @@ # Changelog +## Dendrite 0.13.4 (2023-10-25) + +Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution +algorithm. + +### Fixes: + +- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also + avoids spamming the logs when shutting down.) +- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset" + out of rooms and other missing state events due to calculating the wrong state. +- A bug when setting notifications from Element Android has been fixed by implementing MSC3987 + +### Features + +- Updated dependencies + - Internal NATS Server has been updated from v2.9.19 to v2.9.23 + ## Dendrite 0.13.3 (2023-09-28) ### Fixes: diff --git a/Dockerfile b/Dockerfile index 4ee20933a..8c8f1588f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # # base installs required dependencies and runs go mod download to cache dependencies # -FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base +FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine AS base RUN apk --update --no-cache add bash build-base curl git # @@ -13,7 +13,6 @@ FROM --platform=${BUILDPLATFORM} base AS build WORKDIR /src ARG TARGETOS ARG TARGETARCH -ARG FLAGS RUN --mount=target=. \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ @@ -21,7 +20,7 @@ RUN --mount=target=. \ GOARCH="$TARGETARCH" \ GOOS="linux" \ CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \ - go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/... + go build -v -trimpath -o /out/ ./cmd/... # diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil index efae5496c..b9e387666 100644 --- a/build/docker/Dockerfile.demo-yggdrasil +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.19-alpine AS base +FROM docker.io/golang:1.21-alpine AS base # # Needs to be separate from the main Dockerfile for OpenShift, diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 674152051..32f479960 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: dendrite -version: "0.13.4" -appVersion: "0.13.3" +version: "0.13.5" +appVersion: "0.13.4" description: Dendrite Matrix Homeserver type: application keywords: diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index cb78a54f2..22daa1813 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,7 +1,7 @@ # dendrite -![Version: 0.13.4](https://img.shields.io/badge/Version-0.13.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.3](https://img.shields.io/badge/AppVersion-0.13.3-informational?style=flat-square) +![Version: 0.13.5](https://img.shields.io/badge/Version-0.13.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.4](https://img.shields.io/badge/AppVersion-0.13.4-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -48,16 +48,13 @@ Create a folder `appservices` and place your configurations in there. The confi | signing_key.create | bool | `true` | Create a new signing key, if not exists | | signing_key.existingSecret | string | `""` | Use an existing secret | | resources | object | sets some sane default values | Default resource requests/limits. | -| persistence.storageClass | string | `""` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. | +| persistence.jetstream | object | `{"capacity":"1Gi","existingClaim":""}` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. # If defined, storageClassName: # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. (gp2 on AWS, standard on # GKE, AWS & OpenStack) # storageClass: "" | | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | -| persistence.jetstream.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | -| persistence.media.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | -| persistence.search.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | | strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | diff --git a/internal/version.go b/internal/version.go index 1f8a62bc2..55726ddcb 100644 --- a/internal/version.go +++ b/internal/version.go @@ -18,7 +18,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 13 - VersionPatch = 3 + VersionPatch = 4 VersionTag = "" // example: "rc1" gitRevLen = 7 // 7 matches the displayed characters on github.com From 32f7c4b166c5e74ef34973a1d6a30e5e2d75c3ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Oct 2023 07:59:19 +0100 Subject: [PATCH 24/40] Bump github.com/docker/docker from 24.0.5+incompatible to 24.0.7+incompatible (#3250) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 24.0.5+incompatible to 24.0.7+incompatible.
Release notes

Sourced from github.com/docker/docker's releases.

v24.0.7

24.0.7

For a full list of pull requests and changes in this release, refer to the relevant GitHub milestones:

Bug fixes and enhancements

  • Write overlay2 layer metadata atomically. moby/moby#46703
  • Fix "Rootful-in-Rootless" Docker-in-Docker on systemd version 250 and later. moby/moby#46626
  • Fix dockerd-rootless-setuptools.sh when username contains a backslash. moby/moby#46407
  • Fix a bug that would prevent network sandboxes to be fully deleted when stopping containers with no network attachments and when dockerd --bridge=none is used. moby/moby#46702
  • Fix a bug where cancelling an API request could interrupt container restart. moby/moby#46697
  • Fix an issue where containers would fail to start when providing --ip-range with a range larger than the subnet. docker/for-mac#6870
  • Fix data corruption with zstd output. moby/moby#46709
  • Fix the conditions under which the container's MAC address is applied. moby/moby#46478
  • Improve the performance of the stats collector. moby/moby#46448
  • Fix an issue with source policy rules ending up in the wrong order. moby/moby#46441

Packaging updates

Security

v24.0.6

24.0.6

For a full list of pull requests and changes in this release, refer to the relevant GitHub milestones:

Bug fixes and enhancements

  • containerd storage backend: Fix docker ps failing when a container image is no longer present in the content store. moby/moby#46095
  • containerd storage backend: Fix docker ps -s -a and docker container prune failing when a container image config is no longer present in the content store. moby/moby#46097
  • containerd storage backend: Fix docker inspect failing when a container image config is no longer (or was never) present in the content store. moby/moby#46244
  • containerd storage backend: Fix diff and export with the overlayfs snapshotter by using reference-counted rootfs mounts. moby/moby#46266
  • containerd storage backend: Fix a misleading error message when the image platforms available locally do not match the desired platform. moby/moby#46300
  • containerd storage backend: Fix the FROM scratch Dockerfile instruction with the classic builder. moby/moby#46302
  • containerd storage backend: Fix mismatched image rootfs and manifest layers errors with the classic builder. moby/moby#46310

... (truncated)

Commits
  • 311b9ff Merge pull request #46697 from thaJeztah/24.0_backport_restart_nocancel
  • af60804 Merge pull request from GHSA-jq35-85cj-fj4p
  • 3cf363e Merge pull request #46709 from thaJeztah/24.0_backport_bump_compress
  • 05d7386 daemon: daemon.containerRestart: don't cancel restart on context cancel
  • 649c944 Merge pull request #46703 from thaJeztah/24.0_backport_atomic-layer-data-write
  • 9b20b1a Merge pull request #46702 from thaJeztah/24.0_backport_releaseNetwork_Network...
  • dd37b0b vendor: github.com/klauspost/compress v1.17.2
  • 7058c0d vendor: github.com/klauspost/compress v1.16.5
  • 57bd388 daemon: overlay2: Write layer metadata atomically
  • 05d95fd daemon: release sandbox even when NetworkDisabled
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/docker/docker&package-manager=go_modules&previous-version=24.0.5+incompatible&new-version=24.0.7+incompatible)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/matrix-org/dendrite/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index de60d9b4e..9e7d278fe 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.8 github.com/codeclysm/extract v2.2.0+incompatible github.com/dgraph-io/ristretto v0.1.1 - github.com/docker/docker v24.0.5+incompatible + github.com/docker/docker v24.0.7+incompatible github.com/docker/go-connections v0.4.0 github.com/getsentry/sentry-go v0.14.0 github.com/gologme/log v1.3.0 diff --git a/go.sum b/go.sum index 955cdc11c..e9da63f12 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= -github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= From da7bca0224760a4fe0e10876a9c11da333513a29 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 31 Oct 2023 16:39:45 +0100 Subject: [PATCH 25/40] Some tweaks for the device list updater (#3251) This makes the following changes: - Adds two new metrics observing the usage of the `DeviceListUpdater` workers - Makes the number of workers configurable - Adds a 30s timeout for DB requests when receiving a device list update over federation --- appservice/appservice_test.go | 6 ++-- build/dendritejs-pinecone/main.go | 3 +- build/gobind-yggdrasil/monolith.go | 2 +- clientapi/admin_test.go | 20 +++++------ clientapi/clientapi_test.go | 26 +++++++-------- clientapi/routing/joinroom_test.go | 2 +- clientapi/routing/login_test.go | 2 +- clientapi/routing/register_test.go | 6 ++-- .../monolith/monolith.go | 2 +- cmd/dendrite-demo-yggdrasil/main.go | 2 +- cmd/dendrite/main.go | 2 +- dendrite-sample.yaml | 4 +++ roomserver/roomserver_test.go | 6 ++-- setup/config/config_userapi.go | 5 +++ userapi/consumers/devicelistupdate.go | 6 +++- userapi/internal/device_list_update.go | 33 +++++++++++++++++-- userapi/internal/device_list_update_test.go | 9 ++--- userapi/userapi.go | 3 +- 18 files changed, 92 insertions(+), 47 deletions(-) diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index ddc24477b..bbdeb47df 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -144,7 +144,7 @@ func TestAppserviceInternalAPI(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) runCases(t, asAPI) @@ -239,7 +239,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) t.Run("UserIDExists", func(t *testing.T) { @@ -378,7 +378,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) { // Create required internal APIs rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics) // start the consumer appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI) diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go index 61baed902..d3d5f59fe 100644 --- a/build/dendritejs-pinecone/main.go +++ b/build/dendritejs-pinecone/main.go @@ -38,6 +38,7 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/gomatrixserverlib" @@ -190,7 +191,7 @@ func startup() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics) asQuery := appservice.NewInternalAPI( processCtx, cfg, &natsInstance, userAPI, rsAPI, diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 720ce37eb..791ad261b 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() { processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 66667b03c..b228dd93c 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -45,7 +45,7 @@ func TestAdminCreateToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -196,7 +196,7 @@ func TestAdminListRegistrationTokens(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -314,7 +314,7 @@ func TestAdminGetRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -415,7 +415,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -509,7 +509,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -693,7 +693,7 @@ func TestAdminResetPassword(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for changing the password/login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -791,7 +791,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -863,7 +863,7 @@ func TestAdminEvacuateRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -964,7 +964,7 @@ func TestAdminEvacuateUser(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -1055,7 +1055,7 @@ func TestAdminMarkAsStale(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index f2d617cb9..2bb15fba3 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -121,7 +121,7 @@ func TestGetPutDevices(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -170,7 +170,7 @@ func TestDeleteDevice(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -275,7 +275,7 @@ func TestDeleteDevices(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -442,7 +442,7 @@ func TestSetDisplayname(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -554,7 +554,7 @@ func TestSetAvatarURL(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -632,7 +632,7 @@ func TestTyping(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -716,7 +716,7 @@ func TestMembership(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -955,7 +955,7 @@ func TestCapabilities(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1002,7 +1002,7 @@ func TestTurnserver(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) //rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1100,7 +1100,7 @@ func Test3PID(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1276,7 +1276,7 @@ func TestPushRules(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1663,7 +1663,7 @@ func TestKeys(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -2125,7 +2125,7 @@ func TestKeyBackup(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 933ea8d3a..be3fb429f 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -36,7 +36,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) // Create the users in the userapi diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index 252017db2..4c4fc3533 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -49,7 +49,7 @@ func TestLogin(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for /login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics) diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 0a1986cf7..69b29e9c2 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -416,7 +416,7 @@ func Test_register(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -596,7 +596,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" response := completeRegistration( @@ -637,7 +637,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) expectedDisplayName := "rabbit" jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 41af568a6..5222d0b8e 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -145,7 +145,7 @@ func (p *P2PMonolith) SetupDendrite( ) rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 3ec550113..b07687538 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -213,7 +213,7 @@ func main() { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index f3140b4e2..f38263c6c 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -162,7 +162,7 @@ func main() { // dependency. Other components also need updating after their dependencies are up. rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml index 7affc2599..e143a7398 100644 --- a/dendrite-sample.yaml +++ b/dendrite-sample.yaml @@ -325,6 +325,10 @@ user_api: auto_join_rooms: # - "#main:matrix.org" + # The number of workers to start for the DeviceListUpdater. Defaults to 8. + # This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely. + # worker_count: 8 + # Configuration for Opentracing. # See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on # how this works and how to set it up. diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 90e67b699..22d27ba05 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -58,7 +58,7 @@ func TestUsers(t *testing.T) { }) t.Run("kick users", func(t *testing.T) { - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) rsAPI.SetUserAPI(usrAPI) testKickUsers(t, rsAPI, usrAPI) }) @@ -258,7 +258,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -1050,7 +1050,7 @@ func TestUpgrade(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) rsAPI.SetUserAPI(userAPI) for _, tc := range testCases { diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go index e64a3910c..559de72ac 100644 --- a/setup/config/config_userapi.go +++ b/setup/config/config_userapi.go @@ -21,6 +21,10 @@ type UserAPI struct { // Users who register on this homeserver will automatically // be joined to the rooms listed under this option. AutoJoinRooms []string `yaml:"auto_join_rooms"` + + // The number of workers to start for the DeviceListUpdater. Defaults to 8. + // This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely. + WorkerCount int `yaml:"worker_count"` } const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes @@ -28,6 +32,7 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes func (c *UserAPI) Defaults(opts DefaultOpts) { c.BCryptCost = bcrypt.DefaultCost c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS + c.WorkerCount = 8 if opts.Generate { if !opts.SingleDatabase { c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" diff --git a/userapi/consumers/devicelistupdate.go b/userapi/consumers/devicelistupdate.go index 3389bb808..b3ccb573b 100644 --- a/userapi/consumers/devicelistupdate.go +++ b/userapi/consumers/devicelistupdate.go @@ -17,6 +17,7 @@ package consumers import ( "context" "encoding/json" + "time" "github.com/matrix-org/dendrite/userapi/internal" "github.com/matrix-org/gomatrixserverlib" @@ -82,7 +83,10 @@ func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M return true } - err := t.updater.Update(ctx, m) + timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + + err := t.updater.Update(timeoutCtx, m) if err != nil { logrus.WithFields(logrus.Fields{ "user_id": m.UserID, diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go index 2f33589fe..a4d28188a 100644 --- a/userapi/internal/device_list_update.go +++ b/userapi/internal/device_list_update.go @@ -21,6 +21,7 @@ import ( "fmt" "hash/fnv" "net" + "strconv" "sync" "time" @@ -142,13 +143,36 @@ type KeyChangeProducer interface { ProduceKeyChanges(keys []api.DeviceMessage) error } +var deviceListUpdaterBackpressure = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "keyserver", + Name: "worker_backpressure", + Help: "How many device list updater requests are queued", + }, + []string{"worker_id"}, +) +var deviceListUpdaterServersRetrying = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "keyserver", + Name: "worker_servers_retrying", + Help: "How many servers are queued for retry", + }, + []string{"worker_id"}, +) + // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale. func NewDeviceListUpdater( process *process.ProcessContext, db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer, fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int, rsAPI rsapi.KeyserverRoomserverAPI, thisServer spec.ServerName, + enableMetrics bool, ) *DeviceListUpdater { + if enableMetrics { + prometheus.MustRegister(deviceListUpdaterBackpressure, deviceListUpdaterServersRetrying) + } return &DeviceListUpdater{ process: process, userIDToMutex: make(map[string]*sync.Mutex), @@ -173,7 +197,7 @@ func (u *DeviceListUpdater) Start() error { // to stop (in this transaction) until key requests can be made. ch := make(chan spec.ServerName, 10) u.workerChans[i] = ch - go u.worker(ch) + go u.worker(ch, i) } staleLists, err := u.db.StaleDeviceLists(u.process.Context(), []spec.ServerName{}) @@ -343,6 +367,8 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) { index := int(int64(hash.Sum32()) % int64(len(u.workerChans))) ch := u.assignChannel(userID) + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Inc() + defer deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Dec() u.workerChans[index] <- remoteServer select { case <-ch: @@ -372,7 +398,7 @@ func (u *DeviceListUpdater) clearChannel(userID string) { } } -func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { +func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) { retries := make(map[spec.ServerName]time.Time) retriesMu := &sync.Mutex{} // restarter goroutine which will inject failed servers into ch when it is time @@ -391,9 +417,12 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { for _, srv := range serversToRetry { delete(retries, srv) } + deviceListUpdaterServersRetrying.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Set(float64(len(retries))) retriesMu.Unlock() for _, srv := range serversToRetry { + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Inc() ch <- srv + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() } } }() diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 38fd8b583..14a49bc54 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" @@ -161,7 +162,7 @@ func TestUpdateHavePrevID(t *testing.T) { } ap := &mockDeviceListUpdaterAPI{} producer := &mockKeyChangeProducer{} - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost", caching.DisableMetrics) event := gomatrixserverlib.DeviceListUpdateEvent{ DeviceDisplayName: "Foo Bar", Deleted: false, @@ -233,7 +234,7 @@ func TestUpdateNoPrevID(t *testing.T) { `)), }, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test", caching.DisableMetrics) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -303,7 +304,7 @@ func TestDebounce(t *testing.T) { close(incomingFedReq) return <-fedCh, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost", caching.DisableMetrics) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -406,7 +407,7 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) { updater := NewDeviceListUpdater(processCtx, db, nil, nil, nil, - 0, rsAPI, "test") + 0, rsAPI, "test", caching.DisableMetrics) if err := updater.CleanUp(); err != nil { t.Error(err) } diff --git a/userapi/userapi.go b/userapi/userapi.go index 6b6dac884..34bf119a0 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -46,6 +46,7 @@ func NewInternalAPI( natsInstance *jetstream.NATSInstance, rsAPI rsapi.UserRoomserverAPI, fedClient fedsenderapi.KeyserverFederationAPI, + enableMetrics bool, ) *internal.UserInternalAPI { js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream) appServices := dendriteCfg.Derived.ApplicationServices @@ -99,7 +100,7 @@ func NewInternalAPI( FedClient: fedClient, } - updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, dendriteCfg.Global.ServerName) // 8 workers TODO: configurable + updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, dendriteCfg.UserAPI.WorkerCount, rsAPI, dendriteCfg.Global.ServerName, enableMetrics) userAPI.Updater = updater // Remove users which we don't share a room with anymore if err := updater.CleanUp(); err != nil { From 8b4043473c824b9f6233f19fa9cacf53dc5f1496 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Nov 2023 12:00:59 +0100 Subject: [PATCH 26/40] Bump github.com/nats-io/nkeys from 0.4.4 to 0.4.6 (#3252) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/nats-io/nkeys](https://github.com/nats-io/nkeys) from 0.4.4 to 0.4.6.
Release notes

Sourced from github.com/nats-io/nkeys's releases.

v0.4.5

What's Changed

Full Changelog: https://github.com/nats-io/nkeys/compare/v0.4.4...v0.4.5

Commits
  • 62e5d8c Merge pull request #60 from nats-io/0_4_6
  • f63761b [BUMP] release version and dependencies
  • d2e442e Merge pull request #59 from nats-io/empty
  • 58fb9d6 Make sure to use byte slice to receive proper copy, otherwise empty public ke...
  • 3e454c8 Merge pull request #58 from nats-io/arch-bins
  • 53c0777 bump go to 1.21.x
  • d935834 bump version number
  • 6b488b3 [FIX] added binaries to match nats-server
  • 9fb4151 Merge pull request #56 from nats-io/add-issue-forms
  • 4647ec0 Fix issue config discussions link
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/nats-io/nkeys&package-manager=go_modules&previous-version=0.4.4&new-version=0.4.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/matrix-org/dendrite/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9e7d278fe..e4febed8a 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/nats-io/jwt/v2 v2.5.0 // indirect - github.com/nats-io/nkeys v0.4.4 // indirect + github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/onsi/ginkgo/v2 v2.11.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/go.sum b/go.sum index e9da63f12..ff44d5a1f 100644 --- a/go.sum +++ b/go.sum @@ -248,8 +248,8 @@ github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c= github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= -github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= -github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= +github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= +github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ= From 5c67eb99b3387ddbcf8646b1d6d3d9f2cf8ad03a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Nov 2023 08:13:34 +0100 Subject: [PATCH 27/40] Bump golang.org/x/image from 0.5.0 to 0.10.0 (#3257) Bumps [golang.org/x/image](https://github.com/golang/image) from 0.5.0 to 0.10.0.
Commits
  • cb227cd tiff: limit work when decoding malicious images
  • a5392f0 bmp: support to decode 8-bit format with up to 256 color palette
  • f9550b0 go.mod: update golang.org/x dependencies
  • 81c166c go.mod: update golang.org/x dependencies
  • ed5dba0 go.mod: update golang.org/x dependencies
  • 08ca817 font: have Glyph return !ok for U+FFFD substitute
  • b6ac75b go.mod: update golang.org/x dependencies
  • 1b74412 font/sfnt: set type for all NameID constants
  • f632f7f tiff, tiff/lzw, vector: use single space in comments
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/image&package-manager=go_modules&previous-version=0.5.0&new-version=0.10.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/matrix-org/dendrite/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e4febed8a..8dad4ebf0 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( go.uber.org/atomic v1.10.0 golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 - golang.org/x/image v0.5.0 + golang.org/x/image v0.10.0 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e golang.org/x/sync v0.3.0 golang.org/x/term v0.13.0 diff --git a/go.sum b/go.sum index ff44d5a1f..b173f6b85 100644 --- a/go.sum +++ b/go.sum @@ -366,8 +366,8 @@ golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N0 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY= @@ -376,6 +376,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -386,6 +387,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -394,6 +396,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -418,10 +421,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -429,6 +434,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -445,6 +451,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 5f872f4a828eef28c6332d3fb7e8368380261f27 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:22:02 +0100 Subject: [PATCH 28/40] Fix panic in `QueryNextRoomHierarchyPage` (#3253) Sentry reported the following panic: ``` time="2023-11-01T01:33:56.220583478Z" level=error msg="Request panicked! goroutine 43763845 [running]: runtime/debug.Stack() runtime/debug/stack.go:24 +0x5e github.com/matrix-org/dendrite/internal/httputil.MakeExternalAPI.MakeJSONAPI.Protect.func3.1() github.com/matrix-org/util@v0.0.0-20221111132719-399730281e66/json.go:98 +0x13e panic({0x15b5540?, 0x2453560?}) runtime/panic.go:914 +0x21f github.com/matrix-org/dendrite/internal/httputil.MakeAuthAPI.func1.1() github.com/matrix-org/dendrite/internal/httputil/httpapi.go:91 +0x4a panic({0x15b5540?, 0x2453560?}) runtime/panic.go:914 +0x21f github.com/matrix-org/dendrite/roomserver/internal/query.(*Queryer).QueryNextRoomHierarchyPage(0x413185?, {0x1a576e0, 0xc0436705a0}, {{{0xc01e5fd260, 0x1f}, {0xc01e5fd261, 0x12}, {0xc01e5fd274, 0xb}}, {0xc145cb5200, ...}, ...}, ...) github.com/matrix-org/dendrite/roomserver/internal/query/query_room_hierarchy.go:116 +0xbfe github.com/matrix-org/dendrite/clientapi/routing.QueryRoomHierarchy(0xc0be13b200, 0xc144e65dd0, {0xc01e5fd260?, 0x6?}, {0x7faf140639c8, 0xc00059af20}, 0xc08adca000?) github.com/matrix-org/dendrite/clientapi/routing/room_hierarchy.go:141 +0x68b github.com/matrix-org/dendrite/clientapi/routing.Setup.func35(0xc03e7d5c20?, 0x17c3a57?) github.com/matrix-org/dendrite/clientapi/routing/routing.go:534 +0xbe github.com/matrix-org/dendrite/internal/httputil.MakeAuthAPI.func1(0xc0bd097300) github.com/matrix-org/dendrite/internal/httputil/httpapi.go:108 +0x5ed github.com/matrix-org/util.(*jsonRequestHandlerWrapper).OnIncomingRequest(0xc0bd097200?, 0xc13b7d6fc0?) github.com/matrix-org/util@v0.0.0-20221111132719-399730281e66/json.go:79 +0x19 github.com/matrix-org/dendrite/internal/httputil.MakeExternalAPI.MakeJSONAPI.func2({0x1a54880, 0xc138f28b60}, 0xc0bd097200?) github.com/matrix-org/util@v0.0.0-20221111132719-399730281e66/json.go:141 +0xaa github.com/matrix-org/dendrite/internal/httputil.MakeExternalAPI.MakeJSONAPI.Protect.func3({0x1a54880?, 0xc138f28b60?}, 0x17c01d9?) github.com/matrix-org/util@v0.0.0-20221111132719-399730281e66/json.go:103 +0x63 net/http.HandlerFunc.ServeHTTP(...) net/http/server.go:2136 github.com/matrix-org/dendrite/internal/httputil.MakeExternalAPI.func1({0x1a54880?, 0xc138f28b60?}, 0xc0bd097100) github.com/matrix-org/dendrite/internal/httputil/httpapi.go:191 +0x411 net/http.HandlerFunc.ServeHTTP(0xc0bd097000?, {0x1a54880?, 0xc138f28b60?}, 0xbe1348905308878e?) net/http/server.go:2136 +0x29 github.com/gorilla/mux.(*Router).ServeHTTP(0xc000000000, {0x1a54880, 0xc138f28b60}, 0xc0bd096f00) github.com/gorilla/mux@v1.8.0/mux.go:210 +0x1c5 github.com/matrix-org/dendrite/setup/base.SetupAndServeHTTP.(*Handler).Handle.(*Handler).handle.func5({0x1a54880, 0xc138f28b60}, 0xc0bd096e00) github.com/getsentry/sentry-go@v0.14.0/http/sentryhttp.go:103 +0x298 net/http.HandlerFunc.ServeHTTP(0xc0bd096a00?, {0x1a54880?, 0xc138f28b60?}, 0x7fae6812f5d0?) net/http/server.go:2136 +0x29 github.com/gorilla/mux.(*Router).ServeHTTP(0xc000000a80, {0x1a54880, 0xc138f28b60}, 0xc0bd096900) github.com/gorilla/mux@v1.8.0/mux.go:210 +0x1c5 net/http.serverHandler.ServeHTTP({0xc02884c4e0?}, {0x1a54880?, 0xc138f28b60?}, 0x6?) net/http/server.go:2938 +0x8e net/http.(*conn).serve(0xc1926922d0, {0x1a576e0, 0xc024a6ec90}) net/http/server.go:2009 +0x5f4 created by net/http.(*Server).Serve in goroutine 16979 net/http/server.go:3086 +0x5cb " context=missing panic="runtime error: invalid memory address or nil pointer dereference" ``` [skip ci] --- roomserver/internal/query/query_room_hierarchy.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index 76eba12be..5f55980f0 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -17,6 +17,7 @@ package query import ( "context" "encoding/json" + "errors" "fmt" "sort" @@ -56,6 +57,12 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r break } + // If the context is canceled, we might still have discovered rooms + // return them to the client and let the client know there _may_ be more rooms. + if errors.Is(ctx.Err(), context.Canceled) { + break + } + // pop the stack queuedRoom := unvisited[len(unvisited)-1] unvisited = unvisited[:len(unvisited)-1] @@ -112,6 +119,11 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r pubRoom := publicRoomsChunk(ctx, querier, queuedRoom.RoomID) + if pubRoom == nil { + util.GetLogger(ctx).WithField("room_id", queuedRoom.RoomID).Debug("unable to get publicRoomsChunk") + continue + } + discoveredRooms = append(discoveredRooms, fclient.RoomHierarchyRoom{ PublicRoom: *pubRoom, RoomType: roomType, From ee73a90aea8349e2695ce555ac868bbf45c11d02 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:22:20 +0100 Subject: [PATCH 29/40] Fix potential connection leak (#3247) We didn't rollback/commit after getting events, now we're rolling back since we didn't change anything. --- syncapi/routing/getevent.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go index c089539f0..d0227f4ea 100644 --- a/syncapi/routing/getevent.go +++ b/syncapi/routing/getevent.go @@ -44,7 +44,7 @@ func GetEvent( rsAPI api.SyncRoomserverAPI, ) util.JSONResponse { ctx := req.Context() - db, err := syncDB.NewDatabaseTransaction(ctx) + db, err := syncDB.NewDatabaseSnapshot(ctx) logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "event_id": eventID, "room_id": rawRoomID, @@ -56,6 +56,7 @@ func GetEvent( JSON: spec.InternalServerError{}, } } + defer db.Rollback() // nolint: errcheck roomID, err := spec.NewRoomID(rawRoomID) if err != nil { From 699f5ca8c1f73ff7e4b70f0f9273ffcb1c195cdc Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 9 Nov 2023 08:42:33 +0100 Subject: [PATCH 30/40] More `rows.Close()` and `rows.Err()` (#3262) Looks like we missed some `rows.Close()` Even though `rows.Err()` is mostly not necessary, we should be more consistent in the DB layer. [skip ci] --- .../notary_server_keys_metadata_table.go | 2 +- .../storage/postgres/queue_json_table.go | 2 +- .../storage/postgres/relay_servers_table.go | 2 +- .../storage/sqlite3/joined_hosts_table.go | 2 +- .../notary_server_keys_metadata_table.go | 2 +- .../storage/sqlite3/queue_json_table.go | 2 +- .../storage/sqlite3/relay_servers_table.go | 2 +- .../storage/postgres/relay_queue_json_table.go | 2 +- .../storage/sqlite3/relay_queue_json_table.go | 2 +- roomserver/storage/postgres/events_table.go | 3 ++- .../storage/postgres/membership_table.go | 2 +- roomserver/storage/postgres/rooms_table.go | 8 ++++---- .../storage/postgres/user_room_keys_table.go | 3 ++- roomserver/storage/sqlite3/event_json_table.go | 2 +- .../storage/sqlite3/event_state_keys_table.go | 4 ++-- .../storage/sqlite3/event_types_table.go | 2 +- roomserver/storage/sqlite3/events_table.go | 18 +++++++++++++++--- roomserver/storage/sqlite3/invite_table.go | 5 ++++- roomserver/storage/sqlite3/membership_table.go | 4 +++- .../storage/sqlite3/room_aliases_table.go | 2 +- roomserver/storage/sqlite3/rooms_table.go | 8 ++++---- .../storage/sqlite3/state_snapshot_table.go | 7 +++++-- .../storage/sqlite3/user_room_keys_table.go | 3 ++- setup/mscs/msc2836/storage.go | 2 +- .../postgres/current_room_state_table.go | 2 +- syncapi/storage/postgres/memberships_table.go | 2 ++ syncapi/storage/postgres/peeks_table.go | 2 +- syncapi/storage/postgres/presence_table.go | 2 +- .../sqlite3/current_room_state_table.go | 6 +++--- syncapi/storage/sqlite3/invites_table.go | 2 +- syncapi/storage/sqlite3/memberships_table.go | 2 ++ .../sqlite3/output_room_events_table.go | 4 ++-- .../output_room_events_topology_table.go | 4 +++- syncapi/storage/sqlite3/peeks_table.go | 2 +- syncapi/storage/sqlite3/presence_table.go | 2 +- .../postgres/cross_signing_keys_table.go | 3 ++- .../postgres/cross_signing_sigs_table.go | 3 ++- userapi/storage/postgres/key_backup_table.go | 2 +- userapi/storage/postgres/key_changes_table.go | 3 ++- .../storage/postgres/one_time_keys_table.go | 2 +- userapi/storage/postgres/profile_table.go | 2 +- userapi/storage/postgres/threepid_table.go | 4 +++- userapi/storage/sqlite3/account_data_table.go | 4 +++- .../sqlite3/cross_signing_keys_table.go | 3 ++- .../sqlite3/cross_signing_sigs_table.go | 3 ++- userapi/storage/sqlite3/devices_table.go | 3 ++- userapi/storage/sqlite3/key_backup_table.go | 2 +- userapi/storage/sqlite3/key_changes_table.go | 3 ++- userapi/storage/sqlite3/one_time_keys_table.go | 2 +- userapi/storage/sqlite3/profile_table.go | 2 +- 50 files changed, 101 insertions(+), 61 deletions(-) diff --git a/federationapi/storage/postgres/notary_server_keys_metadata_table.go b/federationapi/storage/postgres/notary_server_keys_metadata_table.go index 7a1ec4122..47aa82b48 100644 --- a/federationapi/storage/postgres/notary_server_keys_metadata_table.go +++ b/federationapi/storage/postgres/notary_server_keys_metadata_table.go @@ -151,7 +151,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn } results = append(results, sk) } - return results, nil + return results, rows.Err() } func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error { diff --git a/federationapi/storage/postgres/queue_json_table.go b/federationapi/storage/postgres/queue_json_table.go index 563738dd5..f92e33d5e 100644 --- a/federationapi/storage/postgres/queue_json_table.go +++ b/federationapi/storage/postgres/queue_json_table.go @@ -109,5 +109,5 @@ func (s *queueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/federationapi/storage/postgres/relay_servers_table.go b/federationapi/storage/postgres/relay_servers_table.go index 9e1bc5d40..1a47816e2 100644 --- a/federationapi/storage/postgres/relay_servers_table.go +++ b/federationapi/storage/postgres/relay_servers_table.go @@ -110,7 +110,7 @@ func (s *relayServersStatements) SelectRelayServers( } result = append(result, spec.ServerName(relayServer)) } - return result, nil + return result, rows.Err() } func (s *relayServersStatements) DeleteRelayServers( diff --git a/federationapi/storage/sqlite3/joined_hosts_table.go b/federationapi/storage/sqlite3/joined_hosts_table.go index 2412cacdb..418194312 100644 --- a/federationapi/storage/sqlite3/joined_hosts_table.go +++ b/federationapi/storage/sqlite3/joined_hosts_table.go @@ -216,5 +216,5 @@ func joinedHostsFromStmt( }) } - return result, nil + return result, rows.Err() } diff --git a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go index 2fd9ef211..d9b98fc4f 100644 --- a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go @@ -154,7 +154,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn } results = append(results, sk) } - return results, nil + return results, rows.Err() } func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error { diff --git a/federationapi/storage/sqlite3/queue_json_table.go b/federationapi/storage/sqlite3/queue_json_table.go index 0e2806d56..33ae06131 100644 --- a/federationapi/storage/sqlite3/queue_json_table.go +++ b/federationapi/storage/sqlite3/queue_json_table.go @@ -135,5 +135,5 @@ func (s *queueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/federationapi/storage/sqlite3/relay_servers_table.go b/federationapi/storage/sqlite3/relay_servers_table.go index 36cabeb4d..232db32af 100644 --- a/federationapi/storage/sqlite3/relay_servers_table.go +++ b/federationapi/storage/sqlite3/relay_servers_table.go @@ -109,7 +109,7 @@ func (s *relayServersStatements) SelectRelayServers( } result = append(result, spec.ServerName(relayServer)) } - return result, nil + return result, rows.Err() } func (s *relayServersStatements) DeleteRelayServers( diff --git a/relayapi/storage/postgres/relay_queue_json_table.go b/relayapi/storage/postgres/relay_queue_json_table.go index 74410fc88..94ae41407 100644 --- a/relayapi/storage/postgres/relay_queue_json_table.go +++ b/relayapi/storage/postgres/relay_queue_json_table.go @@ -109,5 +109,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/relayapi/storage/sqlite3/relay_queue_json_table.go b/relayapi/storage/sqlite3/relay_queue_json_table.go index 502da3b00..a1af82aa0 100644 --- a/relayapi/storage/sqlite3/relay_queue_json_table.go +++ b/relayapi/storage/sqlite3/relay_queue_json_table.go @@ -133,5 +133,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index a00b4b1d7..1c9cd1599 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -249,6 +249,7 @@ func (s *eventStatements) BulkSelectSnapshotsFromEventIDs( if err != nil { return nil, err } + defer internal.CloseAndLogIfError(ctx, rows, "BulkSelectSnapshotsFromEventIDs: rows.close() failed") var eventID string var stateNID types.StateSnapshotNID @@ -563,7 +564,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( } result[eventNID] = roomNID } - return result, nil + return result, rows.Err() } func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 835a43b2d..1a96e3527 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -363,7 +363,7 @@ func (s *membershipStatements) SelectRoomsWithMembership( } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func (s *membershipStatements) SelectJoinedUsersSetForRooms( diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index c8346733d..bc3820b2c 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -137,7 +137,7 @@ func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.T } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) InsertRoomNID( ctx context.Context, txn *sql.Tx, @@ -255,7 +255,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } result[roomNID] = roomVersion } - return result, nil + return result, rows.Err() } func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) { @@ -277,7 +277,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) { @@ -299,7 +299,7 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func roomNIDsAsArray(roomNIDs []types.RoomNID) pq.Int64Array { diff --git a/roomserver/storage/postgres/user_room_keys_table.go b/roomserver/storage/postgres/user_room_keys_table.go index 217ee957f..57e8f213b 100644 --- a/roomserver/storage/postgres/user_room_keys_table.go +++ b/roomserver/storage/postgres/user_room_keys_table.go @@ -162,6 +162,7 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, if errors.Is(err, sql.ErrNoRows) { return nil, nil } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows") resultMap := make(map[types.RoomNID]ed25519.PublicKey) @@ -173,5 +174,5 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, } resultMap[roomNID] = pubkey } - return resultMap, err + return resultMap, rows.Err() } diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index dc26885bb..325951c7f 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -109,5 +109,5 @@ func (s *eventJSONStatements) BulkSelectEventJSON( } result.EventNID = types.EventNID(eventNID) } - return results[:i], nil + return results[:i], rows.Err() } diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index 347524a81..a052d69ed 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -136,7 +136,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID( } result[stateKey] = types.EventStateKeyNID(stateKeyNID) } - return result, nil + return result, rows.Err() } func (s *eventStateKeyStatements) BulkSelectEventStateKey( @@ -167,5 +167,5 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey( } result[types.EventStateKeyNID(stateKeyNID)] = stateKey } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 0581ec194..c030fffea 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -147,5 +147,5 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID( } result[eventType] = types.EventTypeNID(eventTypeNID) } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index c49c6dc38..2c269bced 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -310,6 +310,9 @@ func (s *eventStatements) BulkSelectStateEventByID( } results = append(results, result) } + if err = rows.Err(); err != nil { + return nil, err + } if !excludeRejected && i != len(eventIDs) { // If there are fewer rows returned than IDs then we were asked to lookup event IDs we don't have. // We don't know which ones were missing because we don't return the string IDs in the query. @@ -377,7 +380,7 @@ func (s *eventStatements) BulkSelectStateEventByNID( return nil, err } } - return results[:i], err + return results[:i], rows.Err() } // bulkSelectStateAtEventByID lookups the state at a list of events by event ID. @@ -425,6 +428,9 @@ func (s *eventStatements) BulkSelectStateAtEventByID( ) } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventIDs) { return nil, types.MissingEventError( fmt.Sprintf("storage: event IDs missing from the database (%d != %d)", i, len(eventIDs)), @@ -507,6 +513,9 @@ func (s *eventStatements) BulkSelectStateAtEventAndReference( result.BeforeStateSnapshotNID = types.StateSnapshotNID(stateSnapshotNID) result.EventID = eventID } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -544,6 +553,9 @@ func (s *eventStatements) BulkSelectEventID(ctx context.Context, txn *sql.Tx, ev } results[types.EventNID(eventNID)] = eventID } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -602,7 +614,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e RoomNID: types.RoomNID(roomNID), } } - return results, nil + return results, rows.Err() } func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { @@ -652,7 +664,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( } result[eventNID] = roomNID } - return result, nil + return result, rows.Err() } func eventNIDsAsArray(eventNIDs []types.EventNID) string { diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index ca6e7c511..b678d8add 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -126,6 +126,9 @@ func (s *inviteStatements) UpdateInviteRetired( } eventIDs = append(eventIDs, inviteEventID) } + if err = rows.Err(); err != nil { + return + } // now retire the invites stmt = sqlutil.TxStmt(txn, s.updateInviteRetiredStmt) _, err = stmt.ExecContext(ctx, roomNID, targetUserNID) @@ -157,5 +160,5 @@ func (s *inviteStatements) SelectInviteActiveForUserInRoom( result = append(result, types.EventStateKeyNID(senderUserNID)) eventIDs = append(eventIDs, eventID) } - return result, eventIDs, eventJSON, nil + return result, eventIDs, eventJSON, rows.Err() } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 977788d50..1012c074a 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -250,6 +250,7 @@ func (s *membershipStatements) SelectMembershipsFromRoom( } eventNIDs = append(eventNIDs, eNID) } + err = rows.Err() return } @@ -277,6 +278,7 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership( } eventNIDs = append(eventNIDs, eNID) } + err = rows.Err() return } @@ -313,7 +315,7 @@ func (s *membershipStatements) SelectRoomsWithMembership( } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) { diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 3bdbbaa35..815b42a27 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -121,7 +121,7 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID( aliases = append(aliases, alias) } - + err = rows.Err() return } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 7556b3461..22700a710 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -128,7 +128,7 @@ func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.T } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) { @@ -265,7 +265,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } result[roomNID] = roomVersion } - return result, nil + return result, rows.Err() } func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) { @@ -293,7 +293,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) { @@ -321,5 +321,5 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 2edff0ba8..dcac0b07c 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -133,13 +133,16 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( var stateBlockNIDsJSON string for ; rows.Next(); i++ { result := &results[i] - if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { + if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { return nil, err } - if err := json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { + if err = json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { return nil, err } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(stateNIDs) { return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))) } diff --git a/roomserver/storage/sqlite3/user_room_keys_table.go b/roomserver/storage/sqlite3/user_room_keys_table.go index 434bad295..13906f771 100644 --- a/roomserver/storage/sqlite3/user_room_keys_table.go +++ b/roomserver/storage/sqlite3/user_room_keys_table.go @@ -177,6 +177,7 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, if errors.Is(err, sql.ErrNoRows) { return nil, nil } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows") resultMap := make(map[types.RoomNID]ed25519.PublicKey) @@ -188,5 +189,5 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, } resultMap[roomNID] = pubkey } - return resultMap, err + return resultMap, rows.Err() } diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go index ade2a1616..696d0b0da 100644 --- a/setup/mscs/msc2836/storage.go +++ b/setup/mscs/msc2836/storage.go @@ -301,7 +301,7 @@ func (p *DB) ChildrenForParent(ctx context.Context, eventID, relType string, rec } children = append(children, evInfo) } - return children, nil + return children, rows.Err() } func (p *DB) ParentForChild(ctx context.Context, eventID, relType string) (*eventInfo, error) { diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index b0148bef5..ec0b27adc 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -392,7 +392,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er }) } - return events, nil + return events, rows.Err() } func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) { diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 4fe4260da..e5208b891 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -160,6 +161,7 @@ func (s *membershipsStatements) SelectMemberships( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows") var ( eventID string ) diff --git a/syncapi/storage/postgres/peeks_table.go b/syncapi/storage/postgres/peeks_table.go index 64183073d..1120dce0b 100644 --- a/syncapi/storage/postgres/peeks_table.go +++ b/syncapi/storage/postgres/peeks_table.go @@ -164,7 +164,7 @@ func (s *peekStatements) SelectPeekingDevices( devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID}) result[roomID] = devices } - return result, nil + return result, rows.Err() } func (s *peekStatements) SelectMaxPeekID( diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go index f37b5331e..53acecce5 100644 --- a/syncapi/storage/postgres/presence_table.go +++ b/syncapi/storage/postgres/presence_table.go @@ -144,7 +144,7 @@ func (p *presenceStatements) GetPresenceForUsers( presence.ClientFields.Presence = presence.Presence.String() result = append(result, presence) } - return result, err + return result, rows.Err() } func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 78b2e397c..f430fcc05 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -177,7 +177,7 @@ func (s *currentRoomStateStatements) SelectJoinedUsers( users = append(users, userID) result[roomID] = users } - return result, nil + return result, rows.Err() } // SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room. @@ -236,7 +236,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( } result = append(result, roomID) } - return result, nil + return result, rows.Err() } // SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user. @@ -419,7 +419,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er }) } - return events, nil + return events, rows.Err() } func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) { diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index ebb469d24..e50b5cbf8 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -176,7 +176,7 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange( if lastPos == 0 { lastPos = r.To } - return result, retired, lastPos, nil + return result, retired, lastPos, rows.Err() } func (s *inviteEventsStatements) SelectMaxInviteID( diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index a1b16306c..9e50422e5 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -163,6 +164,7 @@ func (s *membershipsStatements) SelectMemberships( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows") var eventID string for rows.Next() { if err = rows.Scan(&eventID); err != nil { diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 93caee806..c7b11d3ef 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -274,7 +274,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( } } - return stateNeeded, eventIDToEvent, nil + return stateNeeded, eventIDToEvent, rows.Err() } // MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied, @@ -520,7 +520,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { ExcludeFromSync: excludeFromSync, }) } - return result, nil + return result, rows.Err() } func (s *outputRoomEventsStatements) SelectContextEvent( ctx context.Context, txn *sql.Tx, roomID, eventID string, diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 36967d1e7..c00fb7a79 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -137,6 +138,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( } else if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectEventIDsInRange: failed to close rows") // Return the IDs. var eventID string @@ -155,7 +157,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( start = tokens[0] end = tokens[len(tokens)-1] } - + err = rows.Err() return } diff --git a/syncapi/storage/sqlite3/peeks_table.go b/syncapi/storage/sqlite3/peeks_table.go index 5d5200abc..d8998e2b8 100644 --- a/syncapi/storage/sqlite3/peeks_table.go +++ b/syncapi/storage/sqlite3/peeks_table.go @@ -184,7 +184,7 @@ func (s *peekStatements) SelectPeekingDevices( devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID}) result[roomID] = devices } - return result, nil + return result, rows.Err() } func (s *peekStatements) SelectMaxPeekID( diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go index 573fbad6c..40b57e75d 100644 --- a/syncapi/storage/sqlite3/presence_table.go +++ b/syncapi/storage/sqlite3/presence_table.go @@ -169,7 +169,7 @@ func (p *presenceStatements) GetPresenceForUsers( presence.ClientFields.Presence = presence.Presence.String() result = append(result, presence) } - return result, err + return result, rows.Err() } func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { diff --git a/userapi/storage/postgres/cross_signing_keys_table.go b/userapi/storage/postgres/cross_signing_keys_table.go index 138b629d7..deb355718 100644 --- a/userapi/storage/postgres/cross_signing_keys_table.go +++ b/userapi/storage/postgres/cross_signing_keys_table.go @@ -77,7 +77,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( for rows.Next() { var keyTypeInt int16 var keyData spec.Base64Bytes - if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + if err = rows.Scan(&keyTypeInt, &keyData); err != nil { return nil, err } keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] @@ -86,6 +86,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } r[keyType] = keyData } + err = rows.Err() return } diff --git a/userapi/storage/postgres/cross_signing_sigs_table.go b/userapi/storage/postgres/cross_signing_sigs_table.go index 61a381184..cba015e13 100644 --- a/userapi/storage/postgres/cross_signing_sigs_table.go +++ b/userapi/storage/postgres/cross_signing_sigs_table.go @@ -98,7 +98,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( var userID string var keyID gomatrixserverlib.KeyID var signature spec.Base64Bytes - if err := rows.Scan(&userID, &keyID, &signature); err != nil { + if err = rows.Scan(&userID, &keyID, &signature); err != nil { return nil, err } if _, ok := r[userID]; !ok { @@ -106,6 +106,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( } r[userID][keyID] = signature } + err = rows.Err() return } diff --git a/userapi/storage/postgres/key_backup_table.go b/userapi/storage/postgres/key_backup_table.go index 91a34c357..59944a125 100644 --- a/userapi/storage/postgres/key_backup_table.go +++ b/userapi/storage/postgres/key_backup_table.go @@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api. roomData[key.SessionID] = key.KeyBackupSession result[key.RoomID] = roomData } - return result, nil + return result, rows.Err() } diff --git a/userapi/storage/postgres/key_changes_table.go b/userapi/storage/postgres/key_changes_table.go index a00494140..de3a9e9c8 100644 --- a/userapi/storage/postgres/key_changes_table.go +++ b/userapi/storage/postgres/key_changes_table.go @@ -115,7 +115,7 @@ func (s *keyChangesStatements) SelectKeyChanges( for rows.Next() { var userID string var offset int64 - if err := rows.Scan(&userID, &offset); err != nil { + if err = rows.Scan(&userID, &offset); err != nil { return nil, 0, err } if offset > latestOffset { @@ -123,5 +123,6 @@ func (s *keyChangesStatements) SelectKeyChanges( } userIDs = append(userIDs, userID) } + err = rows.Err() return } diff --git a/userapi/storage/postgres/one_time_keys_table.go b/userapi/storage/postgres/one_time_keys_table.go index 972a59147..a00f4d6f6 100644 --- a/userapi/storage/postgres/one_time_keys_table.go +++ b/userapi/storage/postgres/one_time_keys_table.go @@ -134,7 +134,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de } counts.KeyCount[algorithm] = count } - return counts, nil + return counts, rows.Err() } func (s *oneTimeKeysStatements) InsertOneTimeKeys(ctx context.Context, txn *sql.Tx, keys api.OneTimeKeys) (*api.OneTimeKeysCount, error) { diff --git a/userapi/storage/postgres/profile_table.go b/userapi/storage/postgres/profile_table.go index e404c32f2..e4f55ed94 100644 --- a/userapi/storage/postgres/profile_table.go +++ b/userapi/storage/postgres/profile_table.go @@ -165,5 +165,5 @@ func (s *profilesStatements) SelectProfilesBySearch( profiles = append(profiles, profile) } } - return profiles, nil + return profiles, rows.Err() } diff --git a/userapi/storage/postgres/threepid_table.go b/userapi/storage/postgres/threepid_table.go index 15b42a0a6..fc46061dc 100644 --- a/userapi/storage/postgres/threepid_table.go +++ b/userapi/storage/postgres/threepid_table.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib/spec" @@ -94,6 +95,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectThreePIDsForLocalpart: failed to close rows") threepids = []authtypes.ThreePID{} for rows.Next() { @@ -107,7 +109,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart( Medium: medium, }) } - + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/account_data_table.go b/userapi/storage/sqlite3/account_data_table.go index 3a6367c45..240647b32 100644 --- a/userapi/storage/sqlite3/account_data_table.go +++ b/userapi/storage/sqlite3/account_data_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib/spec" @@ -95,6 +96,7 @@ func (s *accountDataStatements) SelectAccountData( if err != nil { return nil, nil, err } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAccountData: failed to close rows") global := map[string]json.RawMessage{} rooms := map[string]map[string]json.RawMessage{} @@ -118,7 +120,7 @@ func (s *accountDataStatements) SelectAccountData( } } - return global, rooms, nil + return global, rooms, rows.Err() } func (s *accountDataStatements) SelectAccountDataByType( diff --git a/userapi/storage/sqlite3/cross_signing_keys_table.go b/userapi/storage/sqlite3/cross_signing_keys_table.go index 5c2ce7039..65b9ff1af 100644 --- a/userapi/storage/sqlite3/cross_signing_keys_table.go +++ b/userapi/storage/sqlite3/cross_signing_keys_table.go @@ -76,7 +76,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( for rows.Next() { var keyTypeInt int16 var keyData spec.Base64Bytes - if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + if err = rows.Scan(&keyTypeInt, &keyData); err != nil { return nil, err } keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] @@ -85,6 +85,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } r[keyType] = keyData } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/cross_signing_sigs_table.go b/userapi/storage/sqlite3/cross_signing_sigs_table.go index 657264115..bf400a00e 100644 --- a/userapi/storage/sqlite3/cross_signing_sigs_table.go +++ b/userapi/storage/sqlite3/cross_signing_sigs_table.go @@ -96,7 +96,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( var userID string var keyID gomatrixserverlib.KeyID var signature spec.Base64Bytes - if err := rows.Scan(&userID, &keyID, &signature); err != nil { + if err = rows.Scan(&userID, &keyID, &signature); err != nil { return nil, err } if _, ok := r[userID]; !ok { @@ -104,6 +104,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( } r[userID][keyID] = signature } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go index 23e823116..5ce285c87 100644 --- a/userapi/storage/sqlite3/devices_table.go +++ b/userapi/storage/sqlite3/devices_table.go @@ -296,6 +296,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( if err != nil { return devices, err } + defer internal.CloseAndLogIfError(ctx, rows, "SelectDevicesByLocalpart: failed to close rows") var dev api.Device var lastseents sql.NullInt64 @@ -325,7 +326,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( devices = append(devices, dev) } - return devices, nil + return devices, rows.Err() } func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []string) ([]api.Device, error) { diff --git a/userapi/storage/sqlite3/key_backup_table.go b/userapi/storage/sqlite3/key_backup_table.go index ed2746310..1cdaca180 100644 --- a/userapi/storage/sqlite3/key_backup_table.go +++ b/userapi/storage/sqlite3/key_backup_table.go @@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api. roomData[key.SessionID] = key.KeyBackupSession result[key.RoomID] = roomData } - return result, nil + return result, rows.Err() } diff --git a/userapi/storage/sqlite3/key_changes_table.go b/userapi/storage/sqlite3/key_changes_table.go index 923bb57eb..7a4898cfb 100644 --- a/userapi/storage/sqlite3/key_changes_table.go +++ b/userapi/storage/sqlite3/key_changes_table.go @@ -113,7 +113,7 @@ func (s *keyChangesStatements) SelectKeyChanges( for rows.Next() { var userID string var offset int64 - if err := rows.Scan(&userID, &offset); err != nil { + if err = rows.Scan(&userID, &offset); err != nil { return nil, 0, err } if offset > latestOffset { @@ -121,5 +121,6 @@ func (s *keyChangesStatements) SelectKeyChanges( } userIDs = append(userIDs, userID) } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/one_time_keys_table.go b/userapi/storage/sqlite3/one_time_keys_table.go index a992d399c..2a5b1280b 100644 --- a/userapi/storage/sqlite3/one_time_keys_table.go +++ b/userapi/storage/sqlite3/one_time_keys_table.go @@ -140,7 +140,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de } counts.KeyCount[algorithm] = count } - return counts, nil + return counts, rows.Err() } func (s *oneTimeKeysStatements) InsertOneTimeKeys( diff --git a/userapi/storage/sqlite3/profile_table.go b/userapi/storage/sqlite3/profile_table.go index a20d7e848..7285110bc 100644 --- a/userapi/storage/sqlite3/profile_table.go +++ b/userapi/storage/sqlite3/profile_table.go @@ -173,5 +173,5 @@ func (s *profilesStatements) SelectProfilesBySearch( profiles = append(profiles, profile) } } - return profiles, nil + return profiles, rows.Err() } From 7863a405a5f41acd2e40b40ec288eebe781eac1a Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 9 Nov 2023 08:43:27 +0100 Subject: [PATCH 31/40] Use `IsBlacklistedOrBackingOff` to determine if we should try to fetch devices (#3254) Use `IsBlacklistedOrBackingOff` from the federation API to check if we should fetch devices. To reduce back pressure, we now only queue retrying servers if there's space in the channel. --- appservice/appservice_test.go | 11 ++- build/dendritejs-pinecone/main.go | 4 +- build/gobind-yggdrasil/monolith.go | 2 +- clientapi/admin_test.go | 20 ++--- clientapi/clientapi_test.go | 31 ++++--- clientapi/routing/joinroom_test.go | 7 +- clientapi/routing/login_test.go | 2 +- clientapi/routing/register_test.go | 6 +- .../monolith/monolith.go | 2 +- cmd/dendrite-demo-yggdrasil/main.go | 9 ++- cmd/dendrite/main.go | 2 +- federationapi/federationapi.go | 3 +- federationapi/internal/api.go | 4 +- federationapi/routing/profile_test.go | 8 +- federationapi/routing/query_test.go | 8 +- federationapi/routing/send_test.go | 8 +- roomserver/roomserver_test.go | 11 ++- userapi/internal/device_list_update.go | 80 ++++++++++++++----- userapi/internal/device_list_update_test.go | 79 +++++++++++++++++- userapi/userapi.go | 5 +- 20 files changed, 212 insertions(+), 90 deletions(-) diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index bbdeb47df..eca63371d 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/stretchr/testify/assert" "github.com/matrix-org/dendrite/appservice" @@ -32,6 +33,10 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestAppserviceInternalAPI(t *testing.T) { // Set expected results @@ -144,7 +149,7 @@ func TestAppserviceInternalAPI(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) runCases(t, asAPI) @@ -239,7 +244,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) t.Run("UserIDExists", func(t *testing.T) { @@ -378,7 +383,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) { // Create required internal APIs rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // start the consumer appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI) diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go index d3d5f59fe..6acc93c7b 100644 --- a/build/dendritejs-pinecone/main.go +++ b/build/dendritejs-pinecone/main.go @@ -191,13 +191,13 @@ func startup() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics) + fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fedSenderAPI.IsBlacklistedOrBackingOff) asQuery := appservice.NewInternalAPI( processCtx, cfg, &natsInstance, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true) rsAPI.SetFederationAPI(fedSenderAPI, keyRing) monolith := setup.Monolith{ diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 791ad261b..2b227d373 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() { processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index b228dd93c..f0e5f004d 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -45,7 +45,7 @@ func TestAdminCreateToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -196,7 +196,7 @@ func TestAdminListRegistrationTokens(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -314,7 +314,7 @@ func TestAdminGetRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -415,7 +415,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -509,7 +509,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -693,7 +693,7 @@ func TestAdminResetPassword(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for changing the password/login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -791,7 +791,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -863,7 +863,7 @@ func TestAdminEvacuateRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -964,7 +964,7 @@ func TestAdminEvacuateUser(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -1055,7 +1055,7 @@ func TestAdminMarkAsStale(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index 2bb15fba3..2ff4b6503 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -17,6 +17,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/routing" "github.com/matrix-org/dendrite/clientapi/threepid" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/pushrules" @@ -49,6 +50,10 @@ type userDevice struct { password string } +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestGetPutDevices(t *testing.T) { alice := test.NewUser(t) bob := test.NewUser(t) @@ -121,7 +126,7 @@ func TestGetPutDevices(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -170,7 +175,7 @@ func TestDeleteDevice(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -275,7 +280,7 @@ func TestDeleteDevices(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -442,7 +447,7 @@ func TestSetDisplayname(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -554,7 +559,7 @@ func TestSetAvatarURL(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -632,7 +637,7 @@ func TestTyping(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -716,7 +721,7 @@ func TestMembership(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -955,7 +960,7 @@ func TestCapabilities(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1002,7 +1007,7 @@ func TestTurnserver(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) //rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1100,7 +1105,7 @@ func Test3PID(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1276,7 +1281,7 @@ func TestPushRules(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1663,7 +1668,7 @@ func TestKeys(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -2125,7 +2130,7 @@ func TestKeyBackup(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index be3fb429f..bd854efa8 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/jetstream" @@ -21,6 +22,10 @@ import ( uapi "github.com/matrix-org/dendrite/userapi/api" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestJoinRoomByIDOrAlias(t *testing.T) { alice := test.NewUser(t) bob := test.NewUser(t) @@ -36,7 +41,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) // Create the users in the userapi diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index 4c4fc3533..1c628b196 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -49,7 +49,7 @@ func TestLogin(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for /login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics) diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 69b29e9c2..98455f80a 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -416,7 +416,7 @@ func Test_register(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -596,7 +596,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" response := completeRegistration( @@ -637,7 +637,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) expectedDisplayName := "rabbit" jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 5222d0b8e..d9f44b5cc 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -145,7 +145,7 @@ func (p *P2PMonolith) SetupDendrite( ) rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index b07687538..1e5186348 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -213,14 +213,15 @@ func main() { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics) - - asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) + + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) + rsAPI.SetAppserviceAPI(asAPI) + rsAPI.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index f38263c6c..5234b7504 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -162,7 +162,7 @@ func main() { // dependency. Other components also need updating after their dependencies are up. rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index e148199fb..e2524f66a 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/consumers" "github.com/matrix-org/dendrite/federationapi/internal" @@ -102,7 +101,7 @@ func NewInternalAPI( caches *caching.Caches, keyRing *gomatrixserverlib.KeyRing, resetBlacklist bool, -) api.FederationInternalAPI { +) *internal.FederationInternalAPI { cfg := &dendriteCfg.FederationAPI federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName) diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go index 3e6f39566..67388a102 100644 --- a/federationapi/internal/api.go +++ b/federationapi/internal/api.go @@ -112,7 +112,7 @@ func NewFederationInternalAPI( } } -func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) { +func (a *FederationInternalAPI) IsBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) { stats := a.statistics.ForServer(s) if stats.Blacklisted() { return stats, &api.FederationClientError{ @@ -151,7 +151,7 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted( s spec.ServerName, request func() (interface{}, error), ) (interface{}, error) { - stats, err := a.isBlacklistedOrBackingOff(s) + stats, err := a.IsBlacklistedOrBackingOff(s) if err != nil { return nil, err } diff --git a/federationapi/routing/profile_test.go b/federationapi/routing/profile_test.go index a31b206c1..ba13e07fc 100644 --- a/federationapi/routing/profile_test.go +++ b/federationapi/routing/profile_test.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -67,11 +66,8 @@ func TestHandleQueryProfile(t *testing.T) { keyRing := serverKeyAPI.KeyRing() fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/query_test.go b/federationapi/routing/query_test.go index bb14ab031..fd0894d15 100644 --- a/federationapi/routing/query_test.go +++ b/federationapi/routing/query_test.go @@ -25,7 +25,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -65,11 +64,8 @@ func TestHandleQueryDirectory(t *testing.T) { keyRing := serverKeyAPI.KeyRing() fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index f629479da..ff4f7bd06 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -23,7 +23,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -62,11 +61,8 @@ func TestHandleSend(t *testing.T) { fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, nil, nil, nil, true) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 22d27ba05..218a0d8a9 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/httputil" @@ -34,6 +35,10 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + type FakeQuerier struct { api.QuerySenderIDAPI } @@ -58,7 +63,7 @@ func TestUsers(t *testing.T) { }) t.Run("kick users", func(t *testing.T) { - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(usrAPI) testKickUsers(t, rsAPI, usrAPI) }) @@ -258,7 +263,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, fsAPI.IsBlacklistedOrBackingOff) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -1050,7 +1055,7 @@ func TestUpgrade(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(userAPI) for _, tc := range testCases { diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go index a4d28188a..b40635160 100644 --- a/userapi/internal/device_list_update.go +++ b/userapi/internal/device_list_update.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" @@ -108,6 +109,8 @@ type DeviceListUpdater struct { userIDToChan map[string]chan bool userIDToChanMu *sync.Mutex rsAPI rsapi.KeyserverRoomserverAPI + + isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error) } // DeviceListUpdaterDatabase is the subset of functionality from storage.Database required for the updater. @@ -167,25 +170,28 @@ func NewDeviceListUpdater( process *process.ProcessContext, db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer, fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int, - rsAPI rsapi.KeyserverRoomserverAPI, thisServer spec.ServerName, + rsAPI rsapi.KeyserverRoomserverAPI, + thisServer spec.ServerName, enableMetrics bool, + isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error), ) *DeviceListUpdater { if enableMetrics { prometheus.MustRegister(deviceListUpdaterBackpressure, deviceListUpdaterServersRetrying) } return &DeviceListUpdater{ - process: process, - userIDToMutex: make(map[string]*sync.Mutex), - mu: &sync.Mutex{}, - db: db, - api: api, - producer: producer, - fedClient: fedClient, - thisServer: thisServer, - workerChans: make([]chan spec.ServerName, numWorkers), - userIDToChan: make(map[string]chan bool), - userIDToChanMu: &sync.Mutex{}, - rsAPI: rsAPI, + process: process, + userIDToMutex: make(map[string]*sync.Mutex), + mu: &sync.Mutex{}, + db: db, + api: api, + producer: producer, + fedClient: fedClient, + thisServer: thisServer, + workerChans: make([]chan spec.ServerName, numWorkers), + userIDToChan: make(map[string]chan bool), + userIDToChanMu: &sync.Mutex{}, + rsAPI: rsAPI, + isBlacklistedOrBackingOffFn: isBlacklistedOrBackingOffFn, } } @@ -362,13 +368,22 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) { if err != nil { return } + _, err = u.isBlacklistedOrBackingOffFn(remoteServer) + var federationClientError *fedsenderapi.FederationClientError + if errors.As(err, &federationClientError) { + if federationClientError.Blacklisted { + return + } + } + hash := fnv.New32a() _, _ = hash.Write([]byte(remoteServer)) index := int(int64(hash.Sum32()) % int64(len(u.workerChans))) ch := u.assignChannel(userID) + // Since workerChans are buffered, we only increment here and let the worker + // decrement it once it is done processing. deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Inc() - defer deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Dec() u.workerChans[index] <- remoteServer select { case <-ch: @@ -405,24 +420,38 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) { go func() { var serversToRetry []spec.ServerName for { - serversToRetry = serversToRetry[:0] // reuse memory - time.Sleep(time.Second) + // nuke serversToRetry by re-slicing it to be "empty". + // The capacity of the slice is unchanged, which ensures we can reuse the memory. + serversToRetry = serversToRetry[:0] + + deviceListUpdaterServersRetrying.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Set(float64(len(retries))) + time.Sleep(time.Second * 2) + + // -2, so we have space for incoming device list updates over federation + maxServers := (cap(ch) - len(ch)) - 2 + if maxServers <= 0 { + continue + } + retriesMu.Lock() now := time.Now() for srv, retryAt := range retries { if now.After(retryAt) { serversToRetry = append(serversToRetry, srv) + if maxServers == len(serversToRetry) { + break + } } } + for _, srv := range serversToRetry { delete(retries, srv) } - deviceListUpdaterServersRetrying.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Set(float64(len(retries))) retriesMu.Unlock() + for _, srv := range serversToRetry { deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Inc() ch <- srv - deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() } } }() @@ -430,8 +459,18 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) { retriesMu.Lock() _, exists := retries[serverName] retriesMu.Unlock() - if exists { - // Don't retry a server that we're already waiting for. + + // If the serverName is coming from retries, maybe it was + // blacklisted in the meantime. + _, err := u.isBlacklistedOrBackingOffFn(serverName) + var federationClientError *fedsenderapi.FederationClientError + // unwrap errors and check for FederationClientError, if found, federationClientError will be not nil + errors.As(err, &federationClientError) + isBlacklisted := federationClientError != nil && federationClientError.Blacklisted + + // Don't retry a server that we're already waiting for or is blacklisted by now. + if exists || isBlacklisted { + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() continue } waitTime, shouldRetry := u.processServer(serverName) @@ -442,6 +481,7 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) { } retriesMu.Unlock() } + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() } } diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 14a49bc54..a2f1869d1 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -27,6 +27,8 @@ import ( "testing" "time" + api2 "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -129,6 +131,10 @@ type mockDeviceListUpdaterAPI struct { func (d *mockDeviceListUpdaterAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) { } +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + type roundTripper struct { fn func(*http.Request) (*http.Response, error) } @@ -162,7 +168,7 @@ func TestUpdateHavePrevID(t *testing.T) { } ap := &mockDeviceListUpdaterAPI{} producer := &mockKeyChangeProducer{} - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost", caching.DisableMetrics) + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff) event := gomatrixserverlib.DeviceListUpdateEvent{ DeviceDisplayName: "Foo Bar", Deleted: false, @@ -234,7 +240,7 @@ func TestUpdateNoPrevID(t *testing.T) { `)), }, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test", caching.DisableMetrics) + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -304,7 +310,7 @@ func TestDebounce(t *testing.T) { close(incomingFedReq) return <-fedCh, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost", caching.DisableMetrics) + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -407,7 +413,7 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) { updater := NewDeviceListUpdater(processCtx, db, nil, nil, nil, - 0, rsAPI, "test", caching.DisableMetrics) + 0, rsAPI, "test", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.CleanUp(); err != nil { t.Error(err) } @@ -475,3 +481,68 @@ func Test_dedupeStateList(t *testing.T) { }) } } + +func TestDeviceListUpdaterIgnoreBlacklisted(t *testing.T) { + unreachableServer := spec.ServerName("notlocalhost") + + updater := DeviceListUpdater{ + workerChans: make([]chan spec.ServerName, 1), + isBlacklistedOrBackingOffFn: func(s spec.ServerName) (*statistics.ServerStatistics, error) { + switch s { + case unreachableServer: + return nil, &api2.FederationClientError{Blacklisted: true} + } + return nil, nil + }, + mu: &sync.Mutex{}, + userIDToChanMu: &sync.Mutex{}, + userIDToChan: make(map[string]chan bool), + userIDToMutex: make(map[string]*sync.Mutex), + } + workerCh := make(chan spec.ServerName) + defer close(workerCh) + updater.workerChans[0] = workerCh + + // happy case + alice := "@alice:localhost" + aliceCh := updater.assignChannel(alice) + defer updater.clearChannel(alice) + + // failing case + bob := "@bob:" + unreachableServer + bobCh := updater.assignChannel(string(bob)) + defer updater.clearChannel(string(bob)) + + expectedServers := map[spec.ServerName]struct{}{ + "localhost": {}, + } + unexpectedServers := make(map[spec.ServerName]struct{}) + + go func() { + for serverName := range workerCh { + switch serverName { + case "localhost": + delete(expectedServers, serverName) + aliceCh <- true // unblock notifyWorkers + case unreachableServer: // this should not happen as it is "filtered" away by the blacklist + unexpectedServers[serverName] = struct{}{} + bobCh <- true + default: + unexpectedServers[serverName] = struct{}{} + } + } + }() + + // alice is not blacklisted + updater.notifyWorkers(alice) + // bob is blacklisted + updater.notifyWorkers(string(bob)) + + for server := range expectedServers { + t.Errorf("Server still in expectedServers map: %s", server) + } + + for server := range unexpectedServers { + t.Errorf("unexpected server in result: %s", server) + } +} diff --git a/userapi/userapi.go b/userapi/userapi.go index 34bf119a0..a1c9b94a9 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -18,10 +18,12 @@ import ( "time" fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/sirupsen/logrus" rsapi "github.com/matrix-org/dendrite/roomserver/api" @@ -47,6 +49,7 @@ func NewInternalAPI( rsAPI rsapi.UserRoomserverAPI, fedClient fedsenderapi.KeyserverFederationAPI, enableMetrics bool, + blacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error), ) *internal.UserInternalAPI { js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream) appServices := dendriteCfg.Derived.ApplicationServices @@ -100,7 +103,7 @@ func NewInternalAPI( FedClient: fedClient, } - updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, dendriteCfg.UserAPI.WorkerCount, rsAPI, dendriteCfg.Global.ServerName, enableMetrics) + updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, dendriteCfg.UserAPI.WorkerCount, rsAPI, dendriteCfg.Global.ServerName, enableMetrics, blacklistedOrBackingOffFn) userAPI.Updater = updater // Remove users which we don't share a room with anymore if err := updater.CleanUp(); err != nil { From fde4225469c81d4efafbc8e4424eee0585beb9bd Mon Sep 17 00:00:00 2001 From: Nikolai Patrick <41409702+Smartich0ke@users.noreply.github.com> Date: Wed, 22 Nov 2023 21:43:41 +1030 Subject: [PATCH 32/40] fix typo (#3266) Fix a tiny spelling mistake in the Grafana dashboard. Literally a 1 character commit lol ### Pull Request Checklist * [x ] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [ x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Nikolai Patrick nikolaipatrick@wws.sa.edu.au` --- helm/dendrite/grafana_dashboards/dendrite-rev2.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/dendrite/grafana_dashboards/dendrite-rev2.json b/helm/dendrite/grafana_dashboards/dendrite-rev2.json index 817f950b3..420d8bf1b 100644 --- a/helm/dendrite/grafana_dashboards/dendrite-rev2.json +++ b/helm/dendrite/grafana_dashboards/dendrite-rev2.json @@ -119,7 +119,7 @@ "refId": "A" } ], - "title": "Registerd Users", + "title": "Registered Users", "type": "stat" }, { From 06e079abacb9c2719f0b73ab193be1e7818c43f6 Mon Sep 17 00:00:00 2001 From: notassigned <22502000+notassigned@users.noreply.github.com> Date: Wed, 22 Nov 2023 06:14:49 -0500 Subject: [PATCH 33/40] Fix broken links in FAQ.md (#3259) The links to CONTRUBITING.md and 4_adminapi.md were broken. ### Pull Request Checklist * [ X] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [X ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: Co-authored-by: kegsay --- docs/FAQ.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 570ba677e..82b1581ea 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -24,7 +24,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting -the development efforts through [contributing](../development/contributing). +the development efforts through [contributing](./development/CONTRIBUTING.md). ## Is there a migration path from Synapse to Dendrite? @@ -105,7 +105,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom Date: Wed, 22 Nov 2023 13:05:24 +0100 Subject: [PATCH 34/40] Add `keydb_server_keys` table tests (#3270) Also moves some of the variable declarations out of the loop to, hopefully, reduce allocations. --- .../storage/postgres/server_key_table.go | 13 +- .../storage/sqlite3/server_key_table.go | 12 +- .../storage/tables/server_key_table_test.go | 116 ++++++++++++++++++ 3 files changed, 129 insertions(+), 12 deletions(-) create mode 100644 federationapi/storage/tables/server_key_table_test.go diff --git a/federationapi/storage/postgres/server_key_table.go b/federationapi/storage/postgres/server_key_table.go index c62446da5..fa58f1ea2 100644 --- a/federationapi/storage/postgres/server_key_table.go +++ b/federationapi/storage/postgres/server_key_table.go @@ -94,12 +94,14 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( } defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + + var serverName string + var keyID string + var key string + var validUntilTS int64 + var expiredTS int64 + var vk gomatrixserverlib.VerifyKey for rows.Next() { - var serverName string - var keyID string - var key string - var validUntilTS int64 - var expiredTS int64 if err = rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil { return nil, err } @@ -107,7 +109,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( ServerName: spec.ServerName(serverName), KeyID: gomatrixserverlib.KeyID(keyID), } - vk := gomatrixserverlib.VerifyKey{} err = vk.Key.Decode(key) if err != nil { return nil, err diff --git a/federationapi/storage/sqlite3/server_key_table.go b/federationapi/storage/sqlite3/server_key_table.go index f28b89940..65a854ce1 100644 --- a/federationapi/storage/sqlite3/server_key_table.go +++ b/federationapi/storage/sqlite3/server_key_table.go @@ -98,12 +98,13 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( err := sqlutil.RunLimitedVariablesQuery( ctx, bulkSelectServerSigningKeysSQL, s.db, iKeyIDs, sqlutil.SQLite3MaxVariables, func(rows *sql.Rows) error { + var serverName string + var keyID string + var key string + var validUntilTS int64 + var expiredTS int64 + var vk gomatrixserverlib.VerifyKey for rows.Next() { - var serverName string - var keyID string - var key string - var validUntilTS int64 - var expiredTS int64 if err := rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil { return fmt.Errorf("bulkSelectServerKeys: %v", err) } @@ -111,7 +112,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( ServerName: spec.ServerName(serverName), KeyID: gomatrixserverlib.KeyID(keyID), } - vk := gomatrixserverlib.VerifyKey{} err := vk.Key.Decode(key) if err != nil { return fmt.Errorf("bulkSelectServerKeys: %v", err) diff --git a/federationapi/storage/tables/server_key_table_test.go b/federationapi/storage/tables/server_key_table_test.go new file mode 100644 index 000000000..e79a086b8 --- /dev/null +++ b/federationapi/storage/tables/server_key_table_test.go @@ -0,0 +1,116 @@ +package tables_test + +import ( + "context" + "testing" + "time" + + "github.com/matrix-org/dendrite/federationapi/storage/postgres" + "github.com/matrix-org/dendrite/federationapi/storage/sqlite3" + "github.com/matrix-org/dendrite/federationapi/storage/tables" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/stretchr/testify/assert" +) + +func mustCreateServerKeyDB(t *testing.T, dbType test.DBType) (tables.FederationServerSigningKeys, func()) { + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + if err != nil { + t.Fatalf("failed to open database: %s", err) + } + var tab tables.FederationServerSigningKeys + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresServerSigningKeysTable(db) + case test.DBTypeSQLite: + tab, err = sqlite3.NewSQLiteServerSigningKeysTable(db) + } + if err != nil { + t.Fatalf("failed to create table: %s", err) + } + return tab, close +} + +func TestServerKeysTable(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + ctx, cancel := context.WithCancel(context.Background()) + tab, close := mustCreateServerKeyDB(t, dbType) + t.Cleanup(func() { + close() + cancel() + }) + + req := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: "localhost", + KeyID: "ed25519:test", + } + expectedTimestamp := spec.AsTimestamp(time.Now().Add(time.Hour)) + res := gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)}, + ExpiredTS: 0, + ValidUntilTS: expectedTimestamp, + } + + // Insert the key + err := tab.UpsertServerKeys(ctx, nil, req, res) + assert.NoError(t, err) + + selectKeys := map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp{ + req: spec.AsTimestamp(time.Now()), + } + gotKeys, err := tab.BulkSelectServerKeys(ctx, nil, selectKeys) + assert.NoError(t, err) + + // Now we should have a key for the req above + assert.NotNil(t, gotKeys[req]) + assert.Equal(t, res, gotKeys[req]) + + // "Expire" the key by setting ExpireTS to a non-zero value and ValidUntilTS to 0 + expectedTimestamp = spec.AsTimestamp(time.Now()) + res.ExpiredTS = expectedTimestamp + res.ValidUntilTS = 0 + + // Update the key + err = tab.UpsertServerKeys(ctx, nil, req, res) + assert.NoError(t, err) + + gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys) + assert.NoError(t, err) + + // The key should be expired + assert.NotNil(t, gotKeys[req]) + assert.Equal(t, res, gotKeys[req]) + + // Upsert a different key to validate querying multiple keys + req2 := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: "notlocalhost", + KeyID: "ed25519:test2", + } + expectedTimestamp2 := spec.AsTimestamp(time.Now().Add(time.Hour)) + res2 := gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)}, + ExpiredTS: 0, + ValidUntilTS: expectedTimestamp2, + } + + err = tab.UpsertServerKeys(ctx, nil, req2, res2) + assert.NoError(t, err) + + // Select multiple keys + selectKeys[req2] = spec.AsTimestamp(time.Now()) + + gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys) + assert.NoError(t, err) + + // We now should receive two keys, one of which is expired + assert.Equal(t, 2, len(gotKeys)) + assert.Equal(t, res2, gotKeys[req2]) + assert.Equal(t, res, gotKeys[req]) + }) +} From f25cce237e14af6041229c8248ded30d37b8cb51 Mon Sep 17 00:00:00 2001 From: CicadaCinema <52425971+CicadaCinema@users.noreply.github.com> Date: Wed, 22 Nov 2023 12:15:16 +0000 Subject: [PATCH 35/40] Refactor registration tests, remove hard-coded username validation (#3138) ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [x] I have already signed off privately This PR is in preparation for #3137 and removes the hard-coded username validation (previously only dependent on `forceEmpty`). --------- Co-authored-by: kegsay <7190048+kegsay@users.noreply.github.com> --- clientapi/routing/register_test.go | 122 +++++++++++++++-------------- 1 file changed, 63 insertions(+), 59 deletions(-) diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 98455f80a..7fa740e7f 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -298,25 +298,29 @@ func Test_register(t *testing.T) { guestsDisabled bool enableRecaptcha bool captchaBody string - wantResponse util.JSONResponse + // in case of an error, the expected response + wantErrorResponse util.JSONResponse + // in case of success, the expected username assigned + wantUsername string }{ { name: "disallow guests", kind: "guest", guestsDisabled: true, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden(`Guest registration is disabled on "test"`), }, }, { - name: "allow guests", - kind: "guest", + name: "allow guests", + kind: "guest", + wantUsername: "1", }, { name: "unknown login type", loginType: "im.not.known", - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusNotImplemented, JSON: spec.Unknown("unknown/unimplemented auth type"), }, @@ -324,25 +328,33 @@ func Test_register(t *testing.T) { { name: "disabled registration", registrationDisabled: true, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden(`Registration is disabled on "test"`), }, }, { - name: "successful registration, numeric ID", - username: "", - password: "someRandomPassword", - forceEmpty: true, + name: "successful registration, numeric ID", + username: "", + password: "someRandomPassword", + forceEmpty: true, + wantUsername: "2", }, { name: "successful registration", username: "success", }, + { + name: "successful registration, sequential numeric ID", + username: "", + password: "someRandomPassword", + forceEmpty: true, + wantUsername: "3", + }, { name: "failing registration - user already exists", username: "success", - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.UserInUse("Desired user ID is already taken."), }, @@ -352,14 +364,14 @@ func Test_register(t *testing.T) { username: "LOWERCASED", // this is going to be lower-cased }, { - name: "invalid username", - username: "#totalyNotValid", - wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid), + name: "invalid username", + username: "#totalyNotValid", + wantErrorResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid), }, { name: "numeric username is forbidden", username: "1337", - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.InvalidUsername("Numeric user IDs are reserved"), }, @@ -367,7 +379,7 @@ func Test_register(t *testing.T) { { name: "disabled recaptcha login", loginType: authtypes.LoginTypeRecaptcha, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Unknown(ErrCaptchaDisabled.Error()), }, @@ -376,7 +388,7 @@ func Test_register(t *testing.T) { name: "enabled recaptcha, no response defined", enableRecaptcha: true, loginType: authtypes.LoginTypeRecaptcha, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.BadJSON(ErrMissingResponse.Error()), }, @@ -386,7 +398,7 @@ func Test_register(t *testing.T) { enableRecaptcha: true, loginType: authtypes.LoginTypeRecaptcha, captchaBody: `notvalid`, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusUnauthorized, JSON: spec.BadJSON(ErrInvalidCaptcha.Error()), }, @@ -398,11 +410,11 @@ func Test_register(t *testing.T) { captchaBody: `success`, }, { - name: "captcha invalid from remote", - enableRecaptcha: true, - loginType: authtypes.LoginTypeRecaptcha, - captchaBody: `i should fail for other reasons`, - wantResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}}, + name: "captcha invalid from remote", + enableRecaptcha: true, + loginType: authtypes.LoginTypeRecaptcha, + captchaBody: `i should fail for other reasons`, + wantErrorResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}}, }, } @@ -486,8 +498,8 @@ func Test_register(t *testing.T) { t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows) } case spec.MatrixError: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse) + if !reflect.DeepEqual(tc.wantErrorResponse, resp) { + t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantErrorResponse) } return case registerResponse: @@ -505,6 +517,13 @@ func Test_register(t *testing.T) { if r.DeviceID == "" { t.Fatalf("missing deviceID in response") } + // if an expected username is provided, assert that it is a match + if tc.wantUsername != "" { + wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test")) + if wantUserID != r.UserID { + t.Fatalf("unexpected userID: %s, want %s", r.UserID, wantUserID) + } + } return default: t.Logf("Got response: %T", resp.JSON) @@ -541,44 +560,29 @@ func Test_register(t *testing.T) { resp = Register(req, userAPI, &cfg.ClientAPI) - switch resp.JSON.(type) { - case spec.InternalServerError: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse) + switch rr := resp.JSON.(type) { + case spec.InternalServerError, spec.MatrixError, util.JSONResponse: + if !reflect.DeepEqual(tc.wantErrorResponse, resp) { + t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantErrorResponse) } return - case spec.MatrixError: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse) + case registerResponse: + // validate the response + if tc.wantUsername != "" { + // if an expected username is provided, assert that it is a match + wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test")) + if wantUserID != rr.UserID { + t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID) + } } - return - case util.JSONResponse: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse) + if rr.DeviceID != *reg.DeviceID { + t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID) } - return - } - - rr, ok := resp.JSON.(registerResponse) - if !ok { - t.Fatalf("expected a registerresponse, got %T", resp.JSON) - } - - // validate the response - if tc.forceEmpty { - // when not supplying a username, one will be generated. Given this _SHOULD_ be - // the second user, set the username accordingly - reg.Username = "2" - } - wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test")) - if wantUserID != rr.UserID { - t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID) - } - if rr.DeviceID != *reg.DeviceID { - t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID) - } - if rr.AccessToken == "" { - t.Fatalf("missing accessToken in response") + if rr.AccessToken == "" { + t.Fatalf("missing accessToken in response") + } + default: + t.Fatalf("expected one of internalservererror, matrixerror, jsonresponse, registerresponse, got %T", resp.JSON) } }) } From c4528b2de8c36657039c3d3f541017ee8964c4ac Mon Sep 17 00:00:00 2001 From: BtbN Date: Wed, 22 Nov 2023 13:15:45 +0100 Subject: [PATCH 36/40] Allow users to kick themselves (#3157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per the spec: https://spec.matrix.org/v1.7/rooms/v10/#authorization-rules "If membership is leave" -> "If the sender matches state_key, allow if and only if that user’s current membership state is invite, join, or knock." I.e. a user can kick themselves. Bridges use this to make a user leave while giving a reason. Some recent change (likely https://github.com/matrix-org/dendrite/commit/8ea1a11105ea7e66aa459537bcbef0de606147cd but I'm not 100% sure) changed that behaviour, resulting in heisenbridge being unable to make users leave while giving a reason. This works fine on Synapse. Signed-off-by: Timo Rothenpieler Co-authored-by: kegsay <7190048+kegsay@users.noreply.github.com> --- clientapi/routing/membership.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 8b8cc47bc..06683c47d 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -181,18 +181,6 @@ func SendKick( return *errRes } - pl, errRes := getPowerlevels(req, rsAPI, roomID) - if errRes != nil { - return *errRes - } - allowedToKick := pl.UserLevel(*senderID) >= pl.Kick - if !allowedToKick { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden("You don't have permission to kick this user, power level too low."), - } - } - bodyUserID, err := spec.NewUserID(body.UserID, true) if err != nil { return util.JSONResponse{ @@ -200,6 +188,19 @@ func SendKick( JSON: spec.BadJSON("body userID is invalid"), } } + + pl, errRes := getPowerlevels(req, rsAPI, roomID) + if errRes != nil { + return *errRes + } + allowedToKick := pl.UserLevel(*senderID) >= pl.Kick || bodyUserID.String() == deviceUserID.String() + if !allowedToKick { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to kick this user, power level too low."), + } + } + var queryRes roomserverAPI.QueryMembershipForUserResponse err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, From b8f91485b47ac6e92a90988b394e8f3611735250 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 22 Nov 2023 15:38:04 +0100 Subject: [PATCH 37/40] Update ACLs when received as outliers (#3008) This should fix #3004 by making sure we also update our in-memory ACLs after joining a new room. Also makes use of more caching in `GetStateEvent` Bonus: Adds some tests, as I was about to use `GetBulkStateContent`, but turns out that `GetStateEvent` is basically doing the same, just that it only gets the `eventTypeNID`/`eventStateKeyNID` once and not for every call. --- federationapi/federationapi.go | 2 +- .../internal/federationclient_test.go | 10 +-- federationapi/internal/perform_test.go | 10 +-- federationapi/queue/destinationqueue.go | 2 - federationapi/queue/queue.go | 36 --------- federationapi/queue/queue_test.go | 13 +--- roomserver/acls/acls.go | 4 +- roomserver/internal/input/input_events.go | 22 ++++++ roomserver/producers/roomevent.go | 2 +- roomserver/roomserver_test.go | 41 ++++++++++ roomserver/storage/tables/interface_test.go | 76 +++++++++++++++++++ 11 files changed, 155 insertions(+), 63 deletions(-) create mode 100644 roomserver/storage/tables/interface_test.go diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index e2524f66a..efbfa3315 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -125,7 +125,7 @@ func NewInternalAPI( queues := queue.NewOutgoingQueues( federationDB, processContext, cfg.Matrix.DisableFederation, - cfg.Matrix.ServerName, federation, rsAPI, &stats, + cfg.Matrix.ServerName, federation, &stats, signingInfo, ) diff --git a/federationapi/internal/federationclient_test.go b/federationapi/internal/federationclient_test.go index 8c562dd61..fe8d84ffb 100644 --- a/federationapi/internal/federationclient_test.go +++ b/federationapi/internal/federationclient_test.go @@ -65,7 +65,7 @@ func TestFederationClientQueryKeys(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -96,7 +96,7 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -126,7 +126,7 @@ func TestFederationClientQueryKeysFailure(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -156,7 +156,7 @@ func TestFederationClientClaimKeys(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -187,7 +187,7 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ diff --git a/federationapi/internal/perform_test.go b/federationapi/internal/perform_test.go index 656755f96..2795a018a 100644 --- a/federationapi/internal/perform_test.go +++ b/federationapi/internal/perform_test.go @@ -70,7 +70,7 @@ func TestPerformWakeupServers(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -116,7 +116,7 @@ func TestQueryRelayServers(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -157,7 +157,7 @@ func TestRemoveRelayServers(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -197,7 +197,7 @@ func TestPerformDirectoryLookup(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -236,7 +236,7 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 880aee0d3..f51e849fa 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -31,7 +31,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/storage/shared/receipt" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/process" ) @@ -53,7 +52,6 @@ type destinationQueue struct { db storage.Database process *process.ProcessContext signing map[spec.ServerName]*fclient.SigningIdentity - rsAPI api.FederationRoomserverAPI client fclient.FederationClient // federation client origin spec.ServerName // origin of requests destination spec.ServerName // destination of requests diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 24b3efd2d..892c26a2c 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -27,12 +27,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" - "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/storage/shared/receipt" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/process" ) @@ -43,7 +41,6 @@ type OutgoingQueues struct { db storage.Database process *process.ProcessContext disabled bool - rsAPI api.FederationRoomserverAPI origin spec.ServerName client fclient.FederationClient statistics *statistics.Statistics @@ -90,7 +87,6 @@ func NewOutgoingQueues( disabled bool, origin spec.ServerName, client fclient.FederationClient, - rsAPI api.FederationRoomserverAPI, statistics *statistics.Statistics, signing []*fclient.SigningIdentity, ) *OutgoingQueues { @@ -98,7 +94,6 @@ func NewOutgoingQueues( disabled: disabled, process: process, db: db, - rsAPI: rsAPI, origin: origin, client: client, statistics: statistics, @@ -162,7 +157,6 @@ func (oqs *OutgoingQueues) getQueue(destination spec.ServerName) *destinationQue queues: oqs, db: oqs.db, process: oqs.process, - rsAPI: oqs.rsAPI, origin: oqs.origin, destination: destination, client: oqs.client, @@ -213,18 +207,6 @@ func (oqs *OutgoingQueues) SendEvent( delete(destmap, local) } - // Check if any of the destinations are prohibited by server ACLs. - for destination := range destmap { - if api.IsServerBannedFromRoom( - oqs.process.Context(), - oqs.rsAPI, - ev.RoomID().String(), - destination, - ) { - delete(destmap, destination) - } - } - // If there are no remaining destinations then give up. if len(destmap) == 0 { return nil @@ -303,24 +285,6 @@ func (oqs *OutgoingQueues) SendEDU( delete(destmap, local) } - // There is absolutely no guarantee that the EDU will have a room_id - // field, as it is not required by the spec. However, if it *does* - // (e.g. typing notifications) then we should try to make sure we don't - // bother sending them to servers that are prohibited by the server - // ACLs. - if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() { - for destination := range destmap { - if api.IsServerBannedFromRoom( - oqs.process.Context(), - oqs.rsAPI, - result.Str, - destination, - ) { - delete(destmap, destination) - } - } - } - // If there are no remaining destinations then give up. if len(destmap) == 0 { return nil diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index e75615e05..73d3b0598 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -34,7 +34,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" - rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" @@ -65,15 +64,6 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase } } -type stubFederationRoomServerAPI struct { - rsapi.FederationRoomserverAPI -} - -func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error { - res.Banned = false - return nil -} - type stubFederationClient struct { fclient.FederationClient shouldTxSucceed bool @@ -126,7 +116,6 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32 txCount: *atomic.NewUint32(0), txRelayCount: *atomic.NewUint32(0), } - rs := &stubFederationRoomServerAPI{} stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline) signingInfo := []*fclient.SigningIdentity{ @@ -136,7 +125,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32 ServerName: "localhost", }, } - queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, rs, &stats, signingInfo) + queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, &stats, signingInfo) return db, fc, queues, processContext, close } diff --git a/roomserver/acls/acls.go b/roomserver/acls/acls.go index 601ce9063..e247c7553 100644 --- a/roomserver/acls/acls.go +++ b/roomserver/acls/acls.go @@ -29,6 +29,8 @@ import ( "github.com/sirupsen/logrus" ) +const MRoomServerACL = "m.room.server_acl" + type ServerACLDatabase interface { // GetKnownRooms returns a list of all rooms we know about. GetKnownRooms(ctx context.Context) ([]string, error) @@ -57,7 +59,7 @@ func NewServerACLs(db ServerACLDatabase) *ServerACLs { // do then we'll process it into memory so that we have the regexes to // hand. for _, room := range rooms { - state, err := db.GetStateEvent(ctx, room, "m.room.server_acl", "") + state, err := db.GetStateEvent(ctx, room, MRoomServerACL, "") if err != nil { logrus.WithError(err).Errorf("Failed to get server ACLs for room %q", room) continue diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 77b50d0e2..520f82a80 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -33,6 +33,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/internal/helpers" userAPI "github.com/matrix-org/dendrite/userapi/api" @@ -491,6 +492,27 @@ func (r *Inputer) processRoomEvent( } } + // If this is a membership event, it is possible we newly joined a federated room and eventually + // missed to update our m.room.server_acl - the following ensures we set the ACLs + // TODO: This probably performs badly in benchmarks + if event.Type() == spec.MRoomMember { + membership, _ := event.Membership() + if membership == spec.Join { + _, serverName, _ := gomatrixserverlib.SplitID('@', *event.StateKey()) + // only handle local membership events + if r.Cfg.Matrix.IsLocalServerName(serverName) { + var aclEvent *types.HeaderedEvent + aclEvent, err = r.DB.GetStateEvent(ctx, event.RoomID().String(), acls.MRoomServerACL, "") + if err != nil { + logrus.WithError(err).Error("failed to get server ACLs") + } + if aclEvent != nil { + r.ACLs.OnServerACLUpdate(aclEvent) + } + } + } + } + // Handle remote room upgrades, e.g. remove published room if event.Type() == "m.room.tombstone" && event.StateKeyEquals("") && !r.Cfg.Matrix.IsLocalServerName(senderDomain) { if err = r.handleRemoteRoomUpgrade(ctx, event); err != nil { diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go index 165304d49..af7e10580 100644 --- a/roomserver/producers/roomevent.go +++ b/roomserver/producers/roomevent.go @@ -73,7 +73,7 @@ func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.Outpu } } - if eventType == "m.room.server_acl" && update.NewRoomEvent.Event.StateKeyEquals("") { + if eventType == acls.MRoomServerACL && update.NewRoomEvent.Event.StateKeyEquals("") { ev := update.NewRoomEvent.Event.PDU defer r.ACLs.OnServerACLUpdate(ev) } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 218a0d8a9..e9cd926d7 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" + "github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/userapi" @@ -1190,3 +1191,43 @@ func TestStateReset(t *testing.T) { } }) } + +func TestNewServerACLs(t *testing.T) { + alice := test.NewUser(t) + roomWithACL := test.NewRoom(t, alice) + + roomWithACL.CreateAndInsert(t, alice, acls.MRoomServerACL, acls.ServerACL{ + Allowed: []string{"*"}, + Denied: []string{"localhost"}, + AllowIPLiterals: false, + }, test.WithStateKey("")) + + roomWithoutACL := test.NewRoom(t, alice) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + // start JetStream listeners + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + // let the RS create the events + err := api.SendEvents(context.Background(), rsAPI, api.KindNew, roomWithACL.Events(), "test", "test", "test", nil, false) + assert.NoError(t, err) + err = api.SendEvents(context.Background(), rsAPI, api.KindNew, roomWithoutACL.Events(), "test", "test", "test", nil, false) + assert.NoError(t, err) + + db, err := storage.Open(processCtx.Context(), cm, &cfg.RoomServer.Database, caches) + assert.NoError(t, err) + // create new server ACLs and verify server is banned/not banned + serverACLs := acls.NewServerACLs(db) + banned := serverACLs.IsServerBannedFromRoom("localhost", roomWithACL.ID) + assert.Equal(t, true, banned) + banned = serverACLs.IsServerBannedFromRoom("localhost", roomWithoutACL.ID) + assert.Equal(t, false, banned) + }) +} diff --git a/roomserver/storage/tables/interface_test.go b/roomserver/storage/tables/interface_test.go new file mode 100644 index 000000000..8727e2436 --- /dev/null +++ b/roomserver/storage/tables/interface_test.go @@ -0,0 +1,76 @@ +package tables + +import ( + "testing" + + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/stretchr/testify/assert" +) + +func TestExtractContentValue(t *testing.T) { + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + tests := []struct { + name string + event *types.HeaderedEvent + want string + }{ + { + name: "returns creator ID for create events", + event: room.Events()[0], + want: alice.ID, + }, + { + name: "returns the alias for canonical alias events", + event: room.CreateEvent(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": "#test:test"}), + want: "#test:test", + }, + { + name: "returns the history_visibility for history visibility events", + event: room.CreateEvent(t, alice, spec.MRoomHistoryVisibility, map[string]string{"history_visibility": "shared"}), + want: "shared", + }, + { + name: "returns the join rules for join_rules events", + event: room.CreateEvent(t, alice, spec.MRoomJoinRules, map[string]string{"join_rule": "public"}), + want: "public", + }, + { + name: "returns the membership for room_member events", + event: room.CreateEvent(t, alice, spec.MRoomMember, map[string]string{"membership": "join"}, test.WithStateKey(alice.ID)), + want: "join", + }, + { + name: "returns the room name for room_name events", + event: room.CreateEvent(t, alice, spec.MRoomName, map[string]string{"name": "testing"}, test.WithStateKey(alice.ID)), + want: "testing", + }, + { + name: "returns the room avatar for avatar events", + event: room.CreateEvent(t, alice, spec.MRoomAvatar, map[string]string{"url": "mxc://testing"}, test.WithStateKey(alice.ID)), + want: "mxc://testing", + }, + { + name: "returns the room topic for topic events", + event: room.CreateEvent(t, alice, spec.MRoomTopic, map[string]string{"topic": "testing"}, test.WithStateKey(alice.ID)), + want: "testing", + }, + { + name: "returns guest_access for guest access events", + event: room.CreateEvent(t, alice, "m.room.guest_access", map[string]string{"guest_access": "forbidden"}, test.WithStateKey(alice.ID)), + want: "forbidden", + }, + { + name: "returns empty string if key can't be found or unknown event", + event: room.CreateEvent(t, alice, "idontexist", nil), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, ExtractContentValue(tt.event), "ExtractContentValue(%v)", tt.event) + }) + } +} From 4f943771fa9c26f5f2a0d17f5c477a3c9b704b6e Mon Sep 17 00:00:00 2001 From: KuhnChris Date: Fri, 24 Nov 2023 22:34:13 +0100 Subject: [PATCH 38/40] Appservice Login (2nd attempt) (#3078) Rebase of #2936 as @vijfhoek wrote he got no time to work on this, and I kind of needed it for my experiments. I checked the tests, and it is working with my example code (i.e. impersonating, registering, creating channel, invite people, write messages). I'm not a huge `go` pro, and still learning, but I tried to fix and/or integrate the changes as best as possible with the current `main` branch changes. If there is anything left, let me know and I'll try to figure it out. Signed-off-by: `Kuhn Christopher ` --------- Signed-off-by: Sijmen Signed-off-by: Sijmen Schoon Co-authored-by: Sijmen Schoon Co-authored-by: Sijmen Schoon Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com> --- clientapi/auth/login.go | 26 +++- clientapi/auth/login_application_service.go | 55 ++++++++ clientapi/auth/login_test.go | 129 +++++++++++++++++-- clientapi/auth/user_interactive.go | 2 +- clientapi/routing/login.go | 22 ++-- clientapi/routing/login_test.go | 38 ++++++ clientapi/routing/receipt.go | 7 +- clientapi/routing/register.go | 14 +- clientapi/routing/sendevent.go | 2 +- internal/validate.go | 136 +++++++++++++++++++- internal/validate_test.go | 134 ++++++++++++++++++- 11 files changed, 530 insertions(+), 35 deletions(-) create mode 100644 clientapi/auth/login_application_service.go diff --git a/clientapi/auth/login.go b/clientapi/auth/login.go index 77835614e..58a27e593 100644 --- a/clientapi/auth/login.go +++ b/clientapi/auth/login.go @@ -15,7 +15,6 @@ package auth import ( - "context" "encoding/json" "io" "net/http" @@ -32,8 +31,13 @@ import ( // called after authorization has completed, with the result of the authorization. // If the final return value is non-nil, an error occurred and the cleanup function // is nil. -func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) { - reqBytes, err := io.ReadAll(r) +func LoginFromJSONReader( + req *http.Request, + useraccountAPI uapi.UserLoginAPI, + userAPI UserInternalAPIForLogin, + cfg *config.ClientAPI, +) (*Login, LoginCleanupFunc, *util.JSONResponse) { + reqBytes, err := io.ReadAll(req.Body) if err != nil { err := &util.JSONResponse{ Code: http.StatusBadRequest, @@ -65,6 +69,20 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U UserAPI: userAPI, Config: cfg, } + case authtypes.LoginTypeApplicationService: + token, err := ExtractAccessToken(req) + if err != nil { + err := &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.MissingToken(err.Error()), + } + return nil, nil, err + } + + typ = &LoginTypeApplicationService{ + Config: cfg, + Token: token, + } default: err := util.JSONResponse{ Code: http.StatusBadRequest, @@ -73,7 +91,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U return nil, nil, &err } - return typ.LoginFromJSON(ctx, reqBytes) + return typ.LoginFromJSON(req.Context(), reqBytes) } // UserInternalAPIForLogin contains the aspects of UserAPI required for logging in. diff --git a/clientapi/auth/login_application_service.go b/clientapi/auth/login_application_service.go new file mode 100644 index 000000000..dd4a9cbb4 --- /dev/null +++ b/clientapi/auth/login_application_service.go @@ -0,0 +1,55 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "context" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/util" +) + +// LoginTypeApplicationService describes how to authenticate as an +// application service +type LoginTypeApplicationService struct { + Config *config.ClientAPI + Token string +} + +// Name implements Type +func (t *LoginTypeApplicationService) Name() string { + return authtypes.LoginTypeApplicationService +} + +// LoginFromJSON implements Type +func (t *LoginTypeApplicationService) LoginFromJSON( + ctx context.Context, reqBytes []byte, +) (*Login, LoginCleanupFunc, *util.JSONResponse) { + var r Login + if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil { + return nil, nil, err + } + + _, err := internal.ValidateApplicationServiceRequest(t.Config, r.Identifier.User, t.Token) + if err != nil { + return nil, nil, err + } + + cleanup := func(ctx context.Context, j *util.JSONResponse) {} + return &r, cleanup, nil +} diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go index 93d3e2713..a2c2a719c 100644 --- a/clientapi/auth/login_test.go +++ b/clientapi/auth/login_test.go @@ -17,7 +17,9 @@ package auth import ( "context" "net/http" + "net/http/httptest" "reflect" + "regexp" "strings" "testing" @@ -33,8 +35,9 @@ func TestLoginFromJSONReader(t *testing.T) { ctx := context.Background() tsts := []struct { - Name string - Body string + Name string + Body string + Token string WantUsername string WantDeviceID string @@ -62,6 +65,30 @@ func TestLoginFromJSONReader(t *testing.T) { WantDeviceID: "adevice", WantDeletedTokens: []string{"atoken"}, }, + { + Name: "appServiceWorksUserID", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:example.com" }, + "device_id": "adevice" + }`, + Token: "astoken", + + WantUsername: "@alice:example.com", + WantDeviceID: "adevice", + }, + { + Name: "appServiceWorksLocalpart", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "alice" }, + "device_id": "adevice" + }`, + Token: "astoken", + + WantUsername: "alice", + WantDeviceID: "adevice", + }, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -72,11 +99,35 @@ func TestLoginFromJSONReader(t *testing.T) { ServerName: serverName, }, }, + Derived: &config.Derived{ + ApplicationServices: []config.ApplicationService{ + { + ID: "anapplicationservice", + ASToken: "astoken", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": { + { + Exclusive: true, + Regex: "@alice:example.com", + RegexpObject: regexp.MustCompile("@alice:example.com"), + }, + }, + }, + }, + }, + }, } - login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) - if err != nil { - t.Fatalf("LoginFromJSONReader failed: %+v", err) + + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body)) + if tst.Token != "" { + req.Header.Add("Authorization", "Bearer "+tst.Token) } + + login, cleanup, jsonErr := LoginFromJSONReader(req, &userAPI, &userAPI, cfg) + if jsonErr != nil { + t.Fatalf("LoginFromJSONReader failed: %+v", jsonErr) + } + cleanup(ctx, &util.JSONResponse{Code: http.StatusOK}) if login.Username() != tst.WantUsername { @@ -104,8 +155,9 @@ func TestBadLoginFromJSONReader(t *testing.T) { ctx := context.Background() tsts := []struct { - Name string - Body string + Name string + Body string + Token string WantErrCode spec.MatrixErrorCode }{ @@ -142,6 +194,45 @@ func TestBadLoginFromJSONReader(t *testing.T) { }`, WantErrCode: spec.ErrorInvalidParam, }, + { + Name: "noASToken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_MISSING_TOKEN", + }, + { + Name: "badASToken", + Token: "badastoken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_UNKNOWN_TOKEN", + }, + { + Name: "badASNamespace", + Token: "astoken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@bob:example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_EXCLUSIVE", + }, + { + Name: "badASUserID", + Token: "astoken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:wrong.example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_INVALID_USERNAME", + }, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -152,8 +243,30 @@ func TestBadLoginFromJSONReader(t *testing.T) { ServerName: serverName, }, }, + Derived: &config.Derived{ + ApplicationServices: []config.ApplicationService{ + { + ID: "anapplicationservice", + ASToken: "astoken", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": { + { + Exclusive: true, + Regex: "@alice:example.com", + RegexpObject: regexp.MustCompile("@alice:example.com"), + }, + }, + }, + }, + }, + }, } - _, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body)) + if tst.Token != "" { + req.Header.Add("Authorization", "Bearer "+tst.Token) + } + + _, cleanup, errRes := LoginFromJSONReader(req, &userAPI, &userAPI, cfg) if errRes == nil { cleanup(ctx, nil) t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode) diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index 92d83ad29..9831450cc 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -55,7 +55,7 @@ type LoginCleanupFunc func(context.Context, *util.JSONResponse) // https://matrix.org/docs/spec/client_server/r0.6.1#identifier-types type LoginIdentifier struct { Type string `json:"type"` - // when type = m.id.user + // when type = m.id.user or m.id.application_service User string `json:"user"` // when type = m.id.thirdparty Medium string `json:"medium"` diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index bc38b8340..0f55c8816 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -19,6 +19,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -40,28 +41,25 @@ type flow struct { Type string `json:"type"` } -func passwordLogin() flows { - f := flows{} - s := flow{ - Type: "m.login.password", - } - f.Flows = append(f.Flows, s) - return f -} - // Login implements GET and POST /login func Login( req *http.Request, userAPI userapi.ClientUserAPI, cfg *config.ClientAPI, ) util.JSONResponse { if req.Method == http.MethodGet { - // TODO: support other forms of login other than password, depending on config options + loginFlows := []flow{{Type: authtypes.LoginTypePassword}} + if len(cfg.Derived.ApplicationServices) > 0 { + loginFlows = append(loginFlows, flow{Type: authtypes.LoginTypeApplicationService}) + } + // TODO: support other forms of login, depending on config options return util.JSONResponse{ Code: http.StatusOK, - JSON: passwordLogin(), + JSON: flows{ + Flows: loginFlows, + }, } } else if req.Method == http.MethodPost { - login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg) + login, cleanup, authErr := auth.LoginFromJSONReader(req, userAPI, userAPI, cfg) if authErr != nil { return *authErr } diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index 1c628b196..3f8934490 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -114,6 +114,44 @@ func TestLogin(t *testing.T) { ctx := context.Background() + // Inject a dummy application service, so we have a "m.login.application_service" + // in the login flows + as := &config.ApplicationService{} + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} + + t.Run("Supported log-in flows are returned", func(t *testing.T) { + req := test.NewRequest(t, http.MethodGet, "/_matrix/client/v3/login") + rec := httptest.NewRecorder() + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("failed to get log-in flows: %s", rec.Body.String()) + } + + t.Logf("response: %s", rec.Body.String()) + resp := flows{} + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + + appServiceFound := false + passwordFound := false + for _, flow := range resp.Flows { + if flow.Type == "m.login.password" { + passwordFound = true + } else if flow.Type == "m.login.application_service" { + appServiceFound = true + } else { + t.Fatalf("got unknown login flow: %s", flow.Type) + } + } + if !appServiceFound { + t.Fatal("m.login.application_service missing from login flows") + } + if !passwordFound { + t.Fatal("m.login.password missing from login flows") + } + }) + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ diff --git a/clientapi/routing/receipt.go b/clientapi/routing/receipt.go index be6542979..1d7e35562 100644 --- a/clientapi/routing/receipt.go +++ b/clientapi/routing/receipt.go @@ -23,13 +23,12 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) -func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { +func SetReceipt(req *http.Request, userAPI userapi.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { timestamp := spec.AsTimestamp(time.Now()) logrus.WithFields(logrus.Fields{ "roomID": roomID, @@ -54,13 +53,13 @@ func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *prod } } - dataReq := api.InputAccountDataRequest{ + dataReq := userapi.InputAccountDataRequest{ UserID: device.UserID, DataType: "m.fully_read", RoomID: roomID, AccountData: data, } - dataRes := api.InputAccountDataResponse{} + dataRes := userapi.InputAccountDataResponse{} if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") return util.ErrorResponse(err) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 090a2fc20..558418a6f 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -647,6 +647,16 @@ func handleGuestRegistration( } } +// localpartMatchesExclusiveNamespaces will check if a given username matches any +// application service's exclusive users namespace +func localpartMatchesExclusiveNamespaces( + cfg *config.ClientAPI, + localpart string, +) bool { + userID := userutil.MakeUserID(localpart, cfg.Matrix.ServerName) + return cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(userID) +} + // handleRegistrationFlow will direct and complete registration flow stages // that the client has requested. // nolint: gocyclo @@ -695,7 +705,7 @@ func handleRegistrationFlow( // If an access token is provided, ignore this check this is an appservice // request and we will validate in validateApplicationService if len(cfg.Derived.ApplicationServices) != 0 && - UsernameMatchesExclusiveNamespaces(cfg, r.Username) { + localpartMatchesExclusiveNamespaces(cfg, r.Username) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.ASExclusive("This username is reserved by an application service."), @@ -772,7 +782,7 @@ func handleApplicationServiceRegistration( // Check application service register user request is valid. // The application service's ID is returned if so. - appserviceID, err := validateApplicationService( + appserviceID, err := internal.ValidateApplicationServiceRequest( cfg, r.Username, accessToken, ) if err != nil { diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 69131966b..44e82aed0 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -224,7 +224,7 @@ func SendEvent( req.Context(), rsAPI, api.KindNew, []*types.HeaderedEvent{ - &types.HeaderedEvent{PDU: e}, + {PDU: e}, }, device.UserDomain(), domain, diff --git a/internal/validate.go b/internal/validate.go index 7f0d8b9e6..da8b35cd3 100644 --- a/internal/validate.go +++ b/internal/validate.go @@ -20,6 +20,9 @@ import ( "net/http" "regexp" + "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) @@ -100,10 +103,139 @@ func UsernameResponse(err error) *util.JSONResponse { // ValidateApplicationServiceUsername returns an error if the username is invalid for an application service func ValidateApplicationServiceUsername(localpart string, domain spec.ServerName) error { - if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength { + userID := userutil.MakeUserID(localpart, domain) + return ValidateApplicationServiceUserID(userID) +} + +func ValidateApplicationServiceUserID(userID string) error { + if len(userID) > maxUsernameLength { return ErrUsernameTooLong - } else if !validUsernameRegex.MatchString(localpart) { + } + + localpart, _, err := gomatrixserverlib.SplitID('@', userID) + if err != nil || !validUsernameRegex.MatchString(localpart) { return ErrUsernameInvalid } + return nil } + +// userIDIsWithinApplicationServiceNamespace checks to see if a given userID +// falls within any of the namespaces of a given Application Service. If no +// Application Service is given, it will check to see if it matches any +// Application Service's namespace. +func userIDIsWithinApplicationServiceNamespace( + cfg *config.ClientAPI, + userID string, + appservice *config.ApplicationService, +) bool { + var localpart, domain, err = gomatrixserverlib.SplitID('@', userID) + if err != nil { + // Not a valid userID + return false + } + + if !cfg.Matrix.IsLocalServerName(domain) { + // This is a federated userID + return false + } + + if localpart == appservice.SenderLocalpart { + // This is the application service bot userID + return true + } + + // Loop through given application service's namespaces and see if any match + for _, namespace := range appservice.NamespaceMap["users"] { + // Application service namespaces are checked for validity in config + if namespace.RegexpObject.MatchString(userID) { + return true + } + } + + return false +} + +// usernameMatchesMultipleExclusiveNamespaces will check if a given username matches +// more than one exclusive namespace. More than one is not allowed +func userIDMatchesMultipleExclusiveNamespaces( + cfg *config.ClientAPI, + userID string, +) bool { + // Check namespaces and see if more than one match + matchCount := 0 + for _, appservice := range cfg.Derived.ApplicationServices { + if appservice.OwnsNamespaceCoveringUserId(userID) { + if matchCount++; matchCount > 1 { + return true + } + } + } + return false +} + +// ValidateApplicationServiceRequest checks if a provided application service +// token corresponds to one that is registered, and, if so, checks if the +// supplied userIDOrLocalpart is within that application service's namespace. +// +// As long as these two requirements are met, the matched application service +// ID will be returned. Otherwise, it will return a JSON response with the +// appropriate error message. +func ValidateApplicationServiceRequest( + cfg *config.ClientAPI, + userIDOrLocalpart string, + accessToken string, +) (string, *util.JSONResponse) { + localpart, domain, err := userutil.ParseUsernameParam(userIDOrLocalpart, cfg.Matrix) + if err != nil { + return "", &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: spec.InvalidUsername(err.Error()), + } + } + + userID := userutil.MakeUserID(localpart, domain) + + // Check if the token if the application service is valid with one we have + // registered in the config. + var matchedApplicationService *config.ApplicationService + for _, appservice := range cfg.Derived.ApplicationServices { + if appservice.ASToken == accessToken { + matchedApplicationService = &appservice + break + } + } + if matchedApplicationService == nil { + return "", &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: spec.UnknownToken("Supplied access_token does not match any known application service"), + } + } + + // Ensure the desired username is within at least one of the application service's namespaces. + if !userIDIsWithinApplicationServiceNamespace(cfg, userID, matchedApplicationService) { + // If we didn't find any matches, return M_EXCLUSIVE + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.ASExclusive(fmt.Sprintf( + "Supplied username %s did not match any namespaces for application service ID: %s", userIDOrLocalpart, matchedApplicationService.ID)), + } + } + + // Check this user does not fit multiple application service namespaces + if userIDMatchesMultipleExclusiveNamespaces(cfg, userID) { + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.ASExclusive(fmt.Sprintf( + "Supplied username %s matches multiple exclusive application service namespaces. Only 1 match allowed", userIDOrLocalpart)), + } + } + + // Check username application service is trying to register is valid + if err := ValidateApplicationServiceUserID(userID); err != nil { + return "", UsernameResponse(err) + } + + // No errors, registration valid + return matchedApplicationService.ID, nil +} diff --git a/internal/validate_test.go b/internal/validate_test.go index e3a10178f..cd2626133 100644 --- a/internal/validate_test.go +++ b/internal/validate_test.go @@ -3,9 +3,11 @@ package internal import ( "net/http" "reflect" + "regexp" "strings" "testing" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) @@ -38,7 +40,7 @@ func Test_validatePassword(t *testing.T) { t.Run(tt.name, func(t *testing.T) { gotErr := ValidatePassword(tt.password) if !reflect.DeepEqual(gotErr, tt.wantError) { - t.Errorf("validatePassword() = %v, wantJSON %v", gotErr, tt.wantError) + t.Errorf("validatePassword() = %v, wantError %v", gotErr, tt.wantError) } if got := PasswordResponse(gotErr); !reflect.DeepEqual(got, tt.wantJSON) { @@ -167,3 +169,133 @@ func Test_validateUsername(t *testing.T) { }) } } + +// This method tests validation of the provided Application Service token and +// username that they're registering +func TestValidateApplicationServiceRequest(t *testing.T) { + // Create a fake application service + regex := "@_appservice_.*" + fakeNamespace := config.ApplicationServiceNamespace{ + Exclusive: true, + Regex: regex, + RegexpObject: regexp.MustCompile(regex), + } + fakeSenderLocalpart := "_appservice_bot" + fakeApplicationService := config.ApplicationService{ + ID: "FakeAS", + URL: "null", + ASToken: "1234", + HSToken: "4321", + SenderLocalpart: fakeSenderLocalpart, + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": {fakeNamespace}, + }, + } + + // Create a second fake application service where userIDs ending in + // "_overlap" overlap with the first. + regex = "@_.*_overlap" + fakeNamespace = config.ApplicationServiceNamespace{ + Exclusive: true, + Regex: regex, + RegexpObject: regexp.MustCompile(regex), + } + fakeApplicationServiceOverlap := config.ApplicationService{ + ID: "FakeASOverlap", + URL: fakeApplicationService.URL, + ASToken: fakeApplicationService.ASToken, + HSToken: fakeApplicationService.HSToken, + SenderLocalpart: "_appservice_bot_overlap", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": {fakeNamespace}, + }, + } + + // Set up a config + fakeConfig := &config.Dendrite{} + fakeConfig.Defaults(config.DefaultOpts{ + Generate: true, + }) + fakeConfig.Global.ServerName = "localhost" + fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService, fakeApplicationServiceOverlap} + + tests := []struct { + name string + localpart string + asToken string + wantError bool + wantASID string + }{ + // Access token is correct, userID omitted so we are acting as SenderLocalpart + { + name: "correct access token but omitted userID", + localpart: fakeSenderLocalpart, + asToken: fakeApplicationService.ASToken, + wantError: false, + wantASID: fakeApplicationService.ID, + }, + // Access token is incorrect, userID omitted so we are acting as SenderLocalpart + { + name: "incorrect access token but omitted userID", + localpart: fakeSenderLocalpart, + asToken: "xxxx", + wantError: true, + wantASID: "", + }, + // Access token is correct, acting as valid userID + { + name: "correct access token and valid userID", + localpart: "_appservice_bob", + asToken: fakeApplicationService.ASToken, + wantError: false, + wantASID: fakeApplicationService.ID, + }, + // Access token is correct, acting as invalid userID + { + name: "correct access token but invalid userID", + localpart: "_something_else", + asToken: fakeApplicationService.ASToken, + wantError: true, + wantASID: "", + }, + // Access token is correct, acting as userID that matches two exclusive namespaces + { + name: "correct access token but non-exclusive userID", + localpart: "_appservice_overlap", + asToken: fakeApplicationService.ASToken, + wantError: true, + wantASID: "", + }, + // Access token is correct, acting as matching userID that is too long + { + name: "correct access token but too long userID", + localpart: "_appservice_" + strings.Repeat("a", maxUsernameLength), + asToken: fakeApplicationService.ASToken, + wantError: true, + wantASID: "", + }, + // Access token is correct, acting as userID that matches but is invalid + { + name: "correct access token and matching but invalid userID", + localpart: "@_appservice_bob::", + asToken: fakeApplicationService.ASToken, + wantError: true, + wantASID: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotASID, gotResp := ValidateApplicationServiceRequest(&fakeConfig.ClientAPI, tt.localpart, tt.asToken) + if tt.wantError && gotResp == nil { + t.Error("expected an error, but succeeded") + } + if !tt.wantError && gotResp != nil { + t.Errorf("expected success, but returned error: %v", *gotResp) + } + if gotASID != tt.wantASID { + t.Errorf("returned '%s', but expected '%s'", gotASID, tt.wantASID) + } + }) + } +} From 210bce9938c858425c9e1db586cf3aaea235182a Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Sat, 25 Nov 2023 19:12:21 +0100 Subject: [PATCH 39/40] Update GMSL to avoid logging unnecessary messages --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8dad4ebf0..387e99d8c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca + github.com/matrix-org/gomatrixserverlib v0.0.0-20231122130434-2beadf141c98 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 diff --git a/go.sum b/go.sum index b173f6b85..0dbd3ac48 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca h1:JCP72vU4Vcmur2071RwYVOSoekR+ZjbC03wZD5lAAK0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231122130434-2beadf141c98 h1:+GsF24sG9WJccf5k54cZj9tLgwW3UON1ujz5u+8SHxc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231122130434-2beadf141c98/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= From 61e5dc47d7525b60e749d77618dc5b66848dac90 Mon Sep 17 00:00:00 2001 From: Cat Date: Sat, 25 Nov 2023 20:24:13 +0100 Subject: [PATCH 40/40] Added Docker commands for Windows (#3267) ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately No tests were added due to it being a small documentation addition Signed-off-by: `Cat Catry ` --- docs/installation/docker/1_docker.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/installation/docker/1_docker.md b/docs/installation/docker/1_docker.md index 1fe792636..f33999725 100644 --- a/docs/installation/docker/1_docker.md +++ b/docs/installation/docker/1_docker.md @@ -26,6 +26,8 @@ docker run --rm --entrypoint="/usr/bin/generate-keys" \ -v $(pwd)/config:/mnt \ matrixdotorg/dendrite-monolith:latest \ -private-key /mnt/matrix_key.pem + +# Windows equivalent: docker run --rm --entrypoint="/usr/bin/generate-keys" -v %cd%/config:/mnt matrixdotorg/dendrite-monolith:latest -private-key /mnt/matrix_key.pem ``` (**NOTE**: This only needs to be executed **once**, as you otherwise overwrite the key) @@ -44,6 +46,8 @@ docker run --rm --entrypoint="/bin/sh" \ -dir /var/dendrite/ \ -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable \ -server YourDomainHere > /mnt/dendrite.yaml" + +# Windows equivalent: docker run --rm --entrypoint="/bin/sh" -v %cd%/config:/mnt matrixdotorg/dendrite-monolith:latest -c "/usr/bin/generate-config -dir /var/dendrite/ -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable -server YourDomainHere > /mnt/dendrite.yaml" ``` You can then change `config/dendrite.yaml` to your liking.