From 9b479dd3bb10da61989e71d22746430262c4b6d2 Mon Sep 17 00:00:00 2001 From: Brian Meek Date: Fri, 9 Sep 2022 19:04:37 -0700 Subject: [PATCH 1/4] Fix test checking which clients connected Signed-off-by: Brian Meek --- userapi/storage/tables/stats_table_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/userapi/storage/tables/stats_table_test.go b/userapi/storage/tables/stats_table_test.go index 7367c06dc..0b021803d 100644 --- a/userapi/storage/tables/stats_table_test.go +++ b/userapi/storage/tables/stats_table_test.go @@ -301,10 +301,10 @@ func Test_UserStatistics(t *testing.T) { }, R30UsersV2: map[string]int64{ "ios": 0, - "android": 0, - "web": 0, + "android": 1, + "web": 1, "electron": 0, - "all": 0, + "all": 2, }, AllUsers: 6, NonBridgedUsers: 5, From 087b3b234479f04ebeccbed1ea2d3d16527a5ef8 Mon Sep 17 00:00:00 2001 From: texuf Date: Wed, 7 Sep 2022 18:01:08 -0700 Subject: [PATCH 2/4] Fix broken notification incremental sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I was not seeing unread notifications in sync, even if they were written to the db Notifications are in their own stream, but the code was trying to tack them onto the join room stream. If the offsets “happened” to line up, you might get a count here or there, but they would be totally wrong (jump from 1 to 0 to 2, etc) To fix, put them in their own top level object, handle them on the client. Signed-off-by: Austin Ellis --- syncapi/streams/stream_notificationdata.go | 11 ++++++++++- syncapi/types/types.go | 22 ++++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go index 8ba9e07ca..ea5dcd85a 100644 --- a/syncapi/streams/stream_notificationdata.go +++ b/syncapi/streams/stream_notificationdata.go @@ -46,10 +46,19 @@ func (p *NotificationDataStreamProvider) IncrementalSync( if counts == nil { continue } - jr.UnreadNotifications.HighlightCount = counts.UnreadHighlightCount jr.UnreadNotifications.NotificationCount = counts.UnreadNotificationCount req.Response.Rooms.Join[roomID] = jr } + + // BEGIN ZION CODE but return all notifications regardless of whether they're in a room we're in. + for roomID, counts := range countsByRoom { + unreadNotificationsData := *types.NewUnreadNotificationsResponse() + + unreadNotificationsData.HighlightCount = counts.UnreadHighlightCount + unreadNotificationsData.NotificationCount = counts.UnreadNotificationCount + req.Response.Rooms.UnreadNotifications[roomID] = unreadNotificationsData + } + // END ZION CODE return to } diff --git a/syncapi/types/types.go b/syncapi/types/types.go index d75d53ca9..47eeddbaa 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -337,10 +337,11 @@ type Response struct { Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` } `json:"presence,omitempty"` Rooms struct { - Join map[string]JoinResponse `json:"join,omitempty"` - Peek map[string]JoinResponse `json:"peek,omitempty"` - Invite map[string]InviteResponse `json:"invite,omitempty"` - Leave map[string]LeaveResponse `json:"leave,omitempty"` + Join map[string]JoinResponse `json:"join,omitempty"` + Peek map[string]JoinResponse `json:"peek,omitempty"` + Invite map[string]InviteResponse `json:"invite,omitempty"` + Leave map[string]LeaveResponse `json:"leave,omitempty"` + UnreadNotifications map[string]UnreadNotificationsResponse `json:"unread_notifications,omitempty"` } `json:"rooms,omitempty"` ToDevice struct { Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"` @@ -360,6 +361,7 @@ func (r *Response) HasUpdates() bool { len(r.Rooms.Join) > 0 || len(r.Rooms.Leave) > 0 || len(r.Rooms.Peek) > 0 || + len(r.Rooms.UnreadNotifications) > 0 || len(r.ToDevice.Events) > 0 || len(r.DeviceLists.Changed) > 0 || len(r.DeviceLists.Left) > 0) @@ -374,6 +376,7 @@ func NewResponse() *Response { res.Rooms.Peek = map[string]JoinResponse{} res.Rooms.Invite = map[string]InviteResponse{} res.Rooms.Leave = map[string]LeaveResponse{} + res.Rooms.UnreadNotifications = map[string]UnreadNotificationsResponse{} // Also pre-intialise empty slices or else we'll insert 'null' instead of '[]' for the value. // TODO: We really shouldn't have to do all this to coerce encoding/json to Do The Right Thing. We should @@ -393,6 +396,7 @@ func (r *Response) IsEmpty() bool { return len(r.Rooms.Join) == 0 && len(r.Rooms.Invite) == 0 && len(r.Rooms.Leave) == 0 && + len(r.Rooms.UnreadNotifications) == 0 && len(r.AccountData.Events) == 0 && len(r.Presence.Events) == 0 && len(r.ToDevice.Events) == 0 @@ -435,6 +439,16 @@ func NewJoinResponse() *JoinResponse { return &res } +type UnreadNotificationsResponse struct { + HighlightCount int `json:"highlight_count"` + NotificationCount int `json:"notification_count"` +} + +func NewUnreadNotificationsResponse() *UnreadNotificationsResponse { + res := UnreadNotificationsResponse{} + return &res +} + // InviteResponse represents a /sync response for a room which is under the 'invite' key. type InviteResponse struct { InviteState struct { From 46f0ed379a23768853df9c917e1eaf16899ceda3 Mon Sep 17 00:00:00 2001 From: texuf Date: Wed, 14 Sep 2022 22:14:21 -0700 Subject: [PATCH 3/4] Hack out read receipt de-duping protection zion hack - always send the notification data on a read receipt the notifications are stored in two different databases, and somehow the notification database prunes data, so trying to mark old notifications as read will fail, but the notification will still exist in the other db todo, revist when this refactor lands: https://github.com/matrix-org/dendrite/pull/2688/files it looks like we cleanup the notification table after a day func (s *notificationsStatements) Clean(ctx context.Context, txn *sql.Tx) error { _, err := sqlutil.TxStmt(txn, s.cleanNotificationsStmt).ExecContext( ctx, time.Now().AddDate(0, 0, -1).UnixNano()/int64(time.Millisecond), // keep non-highlights for a day time.Now().AddDate(0, -1, 0).UnixNano()/int64(time.Millisecond), // keep highlights for a month ) return err } But we don't clean up the notifications in the syncAPI table. When we send a read receipt we first do a updated _, err := s.db.SetNotificationsRead(ctx, localpart, roomID, int64(read.Read), true) and only forward the message on if the table was updated. If a user waits more than a day to send a read receipt, they can't clear their notifications. --- userapi/consumers/syncapi_readupdate.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/userapi/consumers/syncapi_readupdate.go b/userapi/consumers/syncapi_readupdate.go index 54654f757..21b6efc91 100644 --- a/userapi/consumers/syncapi_readupdate.go +++ b/userapi/consumers/syncapi_readupdate.go @@ -95,22 +95,27 @@ func (s *OutputReadUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M log.Tracef("Received read update from sync API: %#v", read) if read.Read > 0 { - updated, err := s.db.SetNotificationsRead(ctx, localpart, roomID, int64(read.Read), true) + /*updated*/ _, err := s.db.SetNotificationsRead(ctx, localpart, roomID, int64(read.Read), true) if err != nil { log.WithError(err).Error("userapi EDU consumer") return false } - if updated { - if err = s.syncProducer.GetAndSendNotificationData(ctx, userID, roomID); err != nil { - log.WithError(err).Error("userapi EDU consumer: GetAndSendNotificationData failed") - return false - } - if err = util.NotifyUserCountsAsync(ctx, s.pgClient, localpart, s.db); err != nil { - log.WithError(err).Error("userapi EDU consumer: NotifyUserCounts failed") - return false - } + // zion hack - always send the notification data + // the notifications are stored in two different databases, and somehow the notification database + // prunes data, so trying to mark old notifications as read will fail, but the notification will still exist in the other db + // todo, revist when this refactor lands: https://github.com/matrix-org/dendrite/pull/2688/files + + //if updated { + if err = s.syncProducer.GetAndSendNotificationData(ctx, userID, roomID); err != nil { + log.WithError(err).Error("userapi EDU consumer: GetAndSendNotificationData failed") + return false } + if err = util.NotifyUserCountsAsync(ctx, s.pgClient, localpart, s.db); err != nil { + log.WithError(err).Error("userapi EDU consumer: NotifyUserCounts failed") + return false + } + //} } if read.FullyRead > 0 { From 6ffe8147f566f106b80af8feefd07815fc31a96b Mon Sep 17 00:00:00 2001 From: Brian Meek Date: Thu, 15 Sep 2022 12:17:41 -0700 Subject: [PATCH 4/4] Add commit hash to version API Signed-off-by: Brian Meek --- clientapi/routing/routing.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e8b38a049..b5c18fbd2 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -19,6 +19,8 @@ import ( "net/http" "strings" + "runtime/debug" + "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/api" @@ -41,6 +43,17 @@ import ( "github.com/sirupsen/logrus" ) +var commitHash = func() string { + if info, ok := debug.ReadBuildInfo(); ok { + for _, setting := range info.Settings { + if setting.Key == "vcs.revision" { + return setting.Value + } + } + } + return "" +}() + // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. // @@ -101,6 +114,7 @@ func Setup( JSON: struct { Versions []string `json:"versions"` UnstableFeatures map[string]bool `json:"unstable_features"` + CommitHash string `json:"commit_hash"` }{Versions: []string{ "r0.0.1", "r0.1.0", @@ -112,7 +126,7 @@ func Setup( "v1.0", "v1.1", "v1.2", - }, UnstableFeatures: unstableFeatures}, + }, UnstableFeatures: unstableFeatures, CommitHash: commitHash}, } }), ).Methods(http.MethodGet, http.MethodOptions)