From d4710217f8d96ee6889e1e4e7c26defc5456b523 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Sun, 2 Oct 2022 11:31:40 +0200 Subject: [PATCH 01/61] Use non-HTTPS as default URL, as most people will be running behind a reverse proxy --- cmd/create-account/main.go | 2 +- docs/administration/1_createusers.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index a9357f6db..52301415f 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -64,7 +64,7 @@ var ( pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin") isAdmin = flag.Bool("admin", false, "Create an admin account") resetPassword = flag.Bool("reset-password", false, "Deprecated") - serverURL = flag.String("url", "https://localhost:8448", "The URL to connect to.") + serverURL = flag.String("url", "http://localhost:8008", "The URL to connect to.") validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`) timeout = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server") ) diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md index 3468398ac..94399a04a 100644 --- a/docs/administration/1_createusers.md +++ b/docs/administration/1_createusers.md @@ -1,3 +1,4 @@ + --- title: Creating user accounts parent: Administration @@ -31,11 +32,11 @@ To create a new **admin account**, add the `-admin` flag: ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin ``` -By default `create-account` uses `https://localhost:8448` to connect to Dendrite, this can be overwritten using +By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using the `-url` flag: ```bash -./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url http://localhost:8008 +./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url https://localhost:8448 ``` An example of using `create-account` when running in **Docker**, having found the `CONTAINERNAME` from `docker ps`: From d32f60249d8e248d8f8334b044e016ee384797aa Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 11:38:20 +0100 Subject: [PATCH 02/61] Modify sync transaction behaviour (#2758) This now uses a transaction per stream, so that errors in one stream don't propagate to another, and we therefore no longer need to do hacks to reopen a new transaction after aborting a failed one. --- syncapi/storage/interface.go | 1 - syncapi/storage/shared/storage_sync.go | 13 -- syncapi/streams/stream_accountdata.go | 1 - syncapi/streams/stream_devicelist.go | 2 - syncapi/streams/stream_invite.go | 1 - syncapi/streams/stream_notificationdata.go | 1 - syncapi/streams/stream_pdu.go | 25 +-- syncapi/streams/stream_presence.go | 1 - syncapi/streams/stream_receipt.go | 1 - syncapi/streams/stream_sendtodevice.go | 1 - syncapi/sync/requestpool.go | 199 +++++++++++++++------ 11 files changed, 155 insertions(+), 91 deletions(-) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index be75f8ad0..4a03aca74 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -29,7 +29,6 @@ import ( type DatabaseTransaction interface { sqlutil.Transaction - Reset() (err error) SharedUsers MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index 6cc83ebc8..0e19d97d2 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -31,19 +31,6 @@ func (d *DatabaseTransaction) Rollback() error { return d.txn.Rollback() } -func (d *DatabaseTransaction) Reset() (err error) { - if d.txn == nil { - return nil - } - if err = d.txn.Rollback(); err != nil { - return err - } - if d.txn, err = d.DB.BeginTx(d.ctx, nil); err != nil { - return err - } - return -} - func (d *DatabaseTransaction) MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) { id, err := d.OutputEvents.SelectMaxEventID(ctx, d.txn) if err != nil { diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index f3e7fbdaa..3f2f7d134 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -54,7 +54,6 @@ func (p *AccountDataStreamProvider) IncrementalSync( ) if err != nil { req.Log.WithError(err).Error("p.DB.GetAccountDataInRange failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go index 307099b8f..7996c2038 100644 --- a/syncapi/streams/stream_devicelist.go +++ b/syncapi/streams/stream_devicelist.go @@ -34,13 +34,11 @@ func (p *DeviceListStreamProvider) IncrementalSync( to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") - _ = snapshot.Reset() return from } err = internal.DeviceOTKCounts(req.Context, p.keyAPI, req.Device.UserID, req.Device.ID, req.Response) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 4c889b8f5..17b3b8434 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -56,7 +56,6 @@ func (p *InviteStreamProvider) IncrementalSync( ) if err != nil { req.Log.WithError(err).Error("p.DB.InviteEventsInRange failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go index 5154dd332..5a81fd09a 100644 --- a/syncapi/streams/stream_notificationdata.go +++ b/syncapi/streams/stream_notificationdata.go @@ -46,7 +46,6 @@ func (p *NotificationDataStreamProvider) IncrementalSync( countsByRoom, err := snapshot.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms) if err != nil { req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 01ddf9ac9..d252265ff 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -75,7 +75,6 @@ func (p *PDUStreamProvider) CompleteSync( joinedRoomIDs, err := snapshot.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join) if err != nil { req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed") - _ = snapshot.Reset() return from } @@ -102,10 +101,10 @@ func (p *PDUStreamProvider) CompleteSync( ) if jerr != nil { req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") - if err = snapshot.Reset(); err != nil { + if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone { return from } - continue // return from + continue } req.Response.Rooms.Join[roomID] = *jr req.Rooms[roomID] = gomatrixserverlib.Join @@ -115,7 +114,6 @@ func (p *PDUStreamProvider) CompleteSync( peeks, err := snapshot.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r) if err != nil { req.Log.WithError(err).Error("p.DB.PeeksInRange failed") - _ = snapshot.Reset() return from } for _, peek := range peeks { @@ -126,10 +124,10 @@ func (p *PDUStreamProvider) CompleteSync( ) if err != nil { req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed") - if err = snapshot.Reset(); err != nil { + if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone { return from } - continue // return from + continue } req.Response.Rooms.Peek[peek.RoomID] = *jr } @@ -160,14 +158,12 @@ func (p *PDUStreamProvider) IncrementalSync( if req.WantFullState { if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed") - _ = snapshot.Reset() - return + return from } } else { if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltas failed") - _ = snapshot.Reset() - return + return from } } @@ -181,7 +177,6 @@ func (p *PDUStreamProvider) IncrementalSync( if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil { req.Log.WithError(err).Error("unable to update event filter with ignored users") - _ = snapshot.Reset() } newPos = from @@ -201,13 +196,10 @@ func (p *PDUStreamProvider) IncrementalSync( var pos types.StreamPosition if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil { req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed") - if err == context.DeadlineExceeded || err == context.Canceled { + if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone { return newPos } - if err = snapshot.Reset(); err != nil { - return from - } - continue // return to + continue } // Reset the position, as it is only for the special case of newly joined rooms if delta.NewlyJoined { @@ -307,7 +299,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") - _ = snapshot.Reset() } if len(delta.StateEvents) > 0 { diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 8a3f01c24..8b87af452 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -67,7 +67,6 @@ func (p *PresenceStreamProvider) IncrementalSync( presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000}) if err != nil { req.Log.WithError(err).Error("p.DB.PresenceAfter failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 79fd65bfe..8818a5533 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -52,7 +52,6 @@ func (p *ReceiptStreamProvider) IncrementalSync( lastPos, receipts, err := snapshot.RoomReceiptsAfter(ctx, joinedRooms, from) if err != nil { req.Log.WithError(err).Error("p.DB.RoomReceiptsAfter failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go index c79efad06..00b67cc42 100644 --- a/syncapi/streams/stream_sendtodevice.go +++ b/syncapi/streams/stream_sendtodevice.go @@ -44,7 +44,6 @@ func (p *SendToDeviceStreamProvider) IncrementalSync( lastPos, events, err := snapshot.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to) if err != nil { req.Log.WithError(err).Error("p.DB.SendToDeviceUpdatesForSync failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index bef0a5683..a71d32ab8 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -306,13 +306,19 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately") } - snapshot, err := rp.db.NewDatabaseSnapshot(req.Context()) - if err != nil { - logrus.WithError(err).Error("Failed to acquire database snapshot for sync request") - return jsonerror.InternalServerError() + withTransaction := func(from types.StreamPosition, f func(snapshot storage.DatabaseTransaction) types.StreamPosition) types.StreamPosition { + var succeeded bool + snapshot, err := rp.db.NewDatabaseSnapshot(req.Context()) + if err != nil { + logrus.WithError(err).Error("Failed to acquire database snapshot for sync request") + return from + } + defer func() { + succeeded = err == nil + sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) + }() + return f(snapshot) } - var succeeded bool - defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) if syncReq.Since.IsEmpty() { // Complete sync @@ -320,72 +326,162 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. // Get the current DeviceListPosition first, as the currentPosition // might advance while processing other streams, resulting in flakey // tests. - DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + DeviceListPosition: withTransaction( + syncReq.Since.DeviceListPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.DeviceListStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - PDUPosition: rp.streams.PDUStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + PDUPosition: withTransaction( + syncReq.Since.PDUPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.PDUStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - TypingPosition: rp.streams.TypingStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + TypingPosition: withTransaction( + syncReq.Since.TypingPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.TypingStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + ReceiptPosition: withTransaction( + syncReq.Since.ReceiptPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.ReceiptStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - InvitePosition: rp.streams.InviteStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + InvitePosition: withTransaction( + syncReq.Since.InvitePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.InviteStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + SendToDevicePosition: withTransaction( + syncReq.Since.SendToDevicePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.SendToDeviceStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + AccountDataPosition: withTransaction( + syncReq.Since.AccountDataPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.AccountDataStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + NotificationDataPosition: withTransaction( + syncReq.Since.NotificationDataPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.NotificationDataStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + PresencePosition: withTransaction( + syncReq.Since.PresencePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.PresenceStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), } } else { // Incremental sync syncReq.Response.NextBatch = types.StreamingToken{ - PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.PDUPosition, currentPos.PDUPosition, + PDUPosition: withTransaction( + syncReq.Since.PDUPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.PDUStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.PDUPosition, currentPos.PDUPosition, + ) + }, ), - TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.TypingPosition, currentPos.TypingPosition, + TypingPosition: withTransaction( + syncReq.Since.TypingPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.TypingStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.TypingPosition, currentPos.TypingPosition, + ) + }, ), - ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, + ReceiptPosition: withTransaction( + syncReq.Since.ReceiptPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.ReceiptStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, + ) + }, ), - InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.InvitePosition, currentPos.InvitePosition, + InvitePosition: withTransaction( + syncReq.Since.InvitePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.InviteStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.InvitePosition, currentPos.InvitePosition, + ) + }, ), - SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, + SendToDevicePosition: withTransaction( + syncReq.Since.SendToDevicePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.SendToDeviceStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, + ) + }, ), - AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, + AccountDataPosition: withTransaction( + syncReq.Since.AccountDataPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.AccountDataStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, + ) + }, ), - NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, + NotificationDataPosition: withTransaction( + syncReq.Since.NotificationDataPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.NotificationDataStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, + ) + }, ), - DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, + DeviceListPosition: withTransaction( + syncReq.Since.DeviceListPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.DeviceListStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, + ) + }, ), - PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.PresencePosition, currentPos.PresencePosition, + PresencePosition: withTransaction( + syncReq.Since.PresencePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.PresenceStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.PresencePosition, currentPos.PresencePosition, + ) + }, ), } // it's possible for there to be no updates for this user even though since < current pos, @@ -411,7 +507,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. } } - succeeded = true return util.JSONResponse{ Code: http.StatusOK, JSON: syncReq.Response, From b7f4bab358ea6de0932037242f71152bfc27c289 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 11:38:31 +0100 Subject: [PATCH 03/61] Hopefully fix P2P `--config` error (re. #2756) --- cmd/dendrite-demo-pinecone/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index da63f9a2c..fb988ebe1 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -89,6 +89,7 @@ func main() { if configFlagSet { cfg = setup.ParseFlags(true) sk = cfg.Global.PrivateKey + pk = sk.Public().(ed25519.PublicKey) } else { keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem" if _, err := os.Stat(keyfile); os.IsNotExist(err) { From c1e16fd41e24eb1c139b4e4acbef1a309d7d573b Mon Sep 17 00:00:00 2001 From: Ashley Nelson Date: Mon, 3 Oct 2022 05:57:21 -0500 Subject: [PATCH 04/61] Fix fragility of selectEventsWithEventIDsSQL queries (#2757) This fixes a temporary workaround with the `selectEventsWithEventIDsSQL` queries where fields need to be artificially added to the queries so the row results match the format of the `syncapi_output_room_events` table. I made similar functions that accept row results from the `syncapi_current_room_state` table and convert them into StreamEvents without the fields that are specific to output room events. There is also a unit test in the first commit to ensure the resulting behavior doesn't change from the modified queries and functions. Fixes #601. ### Pull Request Checklist * [x] I have added tests for PR _or_ I have justified why this PR doesn't need tests. * [x] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off) Signed-off-by: `Ashley Nelson ` Co-authored-by: Neil Alexander --- .../postgres/current_room_state_table.go | 38 ++++++-- .../sqlite3/current_room_state_table.go | 38 ++++++-- .../storage/tables/current_room_state_test.go | 88 +++++++++++++++++++ 3 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 syncapi/storage/tables/current_room_state_test.go diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 4ffd29610..2ccf0be1a 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -104,12 +104,7 @@ const selectStateEventSQL = "" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + - // TODO: The session_id and transaction_id blanks are here because - // the rowsToStreamEvents expects there to be exactly seven columns. We need to - // figure out if these really need to be in the DB, and if so, we need a - // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" + - " FROM syncapi_current_room_state WHERE event_id = ANY($1)" + "SELECT event_id, added_at, headered_event_json, history_visibility FROM syncapi_current_room_state WHERE event_id = ANY($1)" const selectSharedUsersSQL = "" + "SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" + @@ -365,7 +360,36 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( return nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") - return rowsToStreamEvents(rows) + return currentRoomStateRowsToStreamEvents(rows) +} + +func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { + var events []types.StreamEvent + for rows.Next() { + var ( + eventID string + streamPos types.StreamPosition + eventBytes []byte + historyVisibility gomatrixserverlib.HistoryVisibility + ) + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &historyVisibility); err != nil { + return nil, err + } + // TODO: Handle redacted events + var ev gomatrixserverlib.HeaderedEvent + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { + return nil, err + } + + ev.Visibility = historyVisibility + + events = append(events, types.StreamEvent{ + HeaderedEvent: &ev, + StreamPosition: streamPos, + }) + } + + return events, nil } func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) { diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index c4019fed2..ff45e786e 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -88,12 +88,7 @@ const selectStateEventSQL = "" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + - // TODO: The session_id and transaction_id blanks are here because - // the rowsToStreamEvents expects there to be exactly seven columns. We need to - // figure out if these really need to be in the DB, and if so, we need a - // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" + - " FROM syncapi_current_room_state WHERE event_id IN ($1)" + "SELECT event_id, added_at, headered_event_json, history_visibility FROM syncapi_current_room_state WHERE event_id IN ($1)" const selectSharedUsersSQL = "" + "SELECT state_key FROM syncapi_current_room_state WHERE room_id IN(" + @@ -378,7 +373,7 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( return nil, err } start = start + n - events, err := rowsToStreamEvents(rows) + events, err := currentRoomStateRowsToStreamEvents(rows) internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") if err != nil { return nil, err @@ -388,6 +383,35 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( return res, nil } +func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { + var events []types.StreamEvent + for rows.Next() { + var ( + eventID string + streamPos types.StreamPosition + eventBytes []byte + historyVisibility gomatrixserverlib.HistoryVisibility + ) + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &historyVisibility); err != nil { + return nil, err + } + // TODO: Handle redacted events + var ev gomatrixserverlib.HeaderedEvent + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { + return nil, err + } + + ev.Visibility = historyVisibility + + events = append(events, types.StreamEvent{ + HeaderedEvent: &ev, + StreamPosition: streamPos, + }) + } + + return events, nil +} + func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) { result := []*gomatrixserverlib.HeaderedEvent{} for rows.Next() { diff --git a/syncapi/storage/tables/current_room_state_test.go b/syncapi/storage/tables/current_room_state_test.go new file mode 100644 index 000000000..23287c500 --- /dev/null +++ b/syncapi/storage/tables/current_room_state_test.go @@ -0,0 +1,88 @@ +package tables_test + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage/postgres" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/test" +) + +func newCurrentRoomStateTable(t *testing.T, dbType test.DBType) (tables.CurrentRoomState, *sql.DB, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + if err != nil { + t.Fatalf("failed to open db: %s", err) + } + + var tab tables.CurrentRoomState + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresCurrentRoomStateTable(db) + case test.DBTypeSQLite: + var stream sqlite3.StreamIDStatements + if err = stream.Prepare(db); err != nil { + t.Fatalf("failed to prepare stream stmts: %s", err) + } + tab, err = sqlite3.NewSqliteCurrentRoomStateTable(db, &stream) + } + if err != nil { + t.Fatalf("failed to make new table: %s", err) + } + return tab, db, close +} + +func TestCurrentRoomStateTable(t *testing.T) { + ctx := context.Background() + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, db, close := newCurrentRoomStateTable(t, dbType) + defer close() + events := room.CurrentState() + err := sqlutil.WithTransaction(db, func(txn *sql.Tx) error { + for i, ev := range events { + err := tab.UpsertRoomState(ctx, txn, ev, nil, types.StreamPosition(i)) + if err != nil { + return fmt.Errorf("failed to UpsertRoomState: %w", err) + } + } + wantEventIDs := []string{ + events[0].EventID(), events[1].EventID(), events[2].EventID(), events[3].EventID(), + } + gotEvents, err := tab.SelectEventsWithEventIDs(ctx, txn, wantEventIDs) + if err != nil { + return fmt.Errorf("failed to SelectEventsWithEventIDs: %w", err) + } + if len(gotEvents) != len(wantEventIDs) { + return fmt.Errorf("SelectEventsWithEventIDs\ngot %d, want %d results", len(gotEvents), len(wantEventIDs)) + } + gotEventIDs := make(map[string]struct{}, len(gotEvents)) + for _, event := range gotEvents { + if event.ExcludeFromSync { + return fmt.Errorf("SelectEventsWithEventIDs ExcludeFromSync should be false for current room state event %+v", event) + } + gotEventIDs[event.EventID()] = struct{}{} + } + for _, id := range wantEventIDs { + if _, ok := gotEventIDs[id]; !ok { + return fmt.Errorf("SelectEventsWithEventIDs\nexpected id %q not returned", id) + } + } + return nil + }) + if err != nil { + t.Fatalf("err: %v", err) + } + }) +} From fec3ee384bad7f05e13893a97f65590d2284438b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 12:59:56 +0100 Subject: [PATCH 05/61] Stop CPU burn in `PerformMarkAsStaleIfNeeded` --- keyserver/internal/internal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 6309066d7..017c29e84 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -233,7 +233,7 @@ func (a *KeyInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *ap return err } if len(knownDevices) == 0 { - return fmt.Errorf("unknown user %s", req.UserID) + return nil // fmt.Errorf("unknown user %s", req.UserID) } for i := range knownDevices { From 50fa50a3430eed79ddf897d9841fe0b652a32341 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 13:05:58 +0100 Subject: [PATCH 06/61] Update P2P base directories --- cmd/dendrite-demo-pinecone/main.go | 3 +++ cmd/dendrite-demo-yggdrasil/main.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index fb988ebe1..be34365b4 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -143,6 +143,9 @@ func main() { cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true + cfg.MediaAPI.BasePath = config.Path(*instanceDir) + cfg.SyncAPI.Fulltext.Enabled = true + cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir) if err := cfg.Derive(); err != nil { panic(err) } diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index cd0066679..38c25cdec 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -134,6 +134,9 @@ func main() { cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true + cfg.MediaAPI.BasePath = config.Path(*instanceDir) + cfg.SyncAPI.Fulltext.Enabled = true + cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir) if err := cfg.Derive(); err != nil { panic(err) } From 34451d21b8b069e233700dfe4f000f45984be6ea Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 14:35:10 +0100 Subject: [PATCH 07/61] P2P demo tweaks --- build/gobind-pinecone/monolith.go | 38 ++++++++++++++++--------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index b0d3fa285..5e9ff8ef3 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -103,15 +103,15 @@ func (m *DendriteMonolith) SessionCount() int { func (m *DendriteMonolith) RegisterNetworkInterface(name string, index int, mtu int, up bool, broadcast bool, loopback bool, pointToPoint bool, multicast bool, addrs string) { m.PineconeMulticast.RegisterInterface(pineconeMulticast.InterfaceInfo{ - Name: name, - Index: index, - Mtu: mtu, - Up: up, - Broadcast: broadcast, - Loopback: loopback, + Name: name, + Index: index, + Mtu: mtu, + Up: up, + Broadcast: broadcast, + Loopback: loopback, PointToPoint: pointToPoint, - Multicast: multicast, - Addrs: addrs, + Multicast: multicast, + Addrs: addrs, }) } @@ -279,19 +279,21 @@ func (m *DendriteMonolith) Start() { cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.PrivateKey = sk cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) - cfg.Global.JetStream.InMemory = true - cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/%s", m.StorageDirectory, prefix)) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-account.db", m.StorageDirectory, prefix)) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory)) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-syncapi.db", m.StorageDirectory, prefix)) - cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix)) - cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix)) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix)) - cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) - cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) + cfg.Global.JetStream.InMemory = false + cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(m.CacheDirectory, prefix)) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.MediaAPI.BasePath = config.Path(filepath.Join(m.CacheDirectory, "media")) + cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(m.CacheDirectory, "media")) cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true + cfg.SyncAPI.Fulltext.Enabled = true + cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(m.CacheDirectory, "search")) if err = cfg.Derive(); err != nil { panic(err) } From ba66b5a3b9844f4cb3b3f8f2c814894ebb415695 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 14:43:38 +0100 Subject: [PATCH 08/61] Allow multiple static peers in Pinecone iOS/Android demos --- build/gobind-pinecone/monolith.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 5e9ff8ef3..4a96e4bef 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -126,7 +126,9 @@ func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { func (m *DendriteMonolith) SetStaticPeer(uri string) { m.PineconeManager.RemovePeers() - m.PineconeManager.AddPeer(strings.TrimSpace(uri)) + for _, uri := range strings.Split(uri, ",") { + m.PineconeManager.AddPeer(strings.TrimSpace(uri)) + } } func (m *DendriteMonolith) DisconnectType(peertype int) { From 34ed316584df916f6959808669c998580f76d88f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 19:35:26 +0100 Subject: [PATCH 09/61] Fix docs --- docs/administration/1_createusers.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md index 94399a04a..24eba666d 100644 --- a/docs/administration/1_createusers.md +++ b/docs/administration/1_createusers.md @@ -1,4 +1,3 @@ - --- title: Creating user accounts parent: Administration @@ -15,7 +14,7 @@ User accounts can be created on a Dendrite instance in a number of ways. The `create-account` tool is built in the `bin` folder when building Dendrite with the `build.sh` script. -It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires +It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires shared secret registration to be enabled as explained below. An example of using `create-account` to create a **normal account**: @@ -32,7 +31,7 @@ To create a new **admin account**, add the `-admin` flag: ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin ``` -By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using +By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using the `-url` flag: ```bash @@ -44,6 +43,7 @@ An example of using `create-account` when running in **Docker**, having found th ```bash docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME ``` + ```bash docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin ``` From e6c992ba8bcbc59706d6dd55db3b237ebbdc8e7e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 4 Oct 2022 11:41:06 +0200 Subject: [PATCH 10/61] Update Dockerfile (#2342) Updates/adds a new multistage (build-kit) Dockerfile. (if accepted, could make `Dockerfile.monolith` and `Dockerfile.polylith` in `build/docker` obsolete) There's no huge difference between the dockerfiles, except this uses a non-root user when running the container, also doesn't copy the working directory to the image when building. Also adds vulnerabilities scans using [Trivy](https://github.com/aquasecurity/trivy) for the created docker images. (untested) Building images is done using ``` docker build . --target image-monolith -t dendrite-monolith docker build . --target image-polylith -t dendrite-polylith ``` As noted in the comments, only adds `dendrite-polylith-multi` to the polylith image and all required binaries to the monolith image. Probably needs some docs updating, if this is accepted. Co-authored-by: Neil Alexander --- .github/workflows/dendrite.yml | 6 +- .github/workflows/docker.yml | 106 +++++++++++++----- Dockerfile | 99 ++++++++++++++++ .../dendritejs-pinecone/jsServer.go | 0 {cmd => build}/dendritejs-pinecone/main.go | 0 .../dendritejs-pinecone/main_noop.go | 0 .../dendritejs-pinecone/main_test.go | 0 build/docker/Dockerfile.demo-pinecone | 25 ----- build/docker/Dockerfile.monolith | 25 ----- build/docker/Dockerfile.polylith | 25 ----- build/docker/README.md | 14 ++- build/docker/crossbuild.sh | 67 +++++++++++ build/docker/images-build.sh | 5 +- 13 files changed, 257 insertions(+), 115 deletions(-) create mode 100644 Dockerfile rename {cmd => build}/dendritejs-pinecone/jsServer.go (100%) rename {cmd => build}/dendritejs-pinecone/main.go (100%) rename {cmd => build}/dendritejs-pinecone/main_noop.go (100%) rename {cmd => build}/dendritejs-pinecone/main_test.go (100%) delete mode 100644 build/docker/Dockerfile.demo-pinecone delete mode 100644 build/docker/Dockerfile.monolith delete mode 100644 build/docker/Dockerfile.polylith create mode 100644 build/docker/crossbuild.sh diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index be3c7c173..524d36039 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -284,7 +284,7 @@ jobs: API: ${{ matrix.api && 1 }} SYTEST_BRANCH: ${{ github.head_ref }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Sytest run: /bootstrap.sh dendrite working-directory: /src @@ -344,8 +344,8 @@ jobs: sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v2 for dendrite - uses: actions/checkout@v2 + - name: Run actions/checkout@v3 for dendrite + uses: actions/checkout@v3 with: path: dendrite diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b4e24e52f..358037c02 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,26 +21,32 @@ jobs: monolith: name: Monolith image runs-on: ubuntu-latest + needs: build-flags permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -49,27 +55,41 @@ jobs: - name: Build main monolith image if: github.ref_name == 'main' id: docker_build_monolith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: "trivy-results.sarif" + - name: Build release monolith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_monolith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -81,26 +101,32 @@ jobs: polylith: name: Polylith image runs-on: ubuntu-latest + needs: build-flags permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -109,27 +135,40 @@ jobs: - name: Build main polylith image if: github.ref_name == 'main' id: docker_build_polylith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: "trivy-results.sarif" + - name: Build release polylith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_polylith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -141,26 +180,32 @@ jobs: demo-pinecone: name: Pinecone demo image runs-on: ubuntu-latest + needs: build-flags permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -169,12 +214,12 @@ jobs: - name: Build main pinecone demo image if: github.ref_name == 'main' id: docker_build_demo_pinecone - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.demo-pinecone + target: demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -184,12 +229,13 @@ jobs: - name: Build release pinecone demo image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_demo_pinecone_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.demo-pinecone + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..bf5831832 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,99 @@ +#syntax=docker/dockerfile:1.2 + +# +# base installs required dependencies and runs go mod download to cache dependencies +# +FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base +RUN apk --update --no-cache add bash build-base curl + +# +# build creates all needed binaries +# +FROM base AS build +WORKDIR /src +ARG TARGETOS +ARG TARGETARCH +ARG FLAGS +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + sh ./build/docker/crossbuild.sh + +# +# The dendrite base image; mainly creates a user and switches to it +# +FROM alpine:latest AS dendrite-base +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/" +LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C." +RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D +USER dendrite +WORKDIR /home/dendrite + +# +# Builds the polylith image and only contains the polylith binary +# +FROM dendrite-base AS polylith +LABEL org.opencontainers.image.title="Dendrite (Polylith)" + +COPY --from=build /out/dendrite-polylith-multi /usr/bin/ + +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] + +# +# Builds the monolith image and contains all required binaries +# +FROM dendrite-base AS monolith +LABEL org.opencontainers.image.title="Dendrite (Monolith)" + +COPY --from=build /out/create-account /usr/bin/create-account +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] +EXPOSE 8008 8448 + +# +# Builds the P2P demo image and contains all required binaries +# +FROM dendrite-base AS demo-pinecone +LABEL org.opencontainers.image.title="Dendrite (P2P Demo)" + +COPY --from=build /out/create-account /usr/bin/create-account +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] +EXPOSE 8008 8448 + +# +# Builds the Complement image, used for integration tests +# +FROM base AS complement +LABEL org.opencontainers.image.title="Dendrite (Complement)" +RUN apk add --no-cache sqlite openssl ca-certificates +COPY --from=build /out/* /usr/bin/ +RUN rm /usr/bin/dendrite-polylith-multi /usr/bin/dendrite-demo* /usr/bin/dendritejs-pinecone + +WORKDIR /dendrite +RUN /usr/bin/generate-keys --private-key matrix_key.pem && \ + mkdir /ca && \ + openssl genrsa -out /ca/ca.key 2048 && \ + openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt + +ENV SERVER_NAME=localhost +ENV API=0 +EXPOSE 8008 8448 + +# At runtime, generate TLS cert based on the CA now mounted at /ca +# At runtime, replace the SERVER_NAME with what we are told +CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \ + /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ + cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ + /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file diff --git a/cmd/dendritejs-pinecone/jsServer.go b/build/dendritejs-pinecone/jsServer.go similarity index 100% rename from cmd/dendritejs-pinecone/jsServer.go rename to build/dendritejs-pinecone/jsServer.go diff --git a/cmd/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go similarity index 100% rename from cmd/dendritejs-pinecone/main.go rename to build/dendritejs-pinecone/main.go diff --git a/cmd/dendritejs-pinecone/main_noop.go b/build/dendritejs-pinecone/main_noop.go similarity index 100% rename from cmd/dendritejs-pinecone/main_noop.go rename to build/dendritejs-pinecone/main_noop.go diff --git a/cmd/dendritejs-pinecone/main_test.go b/build/dendritejs-pinecone/main_test.go similarity index 100% rename from cmd/dendritejs-pinecone/main_test.go rename to build/dendritejs-pinecone/main_test.go diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone deleted file mode 100644 index 133c63c53..000000000 --- a/build/docker/Dockerfile.demo-pinecone +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith deleted file mode 100644 index 3180e9626..000000000 --- a/build/docker/Dockerfile.monolith +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Monolith)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith deleted file mode 100644 index 79f8a5f23..000000000 --- a/build/docker/Dockerfile.polylith +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Polylith)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] diff --git a/build/docker/README.md b/build/docker/README.md index 261519fde..14a9c8594 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -9,11 +9,15 @@ They can be found on Docker Hub: ## Dockerfiles -The `Dockerfile` builds the base image which contains all of the Dendrite -components. The `Dockerfile.component` file takes the given component, as -specified with `--buildarg component=` from the base image and produce -smaller component-specific images, which are substantially smaller and do -not contain the Go toolchain etc. +The `Dockerfile` is a multistage file which can build all three Dendrite +images depending on the supplied `--target`. From the root of the Dendrite +repository, run: + +``` +docker build . --target monolith -t matrixdotorg/dendrite-monolith +docker build . --target polylith -t matrixdotorg/dendrite-monolith +docker build . --target demo-pinecone -t matrixdotorg/dendrite-monolith +``` ## Compose files diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh new file mode 100644 index 000000000..46e5d7e9b --- /dev/null +++ b/build/docker/crossbuild.sh @@ -0,0 +1,67 @@ +#!/bin/sh +set -e + +# In order to cross-compile with the multi-stage Docker builds, we need to +# ensure that the suitable toolchain for cross-compiling is installed. Since +# the images are Alpine-based, we will use musl. Download and install the +# toolchain inside the build container. + +USERARCH=`go env GOARCH` +GOARCH="$TARGETARCH" +GOOS="linux" + +echo "Target arch: $TARGETARCH" +echo "User arch: $USERARCH" + +if [ "$TARGETARCH" != "$USERARCH" ]; then + if [ "$USERARCH" != "amd64" ]; then + echo "Cross-compiling only supported on amd64" + exit 1 + fi + + echo "Cross compile" + case $GOARCH in + arm64) + curl -s https://more.musl.cc/x86_64-linux-musl/aarch64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=aarch64-linux-musl-gcc + ;; + + amd64) + curl -s https://more.musl.cc/x86_64-linux-musl/x86_64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=x86_64-linux-musl-gcc + ;; + + 386) + curl -s https://more.musl.cc/x86_64-linux-musl/i686-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=i686-linux-musl-gcc + ;; + + arm) + curl -s https://more.musl.cc/x86_64-linux-musl/armv7l-linux-musleabihf-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=armv7l-linux-musleabihf-gcc + ;; + + s390x) + curl -s https://more.musl.cc/x86_64-linux-musl/s390x-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=s390x-linux-musl-gcc + ;; + + ppc64le) + curl -s https://more.musl.cc/x86_64-linux-musl/powerpc64le-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=powerpc64le-linux-musl-gcc + ;; + + *) + echo "Unsupported GOARCH=${GOARCH}" + exit 1 + ;; + esac +else + echo "Native compile" +fi + +# Output the go environment just in case it is useful for debugging. +go env + +# Build Dendrite and tools, statically linking them. +CGO_ENABLED=1 go build -v -ldflags="-linkmode external -extldflags -static ${FLAGS}" -trimpath -o /out/ ./cmd/... diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index c2c140685..1a8326151 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,5 +6,6 @@ TAG=${1:-latest} echo "Building tag '${TAG}'" -docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . -docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file +docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG} +docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG} +docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG} \ No newline at end of file From ede4632835929bf93d0cb57df58c06efd94fa7e0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 10:43:58 +0100 Subject: [PATCH 11/61] Fix Docker GHA --- .github/workflows/docker.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 358037c02..c15b6c1a4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,6 @@ jobs: monolith: name: Monolith image runs-on: ubuntu-latest - needs: build-flags permissions: contents: read packages: write @@ -101,7 +100,6 @@ jobs: polylith: name: Polylith image runs-on: ubuntu-latest - needs: build-flags permissions: contents: read packages: write @@ -180,7 +178,6 @@ jobs: demo-pinecone: name: Pinecone demo image runs-on: ubuntu-latest - needs: build-flags permissions: contents: read packages: write From 98b73652e00edb8304aa4a3471aa29151f025297 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 11:07:54 +0100 Subject: [PATCH 12/61] Try to populate `-ldflags` in Docker builds --- .github/workflows/docker.yml | 6 +++--- build/docker/crossbuild.sh | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c15b6c1a4..e0db280fa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -35,6 +35,7 @@ jobs: BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) [ ${BRANCH} == "main" ] && BRANCH="" echo "BRANCH=${BRANCH}" >> $GITHUB_ENV + echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -59,7 +60,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" target: monolith platforms: ${{ env.PLATFORMS }} push: true @@ -114,6 +114,7 @@ jobs: BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) [ ${BRANCH} == "main" ] && BRANCH="" echo "BRANCH=${BRANCH}" >> $GITHUB_ENV + echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -138,7 +139,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" target: polylith platforms: ${{ env.PLATFORMS }} push: true @@ -192,6 +192,7 @@ jobs: BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) [ ${BRANCH} == "main" ] && BRANCH="" echo "BRANCH=${BRANCH}" >> $GITHUB_ENV + echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -231,7 +232,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" target: demo-pinecone platforms: ${{ env.PLATFORMS }} push: true diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh index 46e5d7e9b..957893a0a 100644 --- a/build/docker/crossbuild.sh +++ b/build/docker/crossbuild.sh @@ -12,6 +12,7 @@ GOOS="linux" echo "Target arch: $TARGETARCH" echo "User arch: $USERARCH" +echo "Link flags: $FLAGS" if [ "$TARGETARCH" != "$USERARCH" ]; then if [ "$USERARCH" != "amd64" ]; then From 085bf5e28b49948209b464c5b12c02775e6c1d70 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 11:33:05 +0100 Subject: [PATCH 13/61] Revert Docker changes --- .github/workflows/dendrite.yml | 6 +- .github/workflows/docker.yml | 103 +++++------------- Dockerfile | 99 ----------------- build/docker/Dockerfile.demo-pinecone | 25 +++++ build/docker/Dockerfile.monolith | 25 +++++ build/docker/Dockerfile.polylith | 25 +++++ build/docker/README.md | 14 +-- build/docker/crossbuild.sh | 68 ------------ build/docker/images-build.sh | 5 +- .../dendritejs-pinecone/jsServer.go | 0 {build => cmd}/dendritejs-pinecone/main.go | 0 .../dendritejs-pinecone/main_noop.go | 0 .../dendritejs-pinecone/main_test.go | 0 13 files changed, 115 insertions(+), 255 deletions(-) delete mode 100644 Dockerfile create mode 100644 build/docker/Dockerfile.demo-pinecone create mode 100644 build/docker/Dockerfile.monolith create mode 100644 build/docker/Dockerfile.polylith delete mode 100644 build/docker/crossbuild.sh rename {build => cmd}/dendritejs-pinecone/jsServer.go (100%) rename {build => cmd}/dendritejs-pinecone/main.go (100%) rename {build => cmd}/dendritejs-pinecone/main_noop.go (100%) rename {build => cmd}/dendritejs-pinecone/main_test.go (100%) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 524d36039..be3c7c173 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -284,7 +284,7 @@ jobs: API: ${{ matrix.api && 1 }} SYTEST_BRANCH: ${{ github.head_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Run Sytest run: /bootstrap.sh dendrite working-directory: /src @@ -344,8 +344,8 @@ jobs: sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v3 for dendrite - uses: actions/checkout@v3 + - name: Run actions/checkout@v2 for dendrite + uses: actions/checkout@v2 with: path: dendrite diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e0db280fa..b4e24e52f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,27 +26,21 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags + uses: actions/checkout@v2 + - name: Get release tag if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -55,40 +49,27 @@ jobs: - name: Build main monolith image if: github.ref_name == 'main' id: docker_build_monolith - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: monolith + file: ./build/docker/Dockerfile.monolith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - format: "sarif" - output: "trivy-results.sarif" - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v1 - with: - sarif_file: "trivy-results.sarif" - - name: Build release monolith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_monolith_release - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" - target: monolith + file: ./build/docker/Dockerfile.monolith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -105,27 +86,21 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags + uses: actions/checkout@v2 + - name: Get release tag if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -134,39 +109,27 @@ jobs: - name: Build main polylith image if: github.ref_name == 'main' id: docker_build_polylith - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: polylith + file: ./build/docker/Dockerfile.polylith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - format: "sarif" - output: "trivy-results.sarif" - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v1 - with: - sarif_file: "trivy-results.sarif" - - name: Build release polylith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_polylith_release - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: polylith + file: ./build/docker/Dockerfile.polylith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -183,27 +146,21 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags + uses: actions/checkout@v2 + - name: Get release tag if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -212,12 +169,12 @@ jobs: - name: Build main pinecone demo image if: github.ref_name == 'main' id: docker_build_demo_pinecone - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: demo-pinecone + file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -227,12 +184,12 @@ jobs: - name: Build release pinecone demo image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_demo_pinecone_release - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: demo-pinecone + file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index bf5831832..000000000 --- a/Dockerfile +++ /dev/null @@ -1,99 +0,0 @@ -#syntax=docker/dockerfile:1.2 - -# -# base installs required dependencies and runs go mod download to cache dependencies -# -FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base -RUN apk --update --no-cache add bash build-base curl - -# -# build creates all needed binaries -# -FROM base AS build -WORKDIR /src -ARG TARGETOS -ARG TARGETARCH -ARG FLAGS -RUN --mount=target=. \ - --mount=type=cache,target=/root/.cache/go-build \ - sh ./build/docker/crossbuild.sh - -# -# The dendrite base image; mainly creates a user and switches to it -# -FROM alpine:latest AS dendrite-base -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" -LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/" -LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C." -RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D -USER dendrite -WORKDIR /home/dendrite - -# -# Builds the polylith image and only contains the polylith binary -# -FROM dendrite-base AS polylith -LABEL org.opencontainers.image.title="Dendrite (Polylith)" - -COPY --from=build /out/dendrite-polylith-multi /usr/bin/ - -ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] - -# -# Builds the monolith image and contains all required binaries -# -FROM dendrite-base AS monolith -LABEL org.opencontainers.image.title="Dendrite (Monolith)" - -COPY --from=build /out/create-account /usr/bin/create-account -COPY --from=build /out/generate-config /usr/bin/generate-config -COPY --from=build /out/generate-keys /usr/bin/generate-keys -COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server - -ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] -EXPOSE 8008 8448 - -# -# Builds the P2P demo image and contains all required binaries -# -FROM dendrite-base AS demo-pinecone -LABEL org.opencontainers.image.title="Dendrite (P2P Demo)" - -COPY --from=build /out/create-account /usr/bin/create-account -COPY --from=build /out/generate-config /usr/bin/generate-config -COPY --from=build /out/generate-keys /usr/bin/generate-keys -COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] -EXPOSE 8008 8448 - -# -# Builds the Complement image, used for integration tests -# -FROM base AS complement -LABEL org.opencontainers.image.title="Dendrite (Complement)" -RUN apk add --no-cache sqlite openssl ca-certificates -COPY --from=build /out/* /usr/bin/ -RUN rm /usr/bin/dendrite-polylith-multi /usr/bin/dendrite-demo* /usr/bin/dendritejs-pinecone - -WORKDIR /dendrite -RUN /usr/bin/generate-keys --private-key matrix_key.pem && \ - mkdir /ca && \ - openssl genrsa -out /ca/ca.key 2048 && \ - openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt - -ENV SERVER_NAME=localhost -ENV API=0 -EXPOSE 8008 8448 - -# At runtime, generate TLS cert based on the CA now mounted at /ca -# At runtime, replace the SERVER_NAME with what we are told -CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \ - /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ - cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ - /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone new file mode 100644 index 000000000..133c63c53 --- /dev/null +++ b/build/docker/Dockerfile.demo-pinecone @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith new file mode 100644 index 000000000..3180e9626 --- /dev/null +++ b/build/docker/Dockerfile.monolith @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Monolith)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith new file mode 100644 index 000000000..79f8a5f23 --- /dev/null +++ b/build/docker/Dockerfile.polylith @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Polylith)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] diff --git a/build/docker/README.md b/build/docker/README.md index 14a9c8594..261519fde 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -9,15 +9,11 @@ They can be found on Docker Hub: ## Dockerfiles -The `Dockerfile` is a multistage file which can build all three Dendrite -images depending on the supplied `--target`. From the root of the Dendrite -repository, run: - -``` -docker build . --target monolith -t matrixdotorg/dendrite-monolith -docker build . --target polylith -t matrixdotorg/dendrite-monolith -docker build . --target demo-pinecone -t matrixdotorg/dendrite-monolith -``` +The `Dockerfile` builds the base image which contains all of the Dendrite +components. The `Dockerfile.component` file takes the given component, as +specified with `--buildarg component=` from the base image and produce +smaller component-specific images, which are substantially smaller and do +not contain the Go toolchain etc. ## Compose files diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh deleted file mode 100644 index 957893a0a..000000000 --- a/build/docker/crossbuild.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh -set -e - -# In order to cross-compile with the multi-stage Docker builds, we need to -# ensure that the suitable toolchain for cross-compiling is installed. Since -# the images are Alpine-based, we will use musl. Download and install the -# toolchain inside the build container. - -USERARCH=`go env GOARCH` -GOARCH="$TARGETARCH" -GOOS="linux" - -echo "Target arch: $TARGETARCH" -echo "User arch: $USERARCH" -echo "Link flags: $FLAGS" - -if [ "$TARGETARCH" != "$USERARCH" ]; then - if [ "$USERARCH" != "amd64" ]; then - echo "Cross-compiling only supported on amd64" - exit 1 - fi - - echo "Cross compile" - case $GOARCH in - arm64) - curl -s https://more.musl.cc/x86_64-linux-musl/aarch64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=aarch64-linux-musl-gcc - ;; - - amd64) - curl -s https://more.musl.cc/x86_64-linux-musl/x86_64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=x86_64-linux-musl-gcc - ;; - - 386) - curl -s https://more.musl.cc/x86_64-linux-musl/i686-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=i686-linux-musl-gcc - ;; - - arm) - curl -s https://more.musl.cc/x86_64-linux-musl/armv7l-linux-musleabihf-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=armv7l-linux-musleabihf-gcc - ;; - - s390x) - curl -s https://more.musl.cc/x86_64-linux-musl/s390x-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=s390x-linux-musl-gcc - ;; - - ppc64le) - curl -s https://more.musl.cc/x86_64-linux-musl/powerpc64le-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=powerpc64le-linux-musl-gcc - ;; - - *) - echo "Unsupported GOARCH=${GOARCH}" - exit 1 - ;; - esac -else - echo "Native compile" -fi - -# Output the go environment just in case it is useful for debugging. -go env - -# Build Dendrite and tools, statically linking them. -CGO_ENABLED=1 go build -v -ldflags="-linkmode external -extldflags -static ${FLAGS}" -trimpath -o /out/ ./cmd/... diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index 1a8326151..c2c140685 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,6 +6,5 @@ TAG=${1:-latest} echo "Building tag '${TAG}'" -docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG} -docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG} -docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG} \ No newline at end of file +docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . +docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file diff --git a/build/dendritejs-pinecone/jsServer.go b/cmd/dendritejs-pinecone/jsServer.go similarity index 100% rename from build/dendritejs-pinecone/jsServer.go rename to cmd/dendritejs-pinecone/jsServer.go diff --git a/build/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go similarity index 100% rename from build/dendritejs-pinecone/main.go rename to cmd/dendritejs-pinecone/main.go diff --git a/build/dendritejs-pinecone/main_noop.go b/cmd/dendritejs-pinecone/main_noop.go similarity index 100% rename from build/dendritejs-pinecone/main_noop.go rename to cmd/dendritejs-pinecone/main_noop.go diff --git a/build/dendritejs-pinecone/main_test.go b/cmd/dendritejs-pinecone/main_test.go similarity index 100% rename from build/dendritejs-pinecone/main_test.go rename to cmd/dendritejs-pinecone/main_test.go From a767102f8a74031b5d0612d985c7589dfe0e0821 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 11:34:52 +0100 Subject: [PATCH 14/61] Reduce `max_open_conns` in monolith sample config --- dendrite-sample.monolith.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index e41e83d7c..f0fa386d1 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -37,7 +37,7 @@ global: # you must configure the "database" block for each component instead. database: connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable - max_open_conns: 100 + max_open_conns: 90 max_idle_conns: 5 conn_max_lifetime: -1 From 3da182212e86daca1d7019efbd424b2bd38b8b3c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 13:02:41 +0100 Subject: [PATCH 15/61] Track reasons why the process is in a degraded state --- setup/base/base.go | 8 +++++++- setup/jetstream/nats.go | 6 +++--- setup/process/process.go | 35 +++++++++++++++++++++++------------ 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/setup/base/base.go b/setup/base/base.go index 0636c7b8d..2e3a3a195 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "database/sql" + "encoding/json" "fmt" "io" "net" @@ -467,8 +468,13 @@ func (b *BaseDendrite) SetupAndServeHTTP( w.WriteHeader(200) }) b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) { - if b.ProcessContext.IsDegraded() { + if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded { w.WriteHeader(503) + _ = json.NewEncoder(w).Encode(struct { + Warnings []string `json:"warnings"` + }{ + Warnings: reasons, + }) return } w.WriteHeader(200) diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 7409fd6c8..af4eb2949 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -169,9 +169,9 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc // We've managed to add the stream in memory. What's on the // disk will be left alone, but our ability to recover from a // future crash will be limited. Yell about it. - sentry.CaptureException(fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible", namespaced.Name)) - logrus.Warn("Stream is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible") - process.Degraded() + err := fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory", namespaced.Name) + sentry.CaptureException(err) + process.Degraded(err) } } } diff --git a/setup/process/process.go b/setup/process/process.go index 06ef60217..b2d2844a8 100644 --- a/setup/process/process.go +++ b/setup/process/process.go @@ -2,19 +2,18 @@ package process import ( "context" - "fmt" "sync" "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "go.uber.org/atomic" ) type ProcessContext struct { - wg *sync.WaitGroup // used to wait for components to shutdown - ctx context.Context // cancelled when Stop is called - shutdown context.CancelFunc // shut down Dendrite - degraded atomic.Bool + mu sync.RWMutex + wg *sync.WaitGroup // used to wait for components to shutdown + ctx context.Context // cancelled when Stop is called + shutdown context.CancelFunc // shut down Dendrite + degraded map[string]struct{} // reasons why the process is degraded } func NewProcessContext() *ProcessContext { @@ -50,13 +49,25 @@ func (b *ProcessContext) WaitForComponentsToFinish() { b.wg.Wait() } -func (b *ProcessContext) Degraded() { - if b.degraded.CompareAndSwap(false, true) { - logrus.Warn("Dendrite is running in a degraded state") - sentry.CaptureException(fmt.Errorf("Process is running in a degraded state")) +func (b *ProcessContext) Degraded(err error) { + b.mu.Lock() + defer b.mu.Unlock() + if _, ok := b.degraded[err.Error()]; !ok { + logrus.WithError(err).Warn("Dendrite has entered a degraded state") + sentry.CaptureException(err) + b.degraded[err.Error()] = struct{}{} } } -func (b *ProcessContext) IsDegraded() bool { - return b.degraded.Load() +func (b *ProcessContext) IsDegraded() (bool, []string) { + b.mu.RLock() + defer b.mu.RUnlock() + if len(b.degraded) == 0 { + return false, nil + } + reasons := make([]string, 0, len(b.degraded)) + for reason := range b.degraded { + reasons = append(reasons, reason) + } + return true, reasons } From ae10aac456e90dc6a3ea56ca4aceb4a4e5aa8e04 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 15:40:04 +0100 Subject: [PATCH 16/61] Don't perform a federated join after invite if we are already joined to the room (#2762) If we are already joined to the room then it shouldn't matter if you were invited or not, so this looks like a bug. --- roomserver/internal/perform/perform_join.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 43be54beb..167b375b7 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -237,7 +237,7 @@ func (r *Joiner) performJoinRoomByID( // Force a federated join if we're dealing with a pending invite // and we aren't in the room. isInvitePending, inviteSender, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID) - if err == nil && isInvitePending { + if err == nil && !serverInRoom && isInvitePending { _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender) if ierr != nil { return "", "", fmt.Errorf("gomatrixserverlib.SplitID: %w", err) From 21f88819858dec022dec5919d6db5554605a9b8b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 16:43:10 +0100 Subject: [PATCH 17/61] Add indexes that optimise `selectStateInRangeSQL` (#2764) This gets rid of some expensive scans on `add_state_ids` and `remove_state_ids`, turning them into much cheaper and faster index scans instead. --- syncapi/storage/postgres/output_room_events_table.go | 2 ++ syncapi/storage/sqlite3/output_room_events_table.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index cb092150d..b562e6804 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -76,6 +76,8 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_add_state_ids_idx ON syncapi_output_room_events ((add_state_ids IS NOT NULL)); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_remove_state_ids_idx ON syncapi_output_room_events ((remove_state_ids IS NOT NULL)); ` const insertEventSQL = "" + diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 165943027..d6a674b9c 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -55,6 +55,8 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_add_state_ids_idx ON syncapi_output_room_events ((add_state_ids IS NOT NULL)); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_remove_state_ids_idx ON syncapi_output_room_events ((remove_state_ids IS NOT NULL)); ` const insertEventSQL = "" + From e0703522932ab4bf41b23d5229c7a4540b59d830 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 5 Oct 2022 11:14:33 +0200 Subject: [PATCH 18/61] Side effect import bleve analyzer languages (#2763) ... to actually allow different languages. Fixes #2761 Binary size increases by ~1MB. --- internal/fulltext/bleve.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go index da8932f5c..7187861dd 100644 --- a/internal/fulltext/bleve.go +++ b/internal/fulltext/bleve.go @@ -21,6 +21,29 @@ import ( "strings" "github.com/blevesearch/bleve/v2" + + // side effect imports to allow all possible languages + _ "github.com/blevesearch/bleve/v2/analysis/lang/ar" + _ "github.com/blevesearch/bleve/v2/analysis/lang/cjk" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ckb" + _ "github.com/blevesearch/bleve/v2/analysis/lang/da" + _ "github.com/blevesearch/bleve/v2/analysis/lang/de" + _ "github.com/blevesearch/bleve/v2/analysis/lang/en" + _ "github.com/blevesearch/bleve/v2/analysis/lang/es" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fa" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fi" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fr" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hi" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hr" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hu" + _ "github.com/blevesearch/bleve/v2/analysis/lang/it" + _ "github.com/blevesearch/bleve/v2/analysis/lang/nl" + _ "github.com/blevesearch/bleve/v2/analysis/lang/no" + _ "github.com/blevesearch/bleve/v2/analysis/lang/pt" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ro" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ru" + _ "github.com/blevesearch/bleve/v2/analysis/lang/sv" + _ "github.com/blevesearch/bleve/v2/analysis/lang/tr" "github.com/blevesearch/bleve/v2/mapping" "github.com/matrix-org/gomatrixserverlib" From ebd137cf6b2fbd767625dc5289b0bef6d1e51971 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 11:07:17 +0100 Subject: [PATCH 19/61] Check PostgreSQL connection count (#2760) This PR queries PostgreSQL for the `max_connections` and `superuser_reserved_connections` settings and then ensures that Dendrite's `max_open_conns` doesn't exceed the allowed value. This is a really common source of configuration problems and can either result in blocking queries or deadlocks, so it seems reasonable that we complain as loudly as possible when it happens. --- internal/sqlutil/sqlutil.go | 39 +++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/internal/sqlutil/sqlutil.go b/internal/sqlutil/sqlutil.go index 0cdae6d30..789bceeac 100644 --- a/internal/sqlutil/sqlutil.go +++ b/internal/sqlutil/sqlutil.go @@ -2,6 +2,7 @@ package sqlutil import ( "database/sql" + "flag" "fmt" "regexp" @@ -9,6 +10,8 @@ import ( "github.com/sirupsen/logrus" ) +var skipSanityChecks = flag.Bool("skip-db-sanity", false, "Ignore sanity checks on the database connections (NOT RECOMMENDED!)") + // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 @@ -37,15 +40,39 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) return nil, err } if driverName != "sqlite3" { - logrus.WithFields(logrus.Fields{ - "MaxOpenConns": dbProperties.MaxOpenConns(), - "MaxIdleConns": dbProperties.MaxIdleConns(), - "ConnMaxLifetime": dbProperties.ConnMaxLifetime(), - "dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), - }).Debug("Setting DB connection limits") + logger := logrus.WithFields(logrus.Fields{ + "max_open_conns": dbProperties.MaxOpenConns(), + "max_idle_conns": dbProperties.MaxIdleConns(), + "conn_max_lifetime": dbProperties.ConnMaxLifetime(), + "data_source_name": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), + }) + logger.Debug("Setting DB connection limits") db.SetMaxOpenConns(dbProperties.MaxOpenConns()) db.SetMaxIdleConns(dbProperties.MaxIdleConns()) db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime()) + + if !*skipSanityChecks { + if dbProperties.MaxOpenConns() == 0 { + logrus.Warnf("WARNING: Configuring 'max_open_conns' to be unlimited is not recommended. This can result in bad performance or deadlocks.") + } + + switch driverName { + case "postgres": + // Perform a quick sanity check if possible that we aren't trying to use more database + // connections than PostgreSQL is willing to give us. + var max, reserved int + if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='max_connections';").Scan(&max); err != nil { + return nil, fmt.Errorf("failed to find maximum connections: %w", err) + } + if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='superuser_reserved_connections';").Scan(&reserved); err != nil { + return nil, fmt.Errorf("failed to find reserved connections: %w", err) + } + if configured, allowed := dbProperties.MaxOpenConns(), max-reserved; configured > allowed { + logrus.Errorf("ERROR: The configured 'max_open_conns' is greater than the %d non-superuser connections that PostgreSQL is configured to allow. This can result in bad performance or deadlocks. Please pay close attention to your configured database connection counts. If you REALLY know what you are doing and want to override this error, pass the --skip-db-sanity option to Dendrite.", allowed) + return nil, fmt.Errorf("database sanity checks failed") + } + } + } } return db, nil } From 8c0c3441d88a612ca8e9ba4f83c6ff29ca73f5d0 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 5 Oct 2022 12:12:42 +0200 Subject: [PATCH 20/61] Add `RoomEventType` nats.Header to avoid unneeded unmarshalling (#2765) --- appservice/consumers/roomserver.go | 5 +++++ federationapi/consumers/roomserver.go | 7 +++++++ roomserver/producers/roomevent.go | 13 ++++++------- setup/jetstream/streams.go | 7 ++++--- test/testrig/jetstream.go | 9 ++++----- userapi/consumers/roomserver.go | 7 ++++--- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index d44f32b38..ac68f4bd4 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -101,6 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage( log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs)) events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs)) for _, msg := range msgs { + // Only handle events we care about + receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType)) + if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInviteEvent { + continue + } // Parse out the event JSON var output api.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go index 349b50b05..a42733628 100644 --- a/federationapi/consumers/roomserver.go +++ b/federationapi/consumers/roomserver.go @@ -79,6 +79,13 @@ func (s *OutputRoomEventConsumer) Start() error { // realises that it cannot update the room state using the deltas. func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called + receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType)) + + // Only handle events we care about + if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInboundPeek { + return true + } + // Parse out the event JSON var output api.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go index 987e6c942..9c4521986 100644 --- a/roomserver/producers/roomevent.go +++ b/roomserver/producers/roomevent.go @@ -17,12 +17,13 @@ package producers import ( "encoding/json" - "github.com/matrix-org/dendrite/roomserver/acls" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/jetstream" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" + + "github.com/matrix-org/dendrite/roomserver/acls" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/jetstream" ) var keyContentFields = map[string]string{ @@ -40,10 +41,8 @@ type RoomEventProducer struct { func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.OutputEvent) error { var err error for _, update := range updates { - msg := &nats.Msg{ - Subject: r.Topic, - Header: nats.Header{}, - } + msg := nats.NewMsg(r.Topic) + msg.Header.Set(jetstream.RoomEventType, string(update.Type)) msg.Header.Set(jetstream.RoomID, roomID) msg.Data, err = json.Marshal(update) if err != nil { diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index ee9810dae..590f0cbd9 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -9,9 +9,10 @@ import ( ) const ( - UserID = "user_id" - RoomID = "room_id" - EventID = "event_id" + UserID = "user_id" + RoomID = "room_id" + EventID = "event_id" + RoomEventType = "output_room_event_type" ) var ( diff --git a/test/testrig/jetstream.go b/test/testrig/jetstream.go index 74cf95062..b880eea43 100644 --- a/test/testrig/jetstream.go +++ b/test/testrig/jetstream.go @@ -4,10 +4,11 @@ import ( "encoding/json" "testing" + "github.com/nats-io/nats.go" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" - "github.com/nats-io/nats.go" ) func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Msg) { @@ -21,10 +22,8 @@ func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Ms func NewOutputEventMsg(t *testing.T, base *base.BaseDendrite, roomID string, update api.OutputEvent) *nats.Msg { t.Helper() - msg := &nats.Msg{ - Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), - Header: nats.Header{}, - } + msg := nats.NewMsg(base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) + msg.Header.Set(jetstream.RoomEventType, string(update.Type)) msg.Header.Set(jetstream.RoomID, roomID) var err error msg.Data, err = json.Marshal(update) diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 952de98f7..a12876946 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -72,15 +72,16 @@ func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called + // Only handle events we care about + if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent { + return true + } var output rsapi.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { // If the message was invalid, log it and move on to the next message in the stream log.WithError(err).Errorf("roomserver output log: message parse failure") return true } - if output.Type != rsapi.OutputTypeNewRoomEvent { - return true - } event := output.NewRoomEvent.Event if event == nil { log.Errorf("userapi consumer: expected event") From 6f602bb0969fac6d455de1088bf3980f77a4017f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 11:16:05 +0100 Subject: [PATCH 21/61] Demote `Failed to query device keys for some users` warning to `level=debug` Many of these warnings are due to dead servers and are quite annoying when they fill up the logs. --- keyserver/internal/device_list_update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index fcfcd092d..8b02f3d6c 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -424,7 +424,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam "succeeded": successCount, "failed": len(userIDs) - successCount, "wait_time": waitTime, - }).Warn("Failed to query device keys for some users") + }).Debug("Failed to query device keys for some users") } return waitTime, !allUsersSucceeded } From c85bc3434fc930423e9e27c8bca9b31b5ad7a441 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 12:47:53 +0100 Subject: [PATCH 22/61] Optimise `QuerySharedUsers` so that we can only work on local users (#2766) Otherwise the sync API key change consumer wastes a lot of time trying to wake up the notifiers for non-local users. --- roomserver/api/query.go | 1 + roomserver/internal/query/query.go | 2 +- roomserver/storage/interface.go | 2 +- .../storage/postgres/membership_table.go | 17 +++++++++----- roomserver/storage/shared/storage.go | 4 ++-- .../storage/sqlite3/membership_table.go | 23 +++++++++++-------- roomserver/storage/tables/interface.go | 2 +- .../storage/tables/membership_table_test.go | 2 +- syncapi/consumers/keychange.go | 6 +++-- 9 files changed, 36 insertions(+), 23 deletions(-) diff --git a/roomserver/api/query.go b/roomserver/api/query.go index aa7dc4735..d63c24785 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -278,6 +278,7 @@ type QuerySharedUsersRequest struct { OtherUserIDs []string ExcludeRoomIDs []string IncludeRoomIDs []string + LocalOnly bool } type QuerySharedUsersResponse struct { diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index b41a92e94..ee8e1cfe7 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -799,7 +799,7 @@ func (r *Queryer) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUser } roomIDs = roomIDs[:j] - users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs) + users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs, req.LocalOnly) if err != nil { return err } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 43e8da7bb..11e175f55 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -157,7 +157,7 @@ type Database interface { // If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned. GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) // JoinedUsersSetInRooms returns how many times each of the given users appears across the given rooms. - JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error) + JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error) // GetLocalServerInRoom returns true if we think we're in a given room or false otherwise. GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) // GetServerInRoom returns true if we think a server is in a given room or false otherwise. diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index bd3fd5592..0150534e1 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -68,14 +68,18 @@ CREATE TABLE IF NOT EXISTS roomserver_membership ( var selectJoinedUsersSetForRoomsAndUserSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid = ANY($1) AND target_nid = ANY($2) AND" + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false)" + + " AND room_nid = ANY($2) AND target_nid = ANY($3)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" var selectJoinedUsersSetForRoomsSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid = ANY($1) AND" + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false) " + + " AND room_nid = ANY($2)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" // Insert a row in to membership table so that it can be locked by the @@ -334,6 +338,7 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms( ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, + localOnly bool, ) (map[types.EventStateKeyNID]int, error) { var ( rows *sql.Rows @@ -342,9 +347,9 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms( stmt := sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsStmt) if len(userNIDs) > 0 { stmt = sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsAndUserStmt) - rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs), pq.Array(userNIDs)) + rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs), pq.Array(userNIDs)) } else { - rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs)) + rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs)) } if err != nil { diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 593abbea1..d83a1ff74 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -1280,7 +1280,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu } // JoinedUsersSetInRooms returns a map of how many times the given users appear in the specified rooms. -func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error) { +func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error) { roomNIDs, err := d.RoomsTable.BulkSelectRoomNIDs(ctx, nil, roomIDs) if err != nil { return nil, err @@ -1295,7 +1295,7 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs [ userNIDs = append(userNIDs, nid) nidToUserID[nid] = id } - userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs) + userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs, localOnly) if err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index f3303eb0e..cd149f0ed 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -44,14 +44,18 @@ const membershipSchema = ` var selectJoinedUsersSetForRoomsAndUserSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid IN ($1) AND target_nid IN ($2) AND" + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false)" + + " AND room_nid IN ($2) AND target_nid IN ($3)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" var selectJoinedUsersSetForRoomsSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid IN ($1) AND " + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false)" + + " AND room_nid IN ($2)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" // Insert a row in to membership table so that it can be locked by the @@ -305,8 +309,9 @@ func (s *membershipStatements) SelectRoomsWithMembership( return roomNIDs, nil } -func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error) { - params := make([]interface{}, 0, len(roomNIDs)+len(userNIDs)) +func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) { + params := make([]interface{}, 0, 1+len(roomNIDs)+len(userNIDs)) + params = append(params, localOnly) for _, v := range roomNIDs { params = append(params, v) } @@ -314,10 +319,10 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, params = append(params, v) } - query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1) + query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1) if len(userNIDs) > 0 { - query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1) - query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)), 1) + query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1) + query = strings.Replace(query, "($3)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)+1), 1) } var rows *sql.Rows var err error diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 68d30f994..d7bcc95ab 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -137,7 +137,7 @@ type Membership interface { UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID, forgotten bool) (bool, error) SelectRoomsWithMembership(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, membershipState MembershipState) ([]types.RoomNID, error) // SelectJoinedUsersSetForRooms returns how many times each of the given users appears across the given rooms. - SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error) + SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) SelectKnownUsers(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, searchString string, limit int) ([]string, error) UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error SelectLocalServerInRoom(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID) (bool, error) diff --git a/roomserver/storage/tables/membership_table_test.go b/roomserver/storage/tables/membership_table_test.go index f789ef4ac..c9541d9d2 100644 --- a/roomserver/storage/tables/membership_table_test.go +++ b/roomserver/storage/tables/membership_table_test.go @@ -79,7 +79,7 @@ func TestMembershipTable(t *testing.T) { assert.NoError(t, err) assert.True(t, inRoom) - userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs) + userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs, false) assert.NoError(t, err) assert.Equal(t, 1, len(userJoinedToRooms)) diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go index dc7d9e207..96ebba7ef 100644 --- a/syncapi/consumers/keychange.go +++ b/syncapi/consumers/keychange.go @@ -111,7 +111,8 @@ func (s *OutputKeyChangeEventConsumer) onDeviceKeyMessage(m api.DeviceMessage, d // work out who we need to notify about the new key var queryRes roomserverAPI.QuerySharedUsersResponse err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{ - UserID: output.UserID, + UserID: output.UserID, + LocalOnly: true, }, &queryRes) if err != nil { logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") @@ -135,7 +136,8 @@ func (s *OutputKeyChangeEventConsumer) onCrossSigningMessage(m api.DeviceMessage // work out who we need to notify about the new key var queryRes roomserverAPI.QuerySharedUsersResponse err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{ - UserID: output.UserID, + UserID: output.UserID, + LocalOnly: true, }, &queryRes) if err != nil { logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") From 0f777d421c56e386b4c483233277f2b96c4da3a0 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 5 Oct 2022 14:47:13 +0200 Subject: [PATCH 23/61] Remove empty fields from `/sync` response (#2755) First attempt at removing empty fields from `/sync` responses. Needs https://github.com/matrix-org/sytest/pull/1298 to keep Sytest happy. Co-authored-by: Neil Alexander --- syncapi/internal/keychange_test.go | 33 +++-- syncapi/streams/stream_accountdata.go | 6 +- syncapi/streams/stream_invite.go | 4 +- syncapi/streams/stream_pdu.go | 10 +- syncapi/streams/stream_receipt.go | 5 +- syncapi/streams/stream_typing.go | 9 +- syncapi/types/types.go | 201 ++++++++++++++++++-------- 7 files changed, 178 insertions(+), 90 deletions(-) diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 3b9c8221c..53f3e5a40 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -170,9 +170,12 @@ func joinResponseWithRooms(syncResponse *types.Response, userID string, roomIDs Content: []byte(`{"membership":"join"}`), }, } - - jr := syncResponse.Rooms.Join[roomID] - jr.State.Events = roomEvents + jr, ok := syncResponse.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() + } + jr.Timeline = &types.Timeline{} + jr.State = &types.ClientEvents{Events: roomEvents} syncResponse.Rooms.Join[roomID] = jr } return syncResponse @@ -191,8 +194,11 @@ func leaveResponseWithRooms(syncResponse *types.Response, userID string, roomIDs }, } - lr := syncResponse.Rooms.Leave[roomID] - lr.Timeline.Events = roomEvents + lr, ok := syncResponse.Rooms.Leave[roomID] + if !ok { + lr = types.NewLeaveResponse() + } + lr.Timeline = &types.Timeline{Events: roomEvents} syncResponse.Rooms.Leave[roomID] = lr } return syncResponse @@ -328,9 +334,13 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) { }, } - jr := syncResponse.Rooms.Join[roomID] - jr.State.Events = roomStateEvents - jr.Timeline.Events = roomTimelineEvents + jr, ok := syncResponse.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() + } + + jr.State = &types.ClientEvents{Events: roomStateEvents} + jr.Timeline = &types.Timeline{Events: roomTimelineEvents} syncResponse.Rooms.Join[roomID] = jr rsAPI := &mockRoomserverAPI{ @@ -442,8 +452,11 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) { }, } - lr := syncResponse.Rooms.Leave[roomID] - lr.Timeline.Events = roomEvents + lr, ok := syncResponse.Rooms.Leave[roomID] + if !ok { + lr = types.NewLeaveResponse() + } + lr.Timeline = &types.Timeline{Events: roomEvents} syncResponse.Rooms.Leave[roomID] = lr rsAPI := &mockRoomserverAPI{ diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 3f2f7d134..3593a6563 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -90,9 +90,9 @@ func (p *AccountDataStreamProvider) IncrementalSync( } } else { if roomData, ok := dataRes.RoomAccountData[roomID][dataType]; ok { - joinData := *types.NewJoinResponse() - if existing, ok := req.Response.Rooms.Join[roomID]; ok { - joinData = existing + joinData, ok := req.Response.Rooms.Join[roomID] + if !ok { + joinData = types.NewJoinResponse() } joinData.AccountData.Events = append( joinData.AccountData.Events, diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 17b3b8434..7875ffa35 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -65,7 +65,7 @@ func (p *InviteStreamProvider) IncrementalSync( continue } ir := types.NewInviteResponse(inviteEvent) - req.Response.Rooms.Invite[roomID] = *ir + req.Response.Rooms.Invite[roomID] = ir } // When doing an initial sync, we don't want to add retired invites, as this @@ -87,7 +87,7 @@ func (p *InviteStreamProvider) IncrementalSync( Type: "m.room.member", Content: gomatrixserverlib.RawJSON(`{"membership":"leave"}`), }) - req.Response.Rooms.Leave[roomID] = *lr + req.Response.Rooms.Leave[roomID] = lr } } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index d252265ff..613ac434f 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -106,7 +106,7 @@ func (p *PDUStreamProvider) CompleteSync( } continue } - req.Response.Rooms.Join[roomID] = *jr + req.Response.Rooms.Join[roomID] = jr req.Rooms[roomID] = gomatrixserverlib.Join } @@ -129,7 +129,7 @@ func (p *PDUStreamProvider) CompleteSync( } continue } - req.Response.Rooms.Peek[peek.RoomID] = *jr + req.Response.Rooms.Peek[peek.RoomID] = jr } } @@ -320,7 +320,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // didn't "remove" events, return that the response is limited. jr.Timeline.Limited = limited && len(events) == len(recentEvents) jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Join[delta.RoomID] = *jr + res.Rooms.Join[delta.RoomID] = jr case gomatrixserverlib.Peek: jr := types.NewJoinResponse() @@ -329,7 +329,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = limited jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Peek[delta.RoomID] = *jr + res.Rooms.Peek[delta.RoomID] = jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -342,7 +342,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // didn't "remove" events, return that the response is limited. lr.Timeline.Limited = limited && len(events) == len(recentEvents) lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Leave[delta.RoomID] = *lr + res.Rooms.Leave[delta.RoomID] = lr } return latestPosition, nil diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 8818a5533..76927cc36 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -4,9 +4,10 @@ import ( "context" "encoding/json" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) type ReceiptStreamProvider struct { @@ -76,7 +77,7 @@ func (p *ReceiptStreamProvider) IncrementalSync( continue } - jr := *types.NewJoinResponse() + jr := types.NewJoinResponse() if existing, ok := req.Response.Rooms.Join[roomID]; ok { jr = existing } diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go index a6f7c7a06..84c199b39 100644 --- a/syncapi/streams/stream_typing.go +++ b/syncapi/streams/stream_typing.go @@ -4,10 +4,11 @@ import ( "context" "encoding/json" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) type TypingStreamProvider struct { @@ -35,9 +36,9 @@ func (p *TypingStreamProvider) IncrementalSync( continue } - jr := *types.NewJoinResponse() - if existing, ok := req.Response.Rooms.Join[roomID]; ok { - jr = existing + jr, ok := req.Response.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() } if users, updated := p.EDUCache.GetTypingUsersIfUpdatedAfter( diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 3b85db4a4..b6d340f93 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -327,29 +327,57 @@ type PrevEventRef struct { PrevSender string `json:"prev_sender"` } +type DeviceLists struct { + Changed []string `json:"changed,omitempty"` + Left []string `json:"left,omitempty"` +} + +type RoomsResponse struct { + Join map[string]*JoinResponse `json:"join,omitempty"` + Peek map[string]*JoinResponse `json:"peek,omitempty"` + Invite map[string]*InviteResponse `json:"invite,omitempty"` + Leave map[string]*LeaveResponse `json:"leave,omitempty"` +} + +type ToDeviceResponse struct { + Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"` +} + // Response represents a /sync API response. See https://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-client-r0-sync type Response struct { - NextBatch StreamingToken `json:"next_batch"` - AccountData struct { - Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` - } `json:"account_data,omitempty"` - Presence struct { - Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` - } `json:"presence,omitempty"` - Rooms struct { - Join map[string]JoinResponse `json:"join,omitempty"` - Peek map[string]JoinResponse `json:"peek,omitempty"` - Invite map[string]InviteResponse `json:"invite,omitempty"` - Leave map[string]LeaveResponse `json:"leave,omitempty"` - } `json:"rooms,omitempty"` - ToDevice struct { - Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"` - } `json:"to_device,omitempty"` - DeviceLists struct { - Changed []string `json:"changed,omitempty"` - Left []string `json:"left,omitempty"` - } `json:"device_lists,omitempty"` - DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"` + NextBatch StreamingToken `json:"next_batch"` + AccountData *ClientEvents `json:"account_data,omitempty"` + Presence *ClientEvents `json:"presence,omitempty"` + Rooms *RoomsResponse `json:"rooms,omitempty"` + ToDevice *ToDeviceResponse `json:"to_device,omitempty"` + DeviceLists *DeviceLists `json:"device_lists,omitempty"` + DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"` +} + +func (r Response) MarshalJSON() ([]byte, error) { + type alias Response + a := alias(r) + if r.AccountData != nil && len(r.AccountData.Events) == 0 { + a.AccountData = nil + } + if r.Presence != nil && len(r.Presence.Events) == 0 { + a.Presence = nil + } + if r.DeviceLists != nil { + if len(r.DeviceLists.Left) == 0 && len(r.DeviceLists.Changed) == 0 { + a.DeviceLists = nil + } + } + if r.Rooms != nil { + if len(r.Rooms.Join) == 0 && len(r.Rooms.Peek) == 0 && + len(r.Rooms.Invite) == 0 && len(r.Rooms.Leave) == 0 { + a.Rooms = nil + } + } + if r.ToDevice != nil && len(r.ToDevice.Events) == 0 { + a.ToDevice = nil + } + return json.Marshal(a) } func (r *Response) HasUpdates() bool { @@ -370,18 +398,21 @@ func NewResponse() *Response { res := Response{} // Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section, // so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors. - res.Rooms.Join = map[string]JoinResponse{} - res.Rooms.Peek = map[string]JoinResponse{} - res.Rooms.Invite = map[string]InviteResponse{} - res.Rooms.Leave = map[string]LeaveResponse{} + res.Rooms = &RoomsResponse{ + Join: map[string]*JoinResponse{}, + Peek: map[string]*JoinResponse{}, + Invite: map[string]*InviteResponse{}, + Leave: map[string]*LeaveResponse{}, + } // Also pre-intialise empty slices or else we'll insert 'null' instead of '[]' for the value. // TODO: We really shouldn't have to do all this to coerce encoding/json to Do The Right Thing. We should // really be using our own Marshal/Unmarshal implementations otherwise this may prove to be a CPU bottleneck. // This also applies to NewJoinResponse, NewInviteResponse and NewLeaveResponse. - res.AccountData.Events = []gomatrixserverlib.ClientEvent{} - res.Presence.Events = []gomatrixserverlib.ClientEvent{} - res.ToDevice.Events = []gomatrixserverlib.SendToDeviceEvent{} + res.AccountData = &ClientEvents{} + res.Presence = &ClientEvents{} + res.DeviceLists = &DeviceLists{} + res.ToDevice = &ToDeviceResponse{} res.DeviceListsOTKCount = map[string]int{} return &res @@ -403,38 +434,73 @@ type UnreadNotifications struct { NotificationCount int `json:"notification_count"` } +type ClientEvents struct { + Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` +} + +type Timeline struct { + Events []gomatrixserverlib.ClientEvent `json:"events"` + Limited bool `json:"limited"` + PrevBatch *TopologyToken `json:"prev_batch,omitempty"` +} + +type Summary struct { + Heroes []string `json:"m.heroes,omitempty"` + JoinedMemberCount *int `json:"m.joined_member_count,omitempty"` + InvitedMemberCount *int `json:"m.invited_member_count,omitempty"` +} + // JoinResponse represents a /sync response for a room which is under the 'join' or 'peek' key. type JoinResponse struct { - Summary struct { - Heroes []string `json:"m.heroes,omitempty"` - JoinedMemberCount *int `json:"m.joined_member_count,omitempty"` - InvitedMemberCount *int `json:"m.invited_member_count,omitempty"` - } `json:"summary"` - State struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"state"` - Timeline struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - Limited bool `json:"limited"` - PrevBatch *TopologyToken `json:"prev_batch,omitempty"` - } `json:"timeline"` - Ephemeral struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"ephemeral"` - AccountData struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"account_data"` + Summary *Summary `json:"summary,omitempty"` + State *ClientEvents `json:"state,omitempty"` + Timeline *Timeline `json:"timeline,omitempty"` + Ephemeral *ClientEvents `json:"ephemeral,omitempty"` + AccountData *ClientEvents `json:"account_data,omitempty"` *UnreadNotifications `json:"unread_notifications,omitempty"` } +func (jr JoinResponse) MarshalJSON() ([]byte, error) { + type alias JoinResponse + a := alias(jr) + if jr.State != nil && len(jr.State.Events) == 0 { + a.State = nil + } + if jr.Ephemeral != nil && len(jr.Ephemeral.Events) == 0 { + a.Ephemeral = nil + } + if jr.AccountData != nil && len(jr.AccountData.Events) == 0 { + a.AccountData = nil + } + if jr.Timeline != nil && len(jr.Timeline.Events) == 0 { + a.Timeline = nil + } + if jr.Summary != nil { + var nilPtr int + joinedEmpty := jr.Summary.JoinedMemberCount == nil || jr.Summary.JoinedMemberCount == &nilPtr + invitedEmpty := jr.Summary.InvitedMemberCount == nil || jr.Summary.InvitedMemberCount == &nilPtr + if joinedEmpty && invitedEmpty && len(jr.Summary.Heroes) == 0 { + a.Summary = nil + } + + } + if jr.UnreadNotifications != nil && + jr.UnreadNotifications.NotificationCount == 0 && jr.UnreadNotifications.HighlightCount == 0 { + a.UnreadNotifications = nil + } + return json.Marshal(a) +} + // NewJoinResponse creates an empty response with initialised arrays. func NewJoinResponse() *JoinResponse { - res := JoinResponse{} - res.State.Events = []gomatrixserverlib.ClientEvent{} - res.Timeline.Events = []gomatrixserverlib.ClientEvent{} - res.Ephemeral.Events = []gomatrixserverlib.ClientEvent{} - res.AccountData.Events = []gomatrixserverlib.ClientEvent{} - return &res + return &JoinResponse{ + Summary: &Summary{}, + State: &ClientEvents{}, + Timeline: &Timeline{}, + Ephemeral: &ClientEvents{}, + AccountData: &ClientEvents{}, + UnreadNotifications: &UnreadNotifications{}, + } } // InviteResponse represents a /sync response for a room which is under the 'invite' key. @@ -469,21 +535,28 @@ func NewInviteResponse(event *gomatrixserverlib.HeaderedEvent) *InviteResponse { // LeaveResponse represents a /sync response for a room which is under the 'leave' key. type LeaveResponse struct { - State struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"state"` - Timeline struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - Limited bool `json:"limited"` - PrevBatch *TopologyToken `json:"prev_batch,omitempty"` - } `json:"timeline"` + State *ClientEvents `json:"state,omitempty"` + Timeline *Timeline `json:"timeline,omitempty"` +} + +func (lr LeaveResponse) MarshalJSON() ([]byte, error) { + type alias LeaveResponse + a := alias(lr) + if lr.State != nil && len(lr.State.Events) == 0 { + a.State = nil + } + if lr.Timeline != nil && len(lr.Timeline.Events) == 0 { + a.Timeline = nil + } + return json.Marshal(a) } // NewLeaveResponse creates an empty response with initialised arrays. func NewLeaveResponse() *LeaveResponse { - res := LeaveResponse{} - res.State.Events = []gomatrixserverlib.ClientEvent{} - res.Timeline.Events = []gomatrixserverlib.ClientEvent{} + res := LeaveResponse{ + State: &ClientEvents{}, + Timeline: &Timeline{}, + } return &res } From 9ba3103f8839243ea8529bf1f563afded3e7b591 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 14:54:19 +0100 Subject: [PATCH 24/61] Document database connection limits --- docs/installation/11_optimisation.md | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/installation/11_optimisation.md b/docs/installation/11_optimisation.md index f2f67c947..686ec2eb9 100644 --- a/docs/installation/11_optimisation.md +++ b/docs/installation/11_optimisation.md @@ -11,6 +11,41 @@ permalink: /installation/start/optimisation Now that you have Dendrite running, the following tweaks will improve the reliability and performance of your installation. +## PostgreSQL connection limit + +A PostgreSQL database engine is configured to allow only a certain number of connections. +This is typically controlled by the `max_connections` and `superuser_reserved_connections` +configuration items in `postgresql.conf`. Once these limits are violated, **PostgreSQL will +immediately stop accepting new connections** until some of the existing connections are closed. +This is a common source of misconfiguration and requires particular care. + +If your PostgreSQL `max_connections` is set to `100` and `superuser_reserved_connections` is +set to `3` then you have an effective connection limit of 97 database connections. It is +therefore important to ensure that Dendrite doesn't violate that limit, otherwise database +queries will unexpectedly fail and this will cause problems both within Dendrite and for users. + +If you are also running other software that uses the same PostgreSQL database engine, then you +must also take into account that some connections will be already used by your other software +and therefore will not be available to Dendrite. Check the configuration of any other software +using the same database engine for their configured connection limits and adjust your calculations +accordingly. + +Dendrite has a `max_open_conns` configuration item in each `database` block to control how many +connections it will open to the database. + +**If you are using the `global` database pool** then you only need to configure the +`max_open_conns` setting once in the `global` section. + +**If you are defining a `database` config per component** then you will need to ensure that +the **sum total** of all configured `max_open_conns` to a given database server do not exceed +the connection limit. If you configure a total that adds up to more connections than are available +then this will cause database queries to fail. + +You may wish to raise the `max_connections` limit on your PostgreSQL server to accommodate +additional connections, in which case you should also update the `max_open_conns` in your +Dendrite configuration accordingly. However be aware that this is only advisable on particularly +powerful servers that can handle the concurrent load of additional queries running at one time. + ## File descriptor limit Most platforms have a limit on how many file descriptors a single process can open. All From e53dcb25a9a0258b48fd4f7afddd2c3c54d55b7a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Oct 2022 10:07:13 +0100 Subject: [PATCH 25/61] Tweak logging for federated room joins --- federationapi/internal/perform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 84702f4ce..4cdd3a5eb 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -259,7 +259,7 @@ func (r *FederationInternalAPI) performJoinUsingServer( if err != nil { return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err) } - logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts") + logrus.WithField("room", roomID).Infof("Joined federated room with %d hosts", len(joinedHosts)) if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil { return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err) } From 3f82bceb70050c1233b7de6d87ffa5510596d145 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Oct 2022 10:51:06 +0100 Subject: [PATCH 26/61] Don't try to talk to ourselves when finding missing events --- roomserver/internal/input/input_events.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 01fd62010..d1b6bc73e 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -173,12 +173,15 @@ func (r *Inputer) processRoomEvent( for _, server := range serverRes.ServerNames { servers[server] = struct{}{} } + // Don't try to talk to ourselves. + delete(servers, r.Cfg.Matrix.ServerName) + // Now build up the list of servers. serverRes.ServerNames = serverRes.ServerNames[:0] - if input.Origin != "" { + if input.Origin != "" && input.Origin != r.Cfg.Matrix.ServerName { serverRes.ServerNames = append(serverRes.ServerNames, input.Origin) delete(servers, input.Origin) } - if senderDomain != input.Origin { + if senderDomain != input.Origin && senderDomain != r.Cfg.Matrix.ServerName { serverRes.ServerNames = append(serverRes.ServerNames, senderDomain) delete(servers, senderDomain) } From ec5d1d681d1362f5746c5cb45e93829d6a68aa4d Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 6 Oct 2022 12:30:24 +0200 Subject: [PATCH 27/61] Always return `one_time_key_counts` on `/keys/upload` (#2769) The OTK count is [required](https://spec.matrix.org/v1.4/client-server-api/#post_matrixclientv3keysupload) in responses to `/keys/upload`, so return those. --- clientapi/routing/keys.go | 4 ++-- keyserver/internal/internal.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go index b7a76b47e..5c3681382 100644 --- a/clientapi/routing/keys.go +++ b/clientapi/routing/keys.go @@ -19,11 +19,12 @@ import ( "net/http" "time" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/keyserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/util" ) type uploadKeysRequest struct { @@ -77,7 +78,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devi } } keyCount := make(map[string]int) - // we only return key counts when the client uploads OTKs if len(uploadRes.OneTimeKeyCounts) > 0 { keyCount = uploadRes.OneTimeKeyCounts[0].KeyCount } diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 017c29e84..a0280dff4 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -70,6 +70,11 @@ func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.Perform if len(req.OneTimeKeys) > 0 { a.uploadOneTimeKeys(ctx, req, res) } + otks, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) + if err != nil { + return err + } + res.OneTimeKeyCounts = []api.OneTimeKeysCount{*otks} return nil } From d605d928bce87b381e2f64b8835619d803e67a54 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Oct 2022 11:56:00 +0100 Subject: [PATCH 28/61] Allow specifying old signing keys with the public key and key ID only (#2770) If the private key file is lost, it's often possible to retrieve the public key from another server elsewhere, so we should make it possible to configure it in that way. --- dendrite-sample.monolith.yaml | 7 ++++- dendrite-sample.polylith.yaml | 7 ++++- federationapi/routing/keys.go | 2 +- setup/config/config.go | 48 +++++++++++++++++++++++------------ setup/config/config_global.go | 7 +++-- 5 files changed, 50 insertions(+), 21 deletions(-) diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index f0fa386d1..eadb74a2a 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -18,12 +18,17 @@ global: private_key: matrix_key.pem # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) - # to old signing private keys that were formerly in use on this domain. These + # to old signing keys that were formerly in use on this domain name. These # keys will not be used for federation request or event signing, but will be # provided to any other homeserver that asks when trying to verify old events. old_private_keys: + # If the old private key file is available: # - private_key: old_matrix_key.pem # expired_at: 1601024554498 + # If only the public key (in base64 format) and key ID are known: + # - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM= + # key_id: ed25519:mykeyid + # expired_at: 1601024554498 # How long a remote server can cache our server signing key before requesting it # again. Increasing this number will reduce the number of requests made by other diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index 0ae4cc8fb..aa7e0cc38 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -18,12 +18,17 @@ global: private_key: matrix_key.pem # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) - # to old signing private keys that were formerly in use on this domain. These + # to old signing keys that were formerly in use on this domain name. These # keys will not be used for federation request or event signing, but will be # provided to any other homeserver that asks when trying to verify old events. old_private_keys: + # If the old private key file is available: # - private_key: old_matrix_key.pem # expired_at: 1601024554498 + # If only the public key (in base64 format) and key ID are known: + # - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM= + # key_id: ed25519:mykeyid + # expired_at: 1601024554498 # How long a remote server can cache our server signing key before requesting it # again. Increasing this number will reduce the number of requests made by other diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index b03d4c1d6..8931830f3 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -160,7 +160,7 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver for _, oldVerifyKey := range cfg.Matrix.OldVerifyKeys { keys.OldVerifyKeys[oldVerifyKey.KeyID] = gomatrixserverlib.OldVerifyKey{ VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PrivateKey.Public().(ed25519.PublicKey)), + Key: oldVerifyKey.PublicKey, }, ExpiredTS: oldVerifyKey.ExpiredAt, } diff --git a/setup/config/config.go b/setup/config/config.go index 5a618d671..e99852ec9 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -231,24 +231,40 @@ func loadConfig( return nil, err } - for i, oldPrivateKey := range c.Global.OldVerifyKeys { - var oldPrivateKeyData []byte + for _, key := range c.Global.OldVerifyKeys { + switch { + case key.PrivateKeyPath != "": + var oldPrivateKeyData []byte + oldPrivateKeyPath := absPath(basePath, key.PrivateKeyPath) + oldPrivateKeyData, err = readFile(oldPrivateKeyPath) + if err != nil { + return nil, fmt.Errorf("failed to read %q: %w", oldPrivateKeyPath, err) + } - oldPrivateKeyPath := absPath(basePath, oldPrivateKey.PrivateKeyPath) - oldPrivateKeyData, err = readFile(oldPrivateKeyPath) - if err != nil { - return nil, err + // NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are + // a number of private keys out there with non-compatible symbols in them due + // to lack of validation in Synapse, we won't enforce that for old verify keys. + keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false) + if perr != nil { + return nil, fmt.Errorf("failed to parse %q: %w", oldPrivateKeyPath, perr) + } + + key.KeyID = keyID + key.PrivateKey = privateKey + key.PublicKey = gomatrixserverlib.Base64Bytes(privateKey.Public().(ed25519.PublicKey)) + + case key.KeyID == "": + return nil, fmt.Errorf("'key_id' must be specified if 'public_key' is specified") + + case len(key.PublicKey) == ed25519.PublicKeySize: + continue + + case len(key.PublicKey) > 0: + return nil, fmt.Errorf("the supplied 'public_key' is the wrong length") + + default: + return nil, fmt.Errorf("either specify a 'private_key' path or supply both 'public_key' and 'key_id'") } - - // NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are - // a number of private keys out there with non-compatible symbols in them due - // to lack of validation in Synapse, we won't enforce that for old verify keys. - keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false) - if perr != nil { - return nil, perr - } - - c.Global.OldVerifyKeys[i].KeyID, c.Global.OldVerifyKeys[i].PrivateKey = keyID, privateKey } c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath)) diff --git a/setup/config/config_global.go b/setup/config/config_global.go index acc608dd7..2efae0d5a 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -27,7 +27,7 @@ type Global struct { // Information about old private keys that used to be used to sign requests and // events on this domain. They will not be used but will be advertised to other // servers that ask for them to help verify old events. - OldVerifyKeys []OldVerifyKeys `yaml:"old_private_keys"` + OldVerifyKeys []*OldVerifyKeys `yaml:"old_private_keys"` // How long a remote server can cache our server key for before requesting it again. // Increasing this number will reduce the number of requests made by remote servers @@ -127,8 +127,11 @@ type OldVerifyKeys struct { // The private key itself. PrivateKey ed25519.PrivateKey `yaml:"-"` + // The public key, in case only that part is known. + PublicKey gomatrixserverlib.Base64Bytes `yaml:"public_key"` + // The key ID of the private key. - KeyID gomatrixserverlib.KeyID `yaml:"-"` + KeyID gomatrixserverlib.KeyID `yaml:"key_id"` // When the private key was designed as "expired", as a UNIX timestamp // in millisecond precision. From 8c5b16678475112f5a1c13d616931283421aee05 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 6 Oct 2022 12:57:13 +0200 Subject: [PATCH 29/61] Use the stream positions of the notifier (#2768) Use the stream positions of the notifier, which might have advanced since setting it at the beginning of the loop. This possibly helps in reducing roundtrips to the SyncAPI, just because we didn't fetch the latest data. Also fixes a minor oversight in the receipts stream. --- syncapi/streams/stream_receipt.go | 6 +++--- syncapi/sync/requestpool.go | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 76927cc36..bba911022 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -77,9 +77,9 @@ func (p *ReceiptStreamProvider) IncrementalSync( continue } - jr := types.NewJoinResponse() - if existing, ok := req.Response.Rooms.Join[roomID]; ok { - jr = existing + jr, ok := req.Response.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() } ev := gomatrixserverlib.ClientEvent{ diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index a71d32ab8..29d92b293 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -407,7 +407,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.PDUStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.PDUPosition, currentPos.PDUPosition, + syncReq.Since.PDUPosition, rp.Notifier.CurrentPosition().PDUPosition, ) }, ), @@ -416,7 +416,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.TypingStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.TypingPosition, currentPos.TypingPosition, + syncReq.Since.TypingPosition, rp.Notifier.CurrentPosition().TypingPosition, ) }, ), @@ -425,7 +425,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.ReceiptStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, + syncReq.Since.ReceiptPosition, rp.Notifier.CurrentPosition().ReceiptPosition, ) }, ), @@ -434,7 +434,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.InviteStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.InvitePosition, currentPos.InvitePosition, + syncReq.Since.InvitePosition, rp.Notifier.CurrentPosition().InvitePosition, ) }, ), @@ -443,7 +443,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.SendToDeviceStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, + syncReq.Since.SendToDevicePosition, rp.Notifier.CurrentPosition().SendToDevicePosition, ) }, ), @@ -452,7 +452,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.AccountDataStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, + syncReq.Since.AccountDataPosition, rp.Notifier.CurrentPosition().AccountDataPosition, ) }, ), @@ -461,7 +461,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.NotificationDataStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, + syncReq.Since.NotificationDataPosition, rp.Notifier.CurrentPosition().NotificationDataPosition, ) }, ), @@ -470,7 +470,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.DeviceListStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, + syncReq.Since.DeviceListPosition, rp.Notifier.CurrentPosition().DeviceListPosition, ) }, ), @@ -479,7 +479,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.PresenceStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.PresencePosition, currentPos.PresencePosition, + syncReq.Since.PresencePosition, rp.Notifier.CurrentPosition().PresencePosition, ) }, ), From 453b50e1d3ce4b4906e050fa8ebe7bcc4c881600 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Fri, 7 Oct 2022 07:32:27 +0200 Subject: [PATCH 30/61] Update README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3bb9a2350..dfef11bae 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co # Create an user account (add -admin for an admin user). # Specify the localpart only, e.g. 'alice' for '@alice:domain.com' -$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice +$ ./bin/create-account --config dendrite.yaml --username alice ``` Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`. @@ -90,7 +90,7 @@ We use a script called Are We Synapse Yet which checks Sytest compliance rates. test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse -servers such as matrix.org reasonably well, although there are still some missing features (like Search). +servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs). We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API). @@ -112,6 +112,7 @@ This means Dendrite supports amongst others: - Guests - User Directory - Presence +- Fulltext search ## Contributing From b9d0e9f7ed7ce1f4be72a25c6f5185a6e809f019 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:54:42 +0200 Subject: [PATCH 31/61] Add test for `QueryDeviceMessages` (#2773) Adds tests for `QueryDeviceMessages` and also includes some optimizations to reduce allocations in the DB layer. --- keyserver/internal/internal.go | 8 +- keyserver/internal/internal_test.go | 156 ++++++++++++++++++ .../storage/postgres/device_keys_table.go | 16 +- .../storage/sqlite3/device_keys_table.go | 16 +- 4 files changed, 172 insertions(+), 24 deletions(-) create mode 100644 keyserver/internal/internal_test.go diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index a0280dff4..06fc4987c 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -212,15 +212,13 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query return nil } maxStreamID := int64(0) + // remove deleted devices + var result []api.DeviceMessage for _, m := range msgs { if m.StreamID > maxStreamID { maxStreamID = m.StreamID } - } - // remove deleted devices - var result []api.DeviceMessage - for _, m := range msgs { - if m.KeyJSON == nil { + if m.KeyJSON == nil || len(m.KeyJSON) == 0 { continue } result = append(result, m) diff --git a/keyserver/internal/internal_test.go b/keyserver/internal/internal_test.go new file mode 100644 index 000000000..8a2c9c5d9 --- /dev/null +++ b/keyserver/internal/internal_test.go @@ -0,0 +1,156 @@ +package internal_test + +import ( + "context" + "reflect" + "testing" + + "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/internal" + "github.com/matrix-org/dendrite/keyserver/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" +) + +func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := storage.NewDatabase(nil, &config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }) + if err != nil { + t.Fatalf("failed to create new user db: %v", err) + } + return db, close +} + +func Test_QueryDeviceMessages(t *testing.T) { + alice := test.NewUser(t) + type args struct { + req *api.QueryDeviceMessagesRequest + res *api.QueryDeviceMessagesResponse + } + tests := []struct { + name string + args args + wantErr bool + want *api.QueryDeviceMessagesResponse + }{ + { + name: "no existing keys", + args: args{ + req: &api.QueryDeviceMessagesRequest{ + UserID: "@doesNotExist:localhost", + }, + res: &api.QueryDeviceMessagesResponse{}, + }, + want: &api.QueryDeviceMessagesResponse{}, + }, + { + name: "existing user returns devices", + args: args{ + req: &api.QueryDeviceMessagesRequest{ + UserID: alice.ID, + }, + res: &api.QueryDeviceMessagesResponse{}, + }, + want: &api.QueryDeviceMessagesResponse{ + StreamID: 6, + Devices: []api.DeviceMessage{ + { + Type: api.TypeDeviceKeyUpdate, StreamID: 5, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + DisplayName: "first device", + UserID: alice.ID, + KeyJSON: []byte("ghi"), + }, + }, + { + Type: api.TypeDeviceKeyUpdate, StreamID: 6, DeviceKeys: &api.DeviceKeys{ + DeviceID: "mySecondDevice", + DisplayName: "second device", + UserID: alice.ID, + KeyJSON: []byte("jkl"), + }, // streamID 6 + }, + }, + }, + }, + } + + deviceMessages := []api.DeviceMessage{ + { // not the user we're looking for + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + UserID: "@doesNotExist:localhost", + }, + // streamID 1 for this user + }, + { // empty keyJSON will be ignored + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + UserID: alice.ID, + }, // streamID 1 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + UserID: alice.ID, + KeyJSON: []byte("abc"), + }, // streamID 2 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + UserID: alice.ID, + KeyJSON: []byte("def"), + }, // streamID 3 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + UserID: alice.ID, + KeyJSON: []byte(""), + }, // streamID 4 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + DisplayName: "first device", + UserID: alice.ID, + KeyJSON: []byte("ghi"), + }, // streamID 5 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "mySecondDevice", + UserID: alice.ID, + KeyJSON: []byte("jkl"), + DisplayName: "second device", + }, // streamID 6 + }, + } + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, closeDB := mustCreateDatabase(t, dbType) + defer closeDB() + if err := db.StoreLocalDeviceKeys(ctx, deviceMessages); err != nil { + t.Fatalf("failed to store local devicesKeys") + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &internal.KeyInternalAPI{ + DB: db, + } + if err := a.QueryDeviceMessages(ctx, tt.args.req, tt.args.res); (err != nil) != tt.wantErr { + t.Errorf("QueryDeviceMessages() error = %v, wantErr %v", err, tt.wantErr) + } + got := tt.args.res + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("QueryDeviceMessages(): got:\n%+v, want:\n%+v", got, tt.want) + } + }) + } + }) +} diff --git a/keyserver/storage/postgres/device_keys_table.go b/keyserver/storage/postgres/device_keys_table.go index ccd20cbd6..2aa11c520 100644 --- a/keyserver/storage/postgres/device_keys_table.go +++ b/keyserver/storage/postgres/device_keys_table.go @@ -20,6 +20,7 @@ import ( "time" "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/keyserver/api" @@ -204,20 +205,17 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID deviceIDMap[d] = true } var result []api.DeviceMessage + var displayName sql.NullString for rows.Next() { dk := api.DeviceMessage{ - Type: api.TypeDeviceKeyUpdate, - DeviceKeys: &api.DeviceKeys{}, + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ + UserID: userID, + }, } - dk.UserID = userID - var keyJSON string - var streamID int64 - var displayName sql.NullString - if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil { + if err := rows.Scan(&dk.DeviceID, &dk.KeyJSON, &dk.StreamID, &displayName); err != nil { return nil, err } - dk.KeyJSON = []byte(keyJSON) - dk.StreamID = streamID if displayName.Valid { dk.DisplayName = displayName.String } diff --git a/keyserver/storage/sqlite3/device_keys_table.go b/keyserver/storage/sqlite3/device_keys_table.go index e77b49b35..73768da5b 100644 --- a/keyserver/storage/sqlite3/device_keys_table.go +++ b/keyserver/storage/sqlite3/device_keys_table.go @@ -137,21 +137,17 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID } defer internal.CloseAndLogIfError(ctx, rows, "selectBatchDeviceKeysStmt: rows.close() failed") var result []api.DeviceMessage + var displayName sql.NullString for rows.Next() { dk := api.DeviceMessage{ - Type: api.TypeDeviceKeyUpdate, - DeviceKeys: &api.DeviceKeys{}, + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ + UserID: userID, + }, } - dk.Type = api.TypeDeviceKeyUpdate - dk.UserID = userID - var keyJSON string - var streamID int64 - var displayName sql.NullString - if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil { + if err := rows.Scan(&dk.DeviceID, &dk.KeyJSON, &dk.StreamID, &displayName); err != nil { return nil, err } - dk.KeyJSON = []byte(keyJSON) - dk.StreamID = streamID if displayName.Valid { dk.DisplayName = displayName.String } From 8d8f4689a0c71f79051554e69fb9113592a4b2e4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 7 Oct 2022 12:21:38 +0100 Subject: [PATCH 32/61] tDatabase transaction tweaks in roomserver --- roomserver/storage/shared/storage.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index d83a1ff74..8859571b7 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -598,7 +598,7 @@ func (d *Database) storeEvent( } // First writer is with a database-provided transaction, so that NIDs are assigned // globally outside of the updater context, to help avoid races. - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error { // TODO: Here we should aim to have two different code paths for new rooms // vs existing ones. @@ -964,9 +964,9 @@ func (d *Database) loadRedactionPair( } if isRedactionEvent { - redactedEvent = d.loadEvent(ctx, info.RedactsEventID) + redactedEvent = d.loadEvent(ctx, txn, info.RedactsEventID) } else { - redactionEvent = d.loadEvent(ctx, info.RedactionEventID) + redactionEvent = d.loadEvent(ctx, txn, info.RedactionEventID) } return redactionEvent, redactedEvent, info.Validated, nil @@ -982,15 +982,15 @@ func (d *Database) applyRedactions(events []types.Event) { } // loadEvent loads a single event or returns nil on any problems/missing event -func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event { - nids, err := d.EventNIDs(ctx, []string{eventID}) +func (d *Database) loadEvent(ctx context.Context, txn *sql.Tx, eventID string) *types.Event { + nids, err := d.eventNIDs(ctx, txn, []string{eventID}, NoFilter) if err != nil { return nil } if len(nids) == 0 { return nil } - evs, err := d.Events(ctx, []types.EventNID{nids[eventID]}) + evs, err := d.events(ctx, txn, []types.EventNID{nids[eventID]}) if err != nil { return nil } @@ -1358,7 +1358,7 @@ func (d *Database) ForgetRoom(ctx context.Context, userID, roomID string, forget } return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.MembershipTable.UpdateForgetMembership(ctx, nil, roomNIDs[0], stateKeyNID, forget) + return d.MembershipTable.UpdateForgetMembership(ctx, txn, roomNIDs[0], stateKeyNID, forget) }) } From 1b5460a920d271aa03c798709f2f1b3b710e4e0e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 7 Oct 2022 13:42:35 +0100 Subject: [PATCH 33/61] Ensure we only wake up a given user once (#2775) This ensures that the sync API notifier only wakes up a given user once for a given stream position. --- syncapi/notifier/notifier.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go index db18c6b77..27f7c37ba 100644 --- a/syncapi/notifier/notifier.go +++ b/syncapi/notifier/notifier.go @@ -48,6 +48,7 @@ type Notifier struct { lastCleanUpTime time.Time // This map is reused to prevent allocations and GC pressure in SharedUsers. _sharedUserMap map[string]struct{} + _wakeupUserMap map[string]struct{} } // NewNotifier creates a new notifier set to the given sync position. @@ -61,6 +62,7 @@ func NewNotifier() *Notifier { lock: &sync.RWMutex{}, lastCleanUpTime: time.Now(), _sharedUserMap: map[string]struct{}{}, + _wakeupUserMap: map[string]struct{}{}, } } @@ -408,12 +410,16 @@ func (n *Notifier) setPeekingDevices(roomIDToPeekingDevices map[string][]types.P // specified user IDs, and also the specified peekingDevices func (n *Notifier) _wakeupUsers(userIDs []string, peekingDevices []types.PeekingDevice, newPos types.StreamingToken) { for _, userID := range userIDs { + n._wakeupUserMap[userID] = struct{}{} + } + for userID := range n._wakeupUserMap { for _, stream := range n._fetchUserStreams(userID) { if stream == nil { continue } stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream } + delete(n._wakeupUserMap, userID) } for _, peekingDevice := range peekingDevices { From 8e231130e97ce716357bbb4af8f82159dc6e684e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 7 Oct 2022 14:05:06 +0100 Subject: [PATCH 34/61] Revert "tDatabase transaction tweaks in roomserver" This reverts commit 8d8f4689a0c71f79051554e69fb9113592a4b2e4. --- roomserver/storage/shared/storage.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 8859571b7..d83a1ff74 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -598,7 +598,7 @@ func (d *Database) storeEvent( } // First writer is with a database-provided transaction, so that NIDs are assigned // globally outside of the updater context, to help avoid races. - err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { // TODO: Here we should aim to have two different code paths for new rooms // vs existing ones. @@ -964,9 +964,9 @@ func (d *Database) loadRedactionPair( } if isRedactionEvent { - redactedEvent = d.loadEvent(ctx, txn, info.RedactsEventID) + redactedEvent = d.loadEvent(ctx, info.RedactsEventID) } else { - redactionEvent = d.loadEvent(ctx, txn, info.RedactionEventID) + redactionEvent = d.loadEvent(ctx, info.RedactionEventID) } return redactionEvent, redactedEvent, info.Validated, nil @@ -982,15 +982,15 @@ func (d *Database) applyRedactions(events []types.Event) { } // loadEvent loads a single event or returns nil on any problems/missing event -func (d *Database) loadEvent(ctx context.Context, txn *sql.Tx, eventID string) *types.Event { - nids, err := d.eventNIDs(ctx, txn, []string{eventID}, NoFilter) +func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event { + nids, err := d.EventNIDs(ctx, []string{eventID}) if err != nil { return nil } if len(nids) == 0 { return nil } - evs, err := d.events(ctx, txn, []types.EventNID{nids[eventID]}) + evs, err := d.Events(ctx, []types.EventNID{nids[eventID]}) if err != nil { return nil } @@ -1358,7 +1358,7 @@ func (d *Database) ForgetRoom(ctx context.Context, userID, roomID string, forget } return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.MembershipTable.UpdateForgetMembership(ctx, txn, roomNIDs[0], stateKeyNID, forget) + return d.MembershipTable.UpdateForgetMembership(ctx, nil, roomNIDs[0], stateKeyNID, forget) }) } From 1ca3f3efb512db4c1827eb571588ec31258782a5 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 7 Oct 2022 16:00:12 +0200 Subject: [PATCH 35/61] Fix issue with DMs shown as normal rooms (#2776) Fixes #2121, test added in https://github.com/matrix-org/complement/pull/494 --- federationapi/api/api.go | 1 + federationapi/internal/perform.go | 19 +++++++++--- roomserver/api/perform.go | 1 + roomserver/internal/helpers/helpers.go | 31 ++++++++++--------- roomserver/internal/perform/perform_join.go | 20 ++++++++++-- roomserver/internal/perform/perform_leave.go | 2 +- roomserver/internal/query/query.go | 2 +- roomserver/storage/interface.go | 5 +-- roomserver/storage/postgres/invite_table.go | 13 ++++---- roomserver/storage/shared/storage.go | 9 +++--- roomserver/storage/sqlite3/invite_table.go | 13 ++++---- roomserver/storage/tables/interface.go | 2 +- .../storage/tables/invite_table_test.go | 11 ++++--- 13 files changed, 82 insertions(+), 47 deletions(-) diff --git a/federationapi/api/api.go b/federationapi/api/api.go index f25538784..362333fc9 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -159,6 +159,7 @@ type PerformJoinRequest struct { // The sorted list of servers to try. Servers will be tried sequentially, after de-duplication. ServerNames types.ServerNames `json:"server_names"` Content map[string]interface{} `json:"content"` + Unsigned map[string]interface{} `json:"unsigned"` } type PerformJoinResponse struct { diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 4cdd3a5eb..28ec48d7b 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -7,14 +7,15 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/federationapi/consumers" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/consumers" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/version" ) // PerformLeaveRequest implements api.FederationInternalAPI @@ -95,6 +96,7 @@ func (r *FederationInternalAPI) PerformJoin( request.Content, serverName, supportedVersions, + request.Unsigned, ); err != nil { logrus.WithError(err).WithFields(logrus.Fields{ "server_name": serverName, @@ -139,6 +141,7 @@ func (r *FederationInternalAPI) performJoinUsingServer( content map[string]interface{}, serverName gomatrixserverlib.ServerName, supportedVersions []gomatrixserverlib.RoomVersion, + unsigned map[string]interface{}, ) error { // Try to perform a make_join using the information supplied in the // request. @@ -267,6 +270,14 @@ func (r *FederationInternalAPI) performJoinUsingServer( // If we successfully performed a send_join above then the other // server now thinks we're a part of the room. Send the newly // returned state to the roomserver to update our local view. + if unsigned != nil { + event, err = event.SetUnsigned(unsigned) + if err != nil { + // non-fatal, log and continue + logrus.WithError(err).Errorf("Failed to set unsigned content") + } + } + if err = roomserverAPI.SendEventWithState( context.Background(), r.rsAPI, diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 20931f807..7a362f969 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -80,6 +80,7 @@ type PerformJoinRequest struct { UserID string `json:"user_id"` Content map[string]interface{} `json:"content"` ServerNames []gomatrixserverlib.ServerName `json:"server_names"` + Unsigned map[string]interface{} `json:"unsigned"` } type PerformJoinResponse struct { diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index cbd1561f7..3b83a0a6d 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -7,6 +7,9 @@ import ( "fmt" "strings" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" @@ -14,8 +17,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) // TODO: temporary package which has helper functions used by both internal/perform packages. @@ -97,35 +98,35 @@ func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverNam func IsInvitePending( ctx context.Context, db storage.Database, roomID, userID string, -) (bool, string, string, error) { +) (bool, string, string, *gomatrixserverlib.Event, error) { // Look up the room NID for the supplied room ID. info, err := db.RoomInfo(ctx, roomID) if err != nil { - return false, "", "", fmt.Errorf("r.DB.RoomInfo: %w", err) + return false, "", "", nil, fmt.Errorf("r.DB.RoomInfo: %w", err) } if info == nil { - return false, "", "", fmt.Errorf("cannot get RoomInfo: unknown room ID %s", roomID) + return false, "", "", nil, fmt.Errorf("cannot get RoomInfo: unknown room ID %s", roomID) } // Look up the state key NID for the supplied user ID. targetUserNIDs, err := db.EventStateKeyNIDs(ctx, []string{userID}) if err != nil { - return false, "", "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err) + return false, "", "", nil, fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err) } targetUserNID, targetUserFound := targetUserNIDs[userID] if !targetUserFound { - return false, "", "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs) + return false, "", "", nil, fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs) } // Let's see if we have an event active for the user in the room. If // we do then it will contain a server name that we can direct the // send_leave to. - senderUserNIDs, eventIDs, err := db.GetInvitesForUser(ctx, info.RoomNID, targetUserNID) + senderUserNIDs, eventIDs, eventJSON, err := db.GetInvitesForUser(ctx, info.RoomNID, targetUserNID) if err != nil { - return false, "", "", fmt.Errorf("r.DB.GetInvitesForUser: %w", err) + return false, "", "", nil, fmt.Errorf("r.DB.GetInvitesForUser: %w", err) } if len(senderUserNIDs) == 0 { - return false, "", "", nil + return false, "", "", nil, nil } userNIDToEventID := make(map[types.EventStateKeyNID]string) for i, nid := range senderUserNIDs { @@ -135,18 +136,20 @@ func IsInvitePending( // Look up the user ID from the NID. senderUsers, err := db.EventStateKeys(ctx, senderUserNIDs) if err != nil { - return false, "", "", fmt.Errorf("r.DB.EventStateKeys: %w", err) + return false, "", "", nil, fmt.Errorf("r.DB.EventStateKeys: %w", err) } if len(senderUsers) == 0 { - return false, "", "", fmt.Errorf("no senderUsers") + return false, "", "", nil, fmt.Errorf("no senderUsers") } senderUser, senderUserFound := senderUsers[senderUserNIDs[0]] if !senderUserFound { - return false, "", "", fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers) + return false, "", "", nil, fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers) } - return true, senderUser, userNIDToEventID[senderUserNIDs[0]], nil + event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false, info.RoomVersion) + + return true, senderUser, userNIDToEventID[senderUserNIDs[0]], event, err } // GetMembershipsAtState filters the state events to diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 167b375b7..262273ff5 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -22,6 +22,10 @@ import ( "time" "github.com/getsentry/sentry-go" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" @@ -32,8 +36,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" ) type Joiner struct { @@ -236,7 +238,7 @@ func (r *Joiner) performJoinRoomByID( // Force a federated join if we're dealing with a pending invite // and we aren't in the room. - isInvitePending, inviteSender, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID) + isInvitePending, inviteSender, _, inviteEvent, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID) if err == nil && !serverInRoom && isInvitePending { _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender) if ierr != nil { @@ -248,6 +250,17 @@ func (r *Joiner) performJoinRoomByID( if inviterDomain != r.Cfg.Matrix.ServerName { req.ServerNames = append(req.ServerNames, inviterDomain) forceFederatedJoin = true + memberEvent := gjson.Parse(string(inviteEvent.JSON())) + // only set unsigned if we've got a content.membership, which we _should_ + if memberEvent.Get("content.membership").Exists() { + req.Unsigned = map[string]interface{}{ + "prev_sender": memberEvent.Get("sender").Str, + "prev_content": map[string]interface{}{ + "is_direct": memberEvent.Get("content.is_direct").Bool(), + "membership": memberEvent.Get("content.membership").Str, + }, + } + } } } @@ -348,6 +361,7 @@ func (r *Joiner) performFederatedJoinRoomByID( UserID: req.UserID, // the user ID joining the room ServerNames: req.ServerNames, // the server to try joining with Content: req.Content, // the membership event content + Unsigned: req.Unsigned, // the unsigned event content, if any } fedRes := fsAPI.PerformJoinResponse{} r.FSAPI.PerformJoin(ctx, &fedReq, &fedRes) diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index ada3aab06..85b659814 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -79,7 +79,7 @@ func (r *Leaver) performLeaveRoomByID( ) ([]api.OutputEvent, error) { // If there's an invite outstanding for the room then respond to // that. - isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID) + isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID) if err == nil && isInvitePending { _, senderDomain, serr := gomatrixserverlib.SplitID('@', senderUser) if serr != nil { diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index ee8e1cfe7..7a424a334 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -872,7 +872,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query // but we don't specify an authorised via user, since the event auth // will allow the join anyway. var pending bool - if pending, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil { + if pending, _, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil { return fmt.Errorf("helpers.IsInvitePending: %w", err) } else if pending { res.Allowed = true diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 11e175f55..ee0624b21 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -17,10 +17,11 @@ package storage import ( "context" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) type Database interface { @@ -104,7 +105,7 @@ type Database interface { // Look up the active invites targeting a user in a room and return the // numeric state key IDs for the user IDs who sent them along with the event IDs for the invites. // Returns an error if there was a problem talking to the database. - GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error) + GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, inviteEventJSON []byte, err error) // Save a given room alias with the room ID it refers to. // Returns an error if there was a problem talking to the database. SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 4cddfe2e9..009fd1ac1 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -61,7 +61,7 @@ const insertInviteEventSQL = "" + " ON CONFLICT DO NOTHING" const selectInviteActiveForUserInRoomSQL = "" + - "SELECT invite_event_id, sender_nid FROM roomserver_invites" + + "SELECT invite_event_id, sender_nid, invite_event_json FROM roomserver_invites" + " WHERE target_nid = $1 AND room_nid = $2" + " AND NOT retired" @@ -141,25 +141,26 @@ func (s *inviteStatements) UpdateInviteRetired( func (s *inviteStatements) SelectInviteActiveForUserInRoom( ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID, -) ([]types.EventStateKeyNID, []string, error) { +) ([]types.EventStateKeyNID, []string, []byte, error) { stmt := sqlutil.TxStmt(txn, s.selectInviteActiveForUserInRoomStmt) rows, err := stmt.QueryContext( ctx, targetUserNID, roomNID, ) if err != nil { - return nil, nil, err + return nil, nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID var eventIDs []string var inviteEventID string var senderUserNID int64 + var eventJSON []byte for rows.Next() { - if err := rows.Scan(&inviteEventID, &senderUserNID); err != nil { - return nil, nil, err + if err := rows.Scan(&inviteEventID, &senderUserNID, &eventJSON); err != nil { + return nil, nil, nil, err } result = append(result, types.EventStateKeyNID(senderUserNID)) eventIDs = append(eventIDs, inviteEventID) } - return result, eventIDs, rows.Err() + return result, eventIDs, eventJSON, rows.Err() } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index d83a1ff74..e401f17dc 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -7,13 +7,14 @@ import ( "fmt" "sort" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/tidwall/gjson" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/tidwall/gjson" ) // Ideally, when we have both events we should redact the event JSON and forget about the redaction, but we currently @@ -445,7 +446,7 @@ func (d *Database) GetInvitesForUser( ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, -) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error) { +) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, inviteEventJSON []byte, err error) { return d.InvitesTable.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) } diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index e051d63af..ca6e7c511 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -44,7 +44,7 @@ const insertInviteEventSQL = "" + " ON CONFLICT DO NOTHING" const selectInviteActiveForUserInRoomSQL = "" + - "SELECT invite_event_id, sender_nid FROM roomserver_invites" + + "SELECT invite_event_id, sender_nid, invite_event_json FROM roomserver_invites" + " WHERE target_nid = $1 AND room_nid = $2" + " AND NOT retired" @@ -136,25 +136,26 @@ func (s *inviteStatements) UpdateInviteRetired( func (s *inviteStatements) SelectInviteActiveForUserInRoom( ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID, -) ([]types.EventStateKeyNID, []string, error) { +) ([]types.EventStateKeyNID, []string, []byte, error) { stmt := sqlutil.TxStmt(txn, s.selectInviteActiveForUserInRoomStmt) rows, err := stmt.QueryContext( ctx, targetUserNID, roomNID, ) if err != nil { - return nil, nil, err + return nil, nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID var eventIDs []string var eventID string var senderUserNID int64 + var eventJSON []byte for rows.Next() { - if err := rows.Scan(&eventID, &senderUserNID); err != nil { - return nil, nil, err + if err := rows.Scan(&eventID, &senderUserNID, &eventJSON); err != nil { + return nil, nil, nil, err } result = append(result, types.EventStateKeyNID(senderUserNID)) eventIDs = append(eventIDs, eventID) } - return result, eventIDs, nil + return result, eventIDs, eventJSON, nil } diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index d7bcc95ab..8be47855f 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -116,7 +116,7 @@ type Invites interface { InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte) (bool, error) UpdateInviteRetired(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) ([]string, error) // SelectInviteActiveForUserInRoom returns a list of sender state key NIDs and invite event IDs matching those nids. - SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, error) + SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, []byte, error) } type MembershipState int64 diff --git a/roomserver/storage/tables/invite_table_test.go b/roomserver/storage/tables/invite_table_test.go index 8df3faa2d..e3eedbf14 100644 --- a/roomserver/storage/tables/invite_table_test.go +++ b/roomserver/storage/tables/invite_table_test.go @@ -4,6 +4,9 @@ import ( "context" "testing" + "github.com/matrix-org/util" + "github.com/stretchr/testify/assert" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" @@ -11,8 +14,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/util" - "github.com/stretchr/testify/assert" ) func mustCreateInviteTable(t *testing.T, dbType test.DBType) (tables.Invites, func()) { @@ -67,7 +68,7 @@ func TestInviteTable(t *testing.T) { assert.NoError(t, err) assert.True(t, newInvite) - stateKeyNIDs, eventIDs, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) + stateKeyNIDs, eventIDs, _, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) assert.NoError(t, err) assert.Equal(t, []string{eventID1, eventID2}, eventIDs) assert.Equal(t, []types.EventStateKeyNID{2, 2}, stateKeyNIDs) @@ -78,13 +79,13 @@ func TestInviteTable(t *testing.T) { assert.Equal(t, []string{eventID1, eventID2}, retiredEventIDs) // This should now be empty - stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) + stateKeyNIDs, eventIDs, _, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) assert.NoError(t, err) assert.Empty(t, eventIDs) assert.Empty(t, stateKeyNIDs) // Non-existent targetUserNID - stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID) + stateKeyNIDs, eventIDs, _, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID) assert.NoError(t, err) assert.Empty(t, stateKeyNIDs) assert.Empty(t, eventIDs) From f1b8df0f49a8a2f2c3c9c4d421b880bd2b008e08 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 7 Oct 2022 15:55:29 +0100 Subject: [PATCH 36/61] Version 0.10.2 (#2778) Changelog and version bump. --- CHANGES.md | 22 ++++++++++++++++++++++ internal/version.go | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index dbe2ccf02..7552e41e5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,27 @@ # Changelog +## Dendrite 0.10.2 (2022-10-07) + +### Features + +* Dendrite will now fail to start if there is an obvious problem with the configured `max_open_conns` when using PostgreSQL database backends, since this can lead to instability and performance issues + * More information on this is available [in the documentation](https://matrix-org.github.io/dendrite/installation/start/optimisation#postgresql-connection-limit) +* Unnecessary/empty fields will no longer be sent in `/sync` responses +* It is now possible to configure `old_private_keys` from previous Matrix installations on the same domain if only public key is known, to make it easier to expire old keys correctly + * You can configure either just the `private_key` path, or you can supply both the `public_key` and `key_id` + +### Fixes + +* The sync transaction behaviour has been modified further so that errors in one stream should not propagate to other streams unnecessarily +* Rooms should now be classified as DM rooms correctly by passing through `is_direct` and unsigned hints +* A bug which caused marking device lists as stale to consume lots of CPU has been fixed +* Users accepting invites should no longer cause unnecessary federated joins if there are already other local users in the room +* The sync API state range queries have been optimised by adding missing indexes +* It should now be possible to configure non-English languages for full-text search in `search.language` +* The roomserver will no longer attempt to perform federated requests to the local server when trying to fetch missing events +* The `/keys/upload` endpoint will now always return the `one_time_keys_counts`, which may help with E2EE reliability +* The sync API will now retrieve the latest stream position before processing each stream rather than at the beginning of the request, to hopefully reduce the number of round-trips to `/sync` + ## Dendrite 0.10.1 (2022-09-30) ### Features diff --git a/internal/version.go b/internal/version.go index d508517be..56b83f852 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 10 - VersionPatch = 1 + VersionPatch = 2 VersionTag = "" // example: "rc1" ) From 980fa55846811eeff89f116c49b38b085143c64e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 10:39:29 +0100 Subject: [PATCH 37/61] Stronger passwordless account checks (fixes #2780) --- userapi/internal/api.go | 2 ++ userapi/storage/shared/storage.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 591faffd6..2f7795dfe 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -838,6 +838,8 @@ func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.Q return nil case bcrypt.ErrMismatchedHashAndPassword: // user exists, but password doesn't match return nil + case bcrypt.ErrHashTooShort: // user exists, but probably a passwordless account + return nil default: res.Exists = true res.Account = acc diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 3ff299f1b..09eeedc9f 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -75,6 +75,9 @@ func (d *Database) GetAccountByPassword( if err != nil { return nil, err } + if hash == "" { + return nil, bcrypt.ErrHashTooShort + } if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil { return nil, err } From 04bab142901b9dc4479b2cf4e5504fbdd9dd3cf1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 10:45:15 +0100 Subject: [PATCH 38/61] Add regression test for 980fa55846811eeff89f116c49b38b085143c64e --- userapi/userapi_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 31a69793b..984fe8854 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -151,6 +151,33 @@ func TestQueryProfile(t *testing.T) { }) } +// TestPasswordlessLoginFails ensures that a passwordless account cannot +// be logged into using an arbitrary password (effectively a regression test +// for https://github.com/matrix-org/dendrite/issues/2780). +func TestPasswordlessLoginFails(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType) + defer close() + _, err := accountDB.CreateAccount(ctx, "auser", "", "", api.AccountTypeAppService) + if err != nil { + t.Fatalf("failed to make account: %s", err) + } + + userReq := &api.QueryAccountByPasswordRequest{ + Localpart: "auser", + PlaintextPassword: "apassword", + } + userRes := &api.QueryAccountByPasswordResponse{} + if err := userAPI.QueryAccountByPassword(ctx, userReq, userRes); err != nil { + t.Fatal(err) + } + if userRes.Exists || userRes.Account != nil { + t.Fatal("QueryAccountByPassword should not return correctly for a passwordless account") + } + }) +} + func TestLoginToken(t *testing.T) { ctx := context.Background() From b32b6d6e8eb5d0a6c22e9215e2fccf4bdd1db8c3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 11:03:52 +0100 Subject: [PATCH 39/61] Update issue and pull request templates --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 22 ++++++++++++---------- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index 206713e04..49c9a37ef 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -7,24 +7,27 @@ about: Create a report to help us improve ### Background information -- **Dendrite version or git SHA**: -- **Monolith or Polylith?**: -- **SQLite3 or Postgres?**: -- **Running in Docker?**: +- **Dendrite version or git SHA**: +- **Monolith or Polylith?**: +- **SQLite3 or Postgres?**: +- **Running in Docker?**: - **`go version`**: - **Client used (if applicable)**: - ### Description - - **What** is the problem: - - **Who** is affected: - - **How** is this bug manifesting: - - **When** did this first appear: +- **What** is the problem: +- **Who** is affected: +- **How** is this bug manifesting: +- **When** did this first appear: + * [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests. -* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off) +* [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Your Name ` From 80a0ab6246aa095f428430c38b13861406dd5c78 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 11:09:40 +0100 Subject: [PATCH 40/61] Further tweak to the issue template --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index 49c9a37ef..f40c56609 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -8,9 +8,10 @@ about: Create a report to help us improve All bug reports must provide the following background information Text between ### Background information From fb6cb2dbcbeb7cd7546ca4d126394720d215c310 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 11:14:16 +0100 Subject: [PATCH 41/61] Tweak `GetAccountByPassword` more --- clientapi/auth/password.go | 6 ++++++ userapi/storage/shared/storage.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index bcb4ca97b..890b18183 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -68,6 +68,12 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.BadJSON("A username must be supplied."), } } + if len(r.Password) == 0 { + return nil, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.BadJSON("A password must be supplied."), + } + } localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName) if err != nil { return nil, &util.JSONResponse{ diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 09eeedc9f..4e28f7b5a 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -75,7 +75,7 @@ func (d *Database) GetAccountByPassword( if err != nil { return nil, err } - if hash == "" { + if len(hash) == 0 && len(plaintextPassword) > 0 { return nil, bcrypt.ErrHashTooShort } if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil { From 0f09e9d196d3375e38a490881e06668a82fb6c40 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 12:19:16 +0200 Subject: [PATCH 42/61] Move /event to the SyncAPI (#2782) This allows us to apply history visibility without having to recalculate it in the roomserver. Unblocks https://github.com/matrix-org/complement/pull/495, fix missing part of https://github.com/matrix-org/dendrite/issues/617 --- clientapi/routing/getevent.go | 138 ------------------------- clientapi/routing/routing.go | 9 -- docs/caddy/polylith/Caddyfile | 2 +- docs/hiawatha/polylith-sample.conf | 4 +- docs/nginx/polylith-sample.conf | 4 +- syncapi/internal/history_visibility.go | 8 +- syncapi/routing/getevent.go | 102 ++++++++++++++++++ syncapi/routing/routing.go | 10 ++ 8 files changed, 124 insertions(+), 153 deletions(-) delete mode 100644 clientapi/routing/getevent.go create mode 100644 syncapi/routing/getevent.go diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go deleted file mode 100644 index 7f5842800..000000000 --- a/clientapi/routing/getevent.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2019 Alex Chen -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "net/http" - - "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/config" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -type getEventRequest struct { - req *http.Request - device *userapi.Device - roomID string - eventID string - cfg *config.ClientAPI - requestedEvent *gomatrixserverlib.Event -} - -// GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId} -// https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid -func GetEvent( - req *http.Request, - device *userapi.Device, - roomID string, - eventID string, - cfg *config.ClientAPI, - rsAPI api.ClientRoomserverAPI, -) util.JSONResponse { - eventsReq := api.QueryEventsByIDRequest{ - EventIDs: []string{eventID}, - } - var eventsResp api.QueryEventsByIDResponse - err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed") - return jsonerror.InternalServerError() - } - - if len(eventsResp.Events) == 0 { - // Event not found locally - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), - } - } - - requestedEvent := eventsResp.Events[0].Event - - r := getEventRequest{ - req: req, - device: device, - roomID: roomID, - eventID: eventID, - cfg: cfg, - requestedEvent: requestedEvent, - } - - stateReq := api.QueryStateAfterEventsRequest{ - RoomID: r.requestedEvent.RoomID(), - PrevEventIDs: r.requestedEvent.PrevEventIDs(), - StateToFetch: []gomatrixserverlib.StateKeyTuple{{ - EventType: gomatrixserverlib.MRoomMember, - StateKey: device.UserID, - }}, - } - var stateResp api.QueryStateAfterEventsResponse - if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed") - return jsonerror.InternalServerError() - } - - if !stateResp.RoomExists { - util.GetLogger(req.Context()).Errorf("Expected to find room for event %s but failed", r.requestedEvent.EventID()) - return jsonerror.InternalServerError() - } - - if !stateResp.PrevEventsExist { - // Missing some events locally; stateResp.StateEvents unavailable. - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), - } - } - - var appService *config.ApplicationService - if device.AppserviceID != "" { - for _, as := range cfg.Derived.ApplicationServices { - if as.ID == device.AppserviceID { - appService = &as - break - } - } - } - - for _, stateEvent := range stateResp.StateEvents { - if appService != nil { - if !appService.IsInterestedInUserID(*stateEvent.StateKey()) { - continue - } - } else if !stateEvent.StateKeyEquals(device.UserID) { - continue - } - membership, err := stateEvent.Membership() - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("stateEvent.Membership failed") - return jsonerror.InternalServerError() - } - if membership == gomatrixserverlib.Join { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: gomatrixserverlib.ToClientEvent(r.requestedEvent, gomatrixserverlib.FormatAll), - } - } - } - - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), - } -} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 7d1c434c4..f1fa66ca6 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -367,15 +367,6 @@ func Setup( nil, cfg, rsAPI, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) - v3mux.Handle("/rooms/{roomID}/event/{eventID}", - httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI) - }), - ).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile index 244e50e7e..906097e4e 100644 --- a/docs/caddy/polylith/Caddyfile +++ b/docs/caddy/polylith/Caddyfile @@ -55,7 +55,7 @@ matrix.example.com { # Change the end of each reverse_proxy line to the correct # address for your various services. @sync_api { - path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ + path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ } reverse_proxy @sync_api sync_api:8073 diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf index 5ed0cb5ae..036140643 100644 --- a/docs/hiawatha/polylith-sample.conf +++ b/docs/hiawatha/polylith-sample.conf @@ -18,8 +18,10 @@ VirtualHost { # /_matrix/client/.*/user/{userId}/filter/{filterID} # /_matrix/client/.*/keys/changes # /_matrix/client/.*/rooms/{roomId}/messages + # /_matrix/client/.*/rooms/{roomId}/context/{eventID} + # /_matrix/client/.*/rooms/{roomId}/event/{eventID} # to sync_api - ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 600 + ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ http://localhost:8073 600 ReverseProxy = /_matrix/client http://localhost:8071 600 ReverseProxy = /_matrix/federation http://localhost:8072 600 ReverseProxy = /_matrix/key http://localhost:8072 600 diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index 274d75658..345d8a6b4 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -28,8 +28,10 @@ server { # /_matrix/client/.*/user/{userId}/filter/{filterID} # /_matrix/client/.*/keys/changes # /_matrix/client/.*/rooms/{roomId}/messages + # /_matrix/client/.*/rooms/{roomId}/context/{eventID} + # /_matrix/client/.*/rooms/{roomId}/event/{eventID} # to sync_api - location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ { + location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ { proxy_pass http://sync_api:8073; } diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go index bbfe19f4c..71d7ddd15 100644 --- a/syncapi/internal/history_visibility.go +++ b/syncapi/internal/history_visibility.go @@ -19,11 +19,13 @@ import ( "math" "time" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage" ) func init() { @@ -189,7 +191,7 @@ func visibilityForEvents( UserID: userID, }, membershipResp) if err != nil { - return result, err + logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'") } // Create a map from eventID -> eventVisibility diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go new file mode 100644 index 000000000..d2cdc1b5f --- /dev/null +++ b/syncapi/routing/getevent.go @@ -0,0 +1,102 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/internal" + "github.com/matrix-org/dendrite/syncapi/storage" + userapi "github.com/matrix-org/dendrite/userapi/api" +) + +// GetEvent implements +// +// GET /_matrix/client/r0/rooms/{roomId}/event/{eventId} +// +// https://spec.matrix.org/v1.4/client-server-api/#get_matrixclientv3roomsroomideventeventid +func GetEvent( + req *http.Request, + device *userapi.Device, + roomID string, + eventID string, + cfg *config.SyncAPI, + syncDB storage.Database, + rsAPI api.SyncRoomserverAPI, +) util.JSONResponse { + ctx := req.Context() + db, err := syncDB.NewDatabaseTransaction(ctx) + logger := util.GetLogger(ctx).WithFields(logrus.Fields{ + "event_id": eventID, + "room_id": roomID, + }) + if err != nil { + logger.WithError(err).Error("GetEvent: syncDB.NewDatabaseTransaction failed") + return jsonerror.InternalServerError() + } + + events, err := db.Events(ctx, []string{eventID}) + if err != nil { + logger.WithError(err).Error("GetEvent: syncDB.Events failed") + return jsonerror.InternalServerError() + } + + // The requested event does not exist in our database + if len(events) == 0 { + logger.Debugf("GetEvent: requested event doesn't exist locally") + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + // If the request is coming from an appservice, get the user from the request + userID := device.UserID + if asUserID := req.FormValue("user_id"); device.AppserviceID != "" && asUserID != "" { + userID = asUserID + } + + // Apply history visibility to determine if the user is allowed to view the event + events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, userID, "event") + if err != nil { + logger.WithError(err).Error("GetEvent: internal.ApplyHistoryVisibilityFilter failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + + // We only ever expect there to be one event + if len(events) != 1 { + // 0 events -> not allowed to view event; > 1 events -> something that shouldn't happen + logger.WithField("event_count", len(events)).Debug("GetEvent: can't return the requested event") + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: gomatrixserverlib.HeaderedToClientEvent(events[0], gomatrixserverlib.FormatAll), + } +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 8f84a1341..069dee81f 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -60,6 +60,16 @@ func Setup( return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, rsAPI, cfg, srp, lazyLoadCache) })).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/rooms/{roomID}/event/{eventID}", + httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, syncDB, rsAPI) + }), + ).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/user/{userId}/filter", httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) From dcc01162874d2634738080614f02cbbeb6bc598f Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 15:38:00 +0200 Subject: [PATCH 43/61] SyTest List Maintenance --- sytest-blacklist | 1 - sytest-whitelist | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/sytest-blacklist b/sytest-blacklist index 5b2e973a6..634c07cf3 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -47,7 +47,6 @@ Notifications can be viewed with GET /notifications # More flakey -If remote user leaves room we no longer receive device updates Guest users can join guest_access rooms # This will fail in HTTP API mode, so blacklisted for now diff --git a/sytest-whitelist b/sytest-whitelist index 31940b884..9ba9df75d 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -742,3 +742,4 @@ User in private room doesn't appear in user directory User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave +Existing members see new member's presence \ No newline at end of file From 39581af3ba657032fbf66ba3719a1cb334c0519b Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 15:49:56 +0200 Subject: [PATCH 44/61] CI update --- .github/workflows/dendrite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index be3c7c173..f8019b3ea 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -342,7 +342,7 @@ jobs: # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest + go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - name: Run actions/checkout@v2 for dendrite uses: actions/checkout@v2 From b000db81ca889b350663493c27eb58654c75e295 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:36:26 +0200 Subject: [PATCH 45/61] Send E2EE related errors to sentry (#2784) Only sends errors if we're not retrying them in NATS. Not sure if those should be scoped/tagged with something like "E2EE". --- federationapi/consumers/keychange.go | 17 ++++++++++++++--- federationapi/consumers/sendtodevice.go | 14 ++++++++++---- federationapi/queue/queue.go | 3 +++ federationapi/routing/send.go | 4 ++++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index f3314bc98..67dfdc1d3 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -18,6 +18,11 @@ import ( "context" "encoding/json" + "github.com/getsentry/sentry-go" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/types" @@ -26,9 +31,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/gomatrixserverlib" - "github.com/nats-io/nats.go" - "github.com/sirupsen/logrus" ) // KeyChangeConsumer consumes events that originate in key server. @@ -78,6 +80,7 @@ func (t *KeyChangeConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) boo msg := msgs[0] // Guaranteed to exist if onMessage is called var m api.DeviceMessage if err := json.Unmarshal(msg.Data, &m); err != nil { + sentry.CaptureException(err) logrus.WithError(err).Errorf("failed to read device message from key change topic") return true } @@ -105,6 +108,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { // only send key change events which originated from us _, originServerName, err := gomatrixserverlib.SplitID('@', m.UserID) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("Failed to extract domain from key change event") return true } @@ -118,6 +122,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { WantMembership: "join", }, &queryRes) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined rooms for user") return true } @@ -125,6 +130,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { // send this key change to all servers who share rooms with this user. destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in") return true } @@ -147,6 +153,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { Keys: m.KeyJSON, } if edu.Content, err = json.Marshal(event); err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("failed to marshal EDU JSON") return true } @@ -160,6 +167,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { output := m.CrossSigningKeyUpdate _, host, err := gomatrixserverlib.SplitID('@', output.UserID) if err != nil { + sentry.CaptureException(err) logrus.WithError(err).Errorf("fedsender key change consumer: user ID parse failure") return true } @@ -176,12 +184,14 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { WantMembership: "join", }, &queryRes) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user") return true } // send this key change to all servers who share rooms with this user. destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in") return true } @@ -196,6 +206,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { Origin: string(t.serverName), } if edu.Content, err = json.Marshal(output); err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to marshal output, dropping") return true } diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go index ffc1d8894..9aec22a3e 100644 --- a/federationapi/consumers/sendtodevice.go +++ b/federationapi/consumers/sendtodevice.go @@ -18,16 +18,18 @@ import ( "context" "encoding/json" + "github.com/getsentry/sentry-go" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" syncTypes "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/nats-io/nats.go" - log "github.com/sirupsen/logrus" ) // OutputSendToDeviceConsumer consumes events that originate in the clientapi. @@ -76,6 +78,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats sender := msg.Header.Get("sender") _, originServerName, err := gomatrixserverlib.SplitID('@', sender) if err != nil { + sentry.CaptureException(err) log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender") return true } @@ -85,12 +88,14 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats // Extract the send-to-device event from msg. var ote syncTypes.OutputSendToDeviceEvent if err = json.Unmarshal(msg.Data, &ote); err != nil { + sentry.CaptureException(err) log.WithError(err).Errorf("output log: message parse failed (expected send-to-device)") return true } _, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID) if err != nil { + sentry.CaptureException(err) log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination") return true } @@ -116,6 +121,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats }, } if edu.Content, err = json.Marshal(tdm); err != nil { + sentry.CaptureException(err) log.WithError(err).Error("failed to marshal EDU JSON") return true } diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 88664fcf9..8245aa5bd 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -307,11 +308,13 @@ func (oqs *OutgoingQueues) SendEDU( ephemeralJSON, err := json.Marshal(e) if err != nil { + sentry.CaptureException(err) return fmt.Errorf("json.Marshal: %w", err) } nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(ephemeralJSON)) if err != nil { + sentry.CaptureException(err) return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 060af676d..b3bbaa394 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -350,6 +351,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { for deviceID, message := range byUser { // TODO: check that the user and the device actually exist here if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { + sentry.CaptureException(err) util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ "sender": directPayload.Sender, "user_id": userID, @@ -360,6 +362,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { } case gomatrixserverlib.MDeviceListUpdate: if err := t.producer.SendDeviceListUpdate(ctx, e.Content, t.Origin); err != nil { + sentry.CaptureException(err) util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate") } case gomatrixserverlib.MReceipt: @@ -395,6 +398,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { } case types.MSigningKeyUpdate: if err := t.producer.SendSigningKeyUpdate(ctx, e.Content, t.Origin); err != nil { + sentry.CaptureException(err) logrus.WithError(err).Errorf("Failed to process signing key update") } case gomatrixserverlib.MPresence: From 6bf1912525724baa1a8bf5e0bf51524d7cee59ba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 16:54:04 +0100 Subject: [PATCH 46/61] Fix joined hosts with `RewritesState` (#2785) This ensures that the joined hosts in the federation API are correct after the state is rewritten. This might fix some races around the time of joining federated rooms. --- federationapi/storage/shared/storage.go | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/federationapi/storage/shared/storage.go b/federationapi/storage/shared/storage.go index a00d782f1..9e40f311c 100644 --- a/federationapi/storage/shared/storage.go +++ b/federationapi/storage/shared/storage.go @@ -70,27 +70,27 @@ func (d *Database) UpdateRoom( ) (joinedHosts []types.JoinedHost, err error) { err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { if purgeRoomFirst { - // If the event is a create event then we'll delete all of the existing - // data for the room. The only reason that a create event would be replayed - // to us in this way is if we're about to receive the entire room state. if err = d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil { return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err) } - } - - joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID) - if err != nil { - return err - } - - for _, add := range addHosts { - err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName) - if err != nil { + for _, add := range addHosts { + if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil { + return err + } + joinedHosts = append(joinedHosts, add) + } + } else { + if joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID); err != nil { + return err + } + for _, add := range addHosts { + if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil { + return err + } + } + if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil { return err } - } - if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil { - return err } return nil }) From 9ed8ff6b938e0452d122c68b38cf4fedcd74c0bf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 11 Oct 2022 10:48:36 +0100 Subject: [PATCH 47/61] Tweak federation `M_NOT_FOUND` errors --- federationapi/routing/events.go | 6 +++++- federationapi/routing/state.go | 15 ++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index 23796edfa..6168912bd 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -20,6 +20,7 @@ import ( "net/http" "time" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -95,7 +96,10 @@ func fetchEvent(ctx context.Context, rsAPI api.FederationRoomserverAPI, eventID } if len(eventsResponse.Events) == 0 { - return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} + return nil, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Event not found"), + } } return eventsResponse.Events[0].Event, nil diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 5377eb88f..1d08d0a82 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -135,23 +135,24 @@ func getState( return nil, nil, &resErr } - if !response.StateKnown { + switch { + case !response.RoomExists: + return nil, nil, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Room not found"), + } + case !response.StateKnown: return nil, nil, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("State not known"), } - } - if response.IsRejected { + case response.IsRejected: return nil, nil, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Event not found"), } } - if !response.RoomExists { - return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} - } - return response.StateEvents, response.AuthChainEvents, nil } From 3920b9f9b6155db69822d0dbcd36acb1eaa51c34 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 11 Oct 2022 10:58:34 +0100 Subject: [PATCH 48/61] Tweak `GetStateDeltas` behaviour (#2788) Improves the control flow of `GetStateDeltas` for clarity and possibly also fixes a bug where duplicate state delta entries could be inserted with different memberships instead of being correctly overridden by `join`. --- syncapi/storage/shared/storage_sync.go | 44 ++++++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index 0e19d97d2..d5b5b3121 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -360,34 +360,50 @@ func (d *DatabaseTransaction) GetStateDeltas( newlyJoinedRooms := make(map[string]bool, len(state)) for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { - if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" { - if membership == gomatrixserverlib.Join && prevMembership != membership { - // send full room state down instead of a delta + // Look for our membership in the state events and skip over any + // membership events that are not related to us. + membership, prevMembership := getMembershipFromEvent(ev.Event, userID) + if membership == "" { + continue + } + + if membership == gomatrixserverlib.Join { + // If our membership is now join but the previous membership wasn't + // then this is a "join transition", so we'll insert this room. + if prevMembership != membership { + // Get the full room state, as we'll send that down for a newly + // joined room instead of a delta. var s []types.StreamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter) - if err != nil { + if s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter); err != nil { if err == sql.ErrNoRows { continue } return nil, nil, err } + + // Add the information for this room into the state so that + // it will get added with all of the rest of the joined rooms. state[roomID] = s newlyJoinedRooms[roomID] = true - continue // we'll add this room in when we do joined rooms } - deltas = append(deltas, types.StateDelta{ - Membership: membership, - MembershipPos: ev.StreamPosition, - StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), - RoomID: roomID, - }) - break + // We won't add joined rooms into the delta at this point as they + // are added later on. + continue } + + deltas = append(deltas, types.StateDelta{ + Membership: membership, + MembershipPos: ev.StreamPosition, + StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + RoomID: roomID, + }) + break } } - // Add in currently joined rooms + // Finally, add in currently joined rooms, including those from the + // join transitions above. for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, types.StateDelta{ Membership: gomatrixserverlib.Join, From 0a9aebdf011921680e5b0646bf50d7900423aa69 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 11 Oct 2022 12:27:21 +0100 Subject: [PATCH 49/61] Private read receipts (#2789) Implement behaviours for `m.read.private` receipts. --- clientapi/routing/account_data.go | 44 ++++++++++++++--------------- clientapi/routing/receipt.go | 36 ++++++++++++++++++----- clientapi/routing/routing.go | 2 +- federationapi/consumers/receipts.go | 8 ++++++ internal/eventutil/types.go | 5 ++-- syncapi/streams/stream_receipt.go | 4 +++ 6 files changed, 66 insertions(+), 33 deletions(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index b28f0bb1f..4742b1240 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -154,33 +154,31 @@ func SaveReadMarker( return *resErr } - if r.FullyRead == "" { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"), + if r.FullyRead != "" { + data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) } } - data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) - if err != nil { - return jsonerror.InternalServerError() - } - - dataReq := api.InputAccountDataRequest{ - UserID: device.UserID, - DataType: "m.fully_read", - RoomID: roomID, - AccountData: data, - } - dataRes := api.InputAccountDataResponse{} - if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") - return util.ErrorResponse(err) - } - - // Handle the read receipt that may be included in the read marker + // Handle the read receipts that may be included in the read marker. if r.Read != "" { - return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read) + return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read", r.Read) + } + if r.ReadPrivate != "" { + return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read.private", r.ReadPrivate) } return util.JSONResponse{ diff --git a/clientapi/routing/receipt.go b/clientapi/routing/receipt.go index 0f9b1b4ff..99217a780 100644 --- a/clientapi/routing/receipt.go +++ b/clientapi/routing/receipt.go @@ -15,19 +15,22 @@ package routing import ( + "encoding/json" "fmt" "net/http" "time" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) -func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { +func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { timestamp := gomatrixserverlib.AsTimestamp(time.Now()) logrus.WithFields(logrus.Fields{ "roomID": roomID, @@ -37,13 +40,32 @@ func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, devi "timestamp": timestamp, }).Debug("Setting receipt") - // currently only m.read is accepted - if receiptType != "m.read" { - return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType)) - } + switch receiptType { + case "m.read", "m.read.private": + if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil { + return util.ErrorResponse(err) + } - if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil { - return util.ErrorResponse(err) + case "m.fully_read": + data, err := json.Marshal(fullyReadEvent{EventID: eventID}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) + } + + default: + return util.MessageResponse(400, fmt.Sprintf("Receipt type '%s' not known", receiptType)) } return util.JSONResponse{ diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f1fa66ca6..e72880ec5 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -1343,7 +1343,7 @@ func Setup( return util.ErrorResponse(err) } - return SetReceipt(req, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"]) + return SetReceipt(req, userAPI, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"]) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/presence/{userId}/status", diff --git a/federationapi/consumers/receipts.go b/federationapi/consumers/receipts.go index 366cb264e..75827cb68 100644 --- a/federationapi/consumers/receipts.go +++ b/federationapi/consumers/receipts.go @@ -81,6 +81,14 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) Type: msg.Header.Get("type"), } + switch receipt.Type { + case "m.read": + // These are allowed to be sent over federation + case "m.read.private", "m.fully_read": + // These must not be sent over federation + return true + } + // only send receipt events which originated from us _, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID) if err != nil { diff --git a/internal/eventutil/types.go b/internal/eventutil/types.go index afc62d8c2..18175d6a0 100644 --- a/internal/eventutil/types.go +++ b/internal/eventutil/types.go @@ -35,8 +35,9 @@ type AccountData struct { } type ReadMarkerJSON struct { - FullyRead string `json:"m.fully_read"` - Read string `json:"m.read"` + FullyRead string `json:"m.fully_read"` + Read string `json:"m.read"` + ReadPrivate string `json:"m.read.private"` } // NotificationData contains statistics about notifications, sent from diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index bba911022..977815078 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -67,6 +67,10 @@ func (p *ReceiptStreamProvider) IncrementalSync( if _, ok := req.IgnoredUsers.List[receipt.UserID]; ok { continue } + // Don't send private read receipts to other users + if receipt.Type == "m.read.private" && req.Device.UserID != receipt.UserID { + continue + } receiptsByRoom[receipt.RoomID] = append(receiptsByRoom[receipt.RoomID], receipt) } From 3c1474f68f2ac5e564d2bd4bd15f01d2a73f2845 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 11 Oct 2022 16:04:02 +0200 Subject: [PATCH 50/61] Fix `/get_missing_events` for rooms with `joined`/`invited` history_visibility (#2787) Sytest was using a wrong `history_visibility` for `invited` (https://github.com/matrix-org/sytest/pull/1303), so `invited` was passing for the wrong reason (-> defaulted to `shared`, as `invite` wasn't understood). This change now handles missing events like Synapse, if a server isn't allowed to see the event, it gets a redacted version of it, making the `get_missing_events` tests pass. --- are-we-synapse-yet.list | 2 +- roomserver/auth/auth.go | 30 +++++++------------ roomserver/internal/helpers/helpers.go | 15 +++++----- .../internal/perform/perform_backfill.go | 5 +++- roomserver/internal/query/query.go | 6 ++-- .../storage/postgres/state_snapshot_table.go | 4 ++- sytest-whitelist | 5 ++-- 7 files changed, 33 insertions(+), 34 deletions(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index c776a7400..81c0f8049 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -643,7 +643,7 @@ fed Inbound federation redacts events from erased users fme Outbound federation can request missing events fme Inbound federation can return missing events for world_readable visibility fme Inbound federation can return missing events for shared visibility -fme Inbound federation can return missing events for invite visibility +fme Inbound federation can return missing events for invited visibility fme Inbound federation can return missing events for joined visibility fme outliers whose auth_events are in a different room are correctly rejected fbk Outbound federation can backfill events diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index aa1d5bc25..31a856e8e 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -13,8 +13,6 @@ package auth import ( - "encoding/json" - "github.com/matrix-org/gomatrixserverlib" ) @@ -30,7 +28,7 @@ func IsServerAllowed( historyVisibility := HistoryVisibilityForRoom(authEvents) // 1. If the history_visibility was set to world_readable, allow. - if historyVisibility == "world_readable" { + if historyVisibility == gomatrixserverlib.HistoryVisibilityWorldReadable { return true } // 2. If the user's membership was join, allow. @@ -39,12 +37,12 @@ func IsServerAllowed( return true } // 3. If history_visibility was set to shared, and the user joined the room at any point after the event was sent, allow. - if historyVisibility == "shared" && serverCurrentlyInRoom { + if historyVisibility == gomatrixserverlib.HistoryVisibilityShared && serverCurrentlyInRoom { return true } // 4. If the user's membership was invite, and the history_visibility was set to invited, allow. invitedUserExists := IsAnyUserOnServerWithMembership(serverName, authEvents, gomatrixserverlib.Invite) - if invitedUserExists && historyVisibility == "invited" { + if invitedUserExists && historyVisibility == gomatrixserverlib.HistoryVisibilityInvited { return true } @@ -52,27 +50,16 @@ func IsServerAllowed( return false } -func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) string { +func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) gomatrixserverlib.HistoryVisibility { // https://matrix.org/docs/spec/client_server/r0.6.0#id87 // By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared. - visibility := "shared" - knownStates := []string{"invited", "joined", "shared", "world_readable"} + visibility := gomatrixserverlib.HistoryVisibilityShared for _, ev := range authEvents { if ev.Type() != gomatrixserverlib.MRoomHistoryVisibility { continue } - // TODO: This should be HistoryVisibilityContent to match things like 'MemberContent'. Do this when moving to GMSL - content := struct { - HistoryVisibility string `json:"history_visibility"` - }{} - if err := json.Unmarshal(ev.Content(), &content); err != nil { - break // value is not understood - } - for _, s := range knownStates { - if s == content.HistoryVisibility { - visibility = s - break - } + if vis, err := ev.HistoryVisibility(); err == nil { + visibility = vis } } return visibility @@ -80,6 +67,9 @@ func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) string { func IsAnyUserOnServerWithMembership(serverName gomatrixserverlib.ServerName, authEvents []*gomatrixserverlib.Event, wantMembership string) bool { for _, ev := range authEvents { + if ev.Type() != gomatrixserverlib.MRoomMember { + continue + } membership, err := ev.Membership() if err != nil || membership != wantMembership { continue diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index 3b83a0a6d..a6de8ac84 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -324,7 +324,7 @@ func slowGetHistoryVisibilityState( func ScanEventTree( ctx context.Context, db storage.Database, info *types.RoomInfo, front []string, visited map[string]bool, limit int, serverName gomatrixserverlib.ServerName, -) ([]types.EventNID, error) { +) ([]types.EventNID, map[string]struct{}, error) { var resultNIDs []types.EventNID var err error var allowed bool @@ -345,6 +345,7 @@ func ScanEventTree( var checkedServerInRoom bool var isServerInRoom bool + redactEventIDs := make(map[string]struct{}) // Loop through the event IDs to retrieve the requested events and go // through the whole tree (up to the provided limit) using the events' @@ -358,7 +359,7 @@ BFSLoop: // Retrieve the events to process from the database. events, err = db.EventsFromIDs(ctx, front) if err != nil { - return resultNIDs, err + return resultNIDs, redactEventIDs, err } if !checkedServerInRoom && len(events) > 0 { @@ -395,16 +396,16 @@ BFSLoop: ) // drop the error, as we will often error at the DB level if we don't have the prev_event itself. Let's // just return what we have. - return resultNIDs, nil + return resultNIDs, redactEventIDs, nil } // If the event hasn't been seen before and the HS // requesting to retrieve it is allowed to do so, add it to // the list of events to retrieve. - if allowed { - next = append(next, pre) - } else { + next = append(next, pre) + if !allowed { util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).Info("Not allowed to see event") + redactEventIDs[pre] = struct{}{} } } } @@ -413,7 +414,7 @@ BFSLoop: front = next } - return resultNIDs, err + return resultNIDs, redactEventIDs, err } func QueryLatestEventsAndState( diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index 69a075733..57e121ea2 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -78,7 +78,7 @@ func (r *Backfiller) PerformBackfill( } // Scan the event tree for events to send back. - resultNIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) + resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) if err != nil { return err } @@ -95,6 +95,9 @@ func (r *Backfiller) PerformBackfill( } for _, event := range loadedEvents { + if _, ok := redactEventIDs[event.EventID()]; ok { + event.Redact() + } response.Events = append(response.Events, event.Headered(info.RoomVersion)) } diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 7a424a334..f41132403 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -453,7 +453,7 @@ func (r *Queryer) QueryMissingEvents( return fmt.Errorf("missing RoomInfo for room %s", events[0].RoomID()) } - resultNIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) + resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) if err != nil { return err } @@ -470,7 +470,9 @@ func (r *Queryer) QueryMissingEvents( if verr != nil { return verr } - + if _, ok := redactEventIDs[event.EventID()]; ok { + event.Redact() + } response.Events = append(response.Events, event.Headered(roomVersion)) } } diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index 99c76befe..a00c026f4 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -21,10 +21,11 @@ import ( "fmt" "github.com/lib/pq" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/util" ) const stateSnapshotSchema = ` @@ -91,6 +92,7 @@ const bulkSelectStateForHistoryVisibilitySQL = ` WHERE state_snapshot_nid = $1 ) ) + ORDER BY depth ASC ) AS roomserver_events INNER JOIN roomserver_event_state_keys ON roomserver_events.event_state_key_nid = roomserver_event_state_keys.event_state_key_nid diff --git a/sytest-whitelist b/sytest-whitelist index 9ba9df75d..a3218ed70 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -306,7 +306,7 @@ Alternative server names do not cause a routing loop Events whose auth_events are in the wrong room do not mess up the room state Inbound federation can return events Inbound federation can return missing events for world_readable visibility -Inbound federation can return missing events for invite visibility +Inbound federation can return missing events for invited visibility Inbound federation can get public room list PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message @@ -742,4 +742,5 @@ User in private room doesn't appear in user directory User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave -Existing members see new member's presence \ No newline at end of file +Existing members see new member's presence +Inbound federation can return missing events for joined visibility \ No newline at end of file From 23a3e04579172de89266e9554428b71172c58495 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 13 Oct 2022 14:50:52 +0100 Subject: [PATCH 51/61] Event relations (#2790) This adds support for tracking `m.relates_to`, as well as adding support for the various `/room/{roomID}/relations/...` endpoints to the CS API. --- docs/caddy/polylith/Caddyfile | 107 ++++++----- docs/hiawatha/polylith-sample.conf | 5 +- docs/nginx/polylith-sample.conf | 5 +- go.mod | 2 +- go.sum | 4 +- syncapi/consumers/roomserver.go | 27 +++ syncapi/routing/relations.go | 124 +++++++++++++ syncapi/routing/routing.go | 43 +++++ syncapi/storage/interface.go | 4 + syncapi/storage/postgres/relations_table.go | 158 +++++++++++++++++ syncapi/storage/postgres/syncserver.go | 5 + syncapi/storage/shared/storage_consumer.go | 35 +++- syncapi/storage/shared/storage_sync.go | 81 +++++++++ syncapi/storage/sqlite3/relations_table.go | 163 +++++++++++++++++ syncapi/storage/sqlite3/stream_id_table.go | 8 + syncapi/storage/sqlite3/syncserver.go | 5 + syncapi/storage/tables/interface.go | 19 ++ syncapi/storage/tables/relations_test.go | 186 ++++++++++++++++++++ syncapi/types/types.go | 13 ++ 19 files changed, 943 insertions(+), 51 deletions(-) create mode 100644 syncapi/routing/relations.go create mode 100644 syncapi/storage/postgres/relations_table.go create mode 100644 syncapi/storage/sqlite3/relations_table.go create mode 100644 syncapi/storage/tables/relations_test.go diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile index 906097e4e..8aeb9317f 100644 --- a/docs/caddy/polylith/Caddyfile +++ b/docs/caddy/polylith/Caddyfile @@ -1,66 +1,85 @@ -# Sample Caddyfile for using Caddy in front of Dendrite. -# -# Customize email address and domain names. -# Optional settings commented out. -# -# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST. -# Documentation: https://caddyserver.com/docs/ -# -# Bonus tip: If your IP address changes, use Caddy's -# dynamic DNS plugin to update your DNS records to -# point to your new IP automatically: -# https://github.com/mholt/caddy-dynamicdns +# Sample Caddyfile for using Caddy in front of Dendrite + # +# Customize email address and domain names + +# Optional settings commented out + +# + +# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST + +# Documentation: + +# + +# Bonus tip: If your IP address changes, use Caddy's + +# dynamic DNS plugin to update your DNS records to + +# point to your new IP automatically + +# + +# # Global options block + { - # In case there is a problem with your certificates. - # email example@example.com + # In case there is a problem with your certificates. + # email example@example.com - # Turn off the admin endpoint if you don't need graceful config - # changes and/or are running untrusted code on your machine. - # admin off + # Turn off the admin endpoint if you don't need graceful config + # changes and/or are running untrusted code on your machine. + # admin off - # Enable this if your clients don't send ServerName in TLS handshakes. - # default_sni example.com + # Enable this if your clients don't send ServerName in TLS handshakes. + # default_sni example.com - # Enable debug mode for verbose logging. - # debug + # Enable debug mode for verbose logging. + # debug - # Use Let's Encrypt's staging endpoint for testing. - # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory + # Use Let's Encrypt's staging endpoint for testing. + # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory - # If you're port-forwarding HTTP/HTTPS ports from 80/443 to something - # else, enable these and put the alternate port numbers here. - # http_port 8080 - # https_port 8443 + # If you're port-forwarding HTTP/HTTPS ports from 80/443 to something + # else, enable these and put the alternate port numbers here. + # http_port 8080 + # https_port 8443 } # The server name of your matrix homeserver. This example shows -# "well-known delegation" from the registered domain to a subdomain, + +# "well-known delegation" from the registered domain to a subdomain + # which is only needed if your server_name doesn't match your Matrix + # homeserver URL (i.e. you can show users a vanity domain that looks + # nice and is easy to remember but still have your Matrix server on -# its own subdomain or hosted service). + +# its own subdomain or hosted service) + example.com { - header /.well-known/matrix/* Content-Type application/json - header /.well-known/matrix/* Access-Control-Allow-Origin * - respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}` - respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}` + header /.well-known/matrix/*Content-Type application/json + header /.well-known/matrix/* Access-Control-Allow-Origin * + respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}` + respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}` } -# The actual domain name whereby your Matrix server is accessed. +# The actual domain name whereby your Matrix server is accessed + matrix.example.com { - # Change the end of each reverse_proxy line to the correct - # address for your various services. - @sync_api { - path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ - } - reverse_proxy @sync_api sync_api:8073 + # Change the end of each reverse_proxy line to the correct + # address for your various services. + @sync_api { + path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ + } + reverse_proxy @sync_api sync_api:8073 - reverse_proxy /_matrix/client* client_api:8071 - reverse_proxy /_matrix/federation* federation_api:8071 - reverse_proxy /_matrix/key* federation_api:8071 - reverse_proxy /_matrix/media* media_api:8071 + reverse_proxy /_matrix/client* client_api:8071 + reverse_proxy /_matrix/federation* federation_api:8071 + reverse_proxy /_matrix/key* federation_api:8071 + reverse_proxy /_matrix/media* media_api:8071 } diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf index 036140643..0093fdcf2 100644 --- a/docs/hiawatha/polylith-sample.conf +++ b/docs/hiawatha/polylith-sample.conf @@ -20,8 +20,11 @@ VirtualHost { # /_matrix/client/.*/rooms/{roomId}/messages # /_matrix/client/.*/rooms/{roomId}/context/{eventID} # /_matrix/client/.*/rooms/{roomId}/event/{eventID} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} # to sync_api - ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ http://localhost:8073 600 + ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ http://localhost:8073 600 ReverseProxy = /_matrix/client http://localhost:8071 600 ReverseProxy = /_matrix/federation http://localhost:8072 600 ReverseProxy = /_matrix/key http://localhost:8072 600 diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index 345d8a6b4..6e81eb5f2 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -30,8 +30,11 @@ server { # /_matrix/client/.*/rooms/{roomId}/messages # /_matrix/client/.*/rooms/{roomId}/context/{eventID} # /_matrix/client/.*/rooms/{roomId}/event/{eventID} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} # to sync_api - location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ { + location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ { proxy_pass http://sync_api:8073; } diff --git a/go.mod b/go.mod index c82f76d41..eefad89e6 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621 + github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64 github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index a99599cb1..0d08ac692 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621 h1:a8IaoSPDxevkgXnOUrtIW9AqVNvXBJAG0gtnX687S7g= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64 h1:QJmfAPC3P0ZHJzYD/QtbNc5EztKlK1ipRWP5SO/m4jw= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index c7a11dbb4..cfbb05327 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -148,6 +148,16 @@ func (s *OutputRoomEventConsumer) onRedactEvent( log.WithError(err).Error("RedactEvent error'd") return err } + + if err = s.db.RedactRelations(ctx, msg.RedactedBecause.RoomID(), msg.RedactedEventID); err != nil { + log.WithFields(log.Fields{ + "room_id": msg.RedactedBecause.RoomID(), + "event_id": msg.RedactedBecause.EventID(), + "redacted_event_id": msg.RedactedEventID, + }).WithError(err).Warn("Failed to redact relations") + return err + } + // fake a room event so we notify clients about the redaction, as if it were // a normal event. return s.onNewRoomEvent(ctx, api.OutputNewRoomEvent{ @@ -271,6 +281,14 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( return err } + if err = s.db.UpdateRelations(ctx, ev); err != nil { + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "type": ev.Type(), + }).WithError(err).Warn("Failed to update relations") + return err + } + s.pduStream.Advance(pduPos) s.notifier.OnNewEvent(ev, ev.RoomID(), nil, types.StreamingToken{PDUPosition: pduPos}) @@ -315,6 +333,15 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( }).WithError(err).Warn("failed to index fulltext element") } + if err = s.db.UpdateRelations(ctx, ev); err != nil { + log.WithFields(log.Fields{ + "room_id": ev.RoomID(), + "event_id": ev.EventID(), + "type": ev.Type(), + }).WithError(err).Warn("Failed to update relations") + return err + } + if pduPos, err = s.notifyJoinedPeeks(ctx, ev, pduPos); err != nil { log.WithError(err).Errorf("Failed to notifyJoinedPeeks for PDU pos %d", pduPos) return err diff --git a/syncapi/routing/relations.go b/syncapi/routing/relations.go new file mode 100644 index 000000000..fee61b0df --- /dev/null +++ b/syncapi/routing/relations.go @@ -0,0 +1,124 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + "strconv" + + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/internal" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" +) + +type RelationsResponse struct { + Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` + NextBatch string `json:"next_batch,omitempty"` + PrevBatch string `json:"prev_batch,omitempty"` +} + +// nolint:gocyclo +func Relations( + req *http.Request, device *userapi.Device, + syncDB storage.Database, + rsAPI api.SyncRoomserverAPI, + roomID, eventID, relType, eventType string, +) util.JSONResponse { + var err error + var from, to types.StreamPosition + var limit int + dir := req.URL.Query().Get("dir") + if f := req.URL.Query().Get("from"); f != "" { + if from, err = types.NewStreamPositionFromString(f); err != nil { + return util.ErrorResponse(err) + } + } + if t := req.URL.Query().Get("to"); t != "" { + if to, err = types.NewStreamPositionFromString(t); err != nil { + return util.ErrorResponse(err) + } + } + if l := req.URL.Query().Get("limit"); l != "" { + if limit, err = strconv.Atoi(l); err != nil { + return util.ErrorResponse(err) + } + } + if limit == 0 || limit > 50 { + limit = 50 + } + if dir == "" { + dir = "b" + } + if dir != "b" && dir != "f" { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Bad or missing dir query parameter (should be either 'b' or 'f')"), + } + } + + snapshot, err := syncDB.NewDatabaseSnapshot(req.Context()) + if err != nil { + logrus.WithError(err).Error("Failed to get snapshot for relations") + return jsonerror.InternalServerError() + } + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) + + res := &RelationsResponse{ + Chunk: []gomatrixserverlib.ClientEvent{}, + } + var events []types.StreamEvent + events, res.PrevBatch, res.NextBatch, err = snapshot.RelationsFor( + req.Context(), roomID, eventID, relType, eventType, from, to, dir == "b", limit, + ) + if err != nil { + return util.ErrorResponse(err) + } + + headeredEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(events)) + for _, event := range events { + headeredEvents = append(headeredEvents, event.HeaderedEvent) + } + + // Apply history visibility to the result events. + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(req.Context(), snapshot, rsAPI, headeredEvents, nil, device.UserID, "relations") + if err != nil { + return util.ErrorResponse(err) + } + + // Convert the events into client events, and optionally filter based on the event + // type if it was specified. + res.Chunk = make([]gomatrixserverlib.ClientEvent, 0, len(filteredEvents)) + for _, event := range filteredEvents { + res.Chunk = append( + res.Chunk, + gomatrixserverlib.ToClientEvent(event.Event, gomatrixserverlib.FormatAll), + ) + } + + succeeded = true + return util.JSONResponse{ + Code: http.StatusOK, + JSON: res, + } +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 069dee81f..71fa93c1e 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -45,6 +45,7 @@ func Setup( lazyLoadCache caching.LazyLoadCache, fts *fulltext.Search, ) { + v1unstablemux := csMux.PathPrefix("/{apiversion:(?:v1|unstable)}/").Subrouter() v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter() // TODO: Add AS support for all handlers below. @@ -110,6 +111,48 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) + v1unstablemux.Handle("/rooms/{roomId}/relations/{eventId}", + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + + return Relations( + req, device, syncDB, rsAPI, + vars["roomId"], vars["eventId"], "", "", + ) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v1unstablemux.Handle("/rooms/{roomId}/relations/{eventId}/{relType}", + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + + return Relations( + req, device, syncDB, rsAPI, + vars["roomId"], vars["eventId"], vars["relType"], "", + ) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v1unstablemux.Handle("/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}", + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + + return Relations( + req, device, syncDB, rsAPI, + vars["roomId"], vars["eventId"], vars["relType"], vars["eventType"], + ) + }), + ).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/search", httputil.MakeAuthAPI("search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { if !cfg.Fulltext.Enabled { diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 4a03aca74..02d45f801 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -38,6 +38,7 @@ type DatabaseTransaction interface { MaxStreamPositionForSendToDeviceMessages(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) + MaxStreamPositionForRelations(ctx context.Context) (types.StreamPosition, error) CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) @@ -107,6 +108,7 @@ type DatabaseTransaction interface { GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) + RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error) } type Database interface { @@ -174,6 +176,8 @@ type Database interface { StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) + UpdateRelations(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error + RedactRelations(ctx context.Context, roomID, redactedEventID string) error } type Presence interface { diff --git a/syncapi/storage/postgres/relations_table.go b/syncapi/storage/postgres/relations_table.go new file mode 100644 index 000000000..5a76e9c33 --- /dev/null +++ b/syncapi/storage/postgres/relations_table.go @@ -0,0 +1,158 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +const relationsSchema = ` +CREATE SEQUENCE IF NOT EXISTS syncapi_relation_id; + +CREATE TABLE IF NOT EXISTS syncapi_relations ( + id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_relation_id'), + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + child_event_id TEXT NOT NULL, + child_event_type TEXT NOT NULL, + rel_type TEXT NOT NULL, + CONSTRAINT syncapi_relations_unique UNIQUE (room_id, event_id, child_event_id, rel_type) +); +` + +const insertRelationSQL = "" + + "INSERT INTO syncapi_relations (" + + " room_id, event_id, child_event_id, child_event_type, rel_type" + + ") VALUES ($1, $2, $3, $4, $5) " + + " ON CONFLICT DO NOTHING" + +const deleteRelationSQL = "" + + "DELETE FROM syncapi_relations WHERE room_id = $1 AND child_event_id = $2" + +const selectRelationsInRangeAscSQL = "" + + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id > $5 AND id <= $6" + + " ORDER BY id ASC LIMIT $7" + +const selectRelationsInRangeDescSQL = "" + + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id >= $5 AND id < $6" + + " ORDER BY id DESC LIMIT $7" + +const selectMaxRelationIDSQL = "" + + "SELECT COALESCE(MAX(id), 0) FROM syncapi_relations" + +type relationsStatements struct { + insertRelationStmt *sql.Stmt + selectRelationsInRangeAscStmt *sql.Stmt + selectRelationsInRangeDescStmt *sql.Stmt + deleteRelationStmt *sql.Stmt + selectMaxRelationIDStmt *sql.Stmt +} + +func NewPostgresRelationsTable(db *sql.DB) (tables.Relations, error) { + s := &relationsStatements{} + _, err := db.Exec(relationsSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.insertRelationStmt, insertRelationSQL}, + {&s.selectRelationsInRangeAscStmt, selectRelationsInRangeAscSQL}, + {&s.selectRelationsInRangeDescStmt, selectRelationsInRangeDescSQL}, + {&s.deleteRelationStmt, deleteRelationSQL}, + {&s.selectMaxRelationIDStmt, selectMaxRelationIDSQL}, + }.Prepare(db) +} + +func (s *relationsStatements) InsertRelation( + ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string, +) (err error) { + _, err = sqlutil.TxStmt(txn, s.insertRelationStmt).ExecContext( + ctx, roomID, eventID, childEventID, childEventType, relType, + ) + return +} + +func (s *relationsStatements) DeleteRelation( + ctx context.Context, txn *sql.Tx, roomID, childEventID string, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteRelationStmt) + _, err := stmt.ExecContext( + ctx, roomID, childEventID, + ) + return err +} + +// SelectRelationsInRange returns a map rel_type -> []child_event_id +func (s *relationsStatements) SelectRelationsInRange( + ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, + r types.Range, limit int, +) (map[string][]types.RelationEntry, types.StreamPosition, error) { + var lastPos types.StreamPosition + var stmt *sql.Stmt + if r.Backwards { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt) + } else { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt) + } + rows, err := stmt.QueryContext(ctx, roomID, eventID, relType, eventType, r.Low(), r.High(), limit) + if err != nil { + return nil, lastPos, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectRelationsInRange: rows.close() failed") + result := map[string][]types.RelationEntry{} + var ( + id types.StreamPosition + childEventID string + relationType string + ) + for rows.Next() { + if err = rows.Scan(&id, &childEventID, &relationType); err != nil { + return nil, lastPos, err + } + if id > lastPos { + lastPos = id + } + result[relationType] = append(result[relationType], types.RelationEntry{ + Position: id, + EventID: childEventID, + }) + } + if lastPos == 0 { + lastPos = r.To + } + return result, lastPos, rows.Err() +} + +func (s *relationsStatements) SelectMaxRelationID( + ctx context.Context, txn *sql.Tx, +) (id int64, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMaxRelationIDStmt) + err = stmt.QueryRowContext(ctx).Scan(&id) + return +} diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 979ff6647..850d24a07 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -98,6 +98,10 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) if err != nil { return nil, err } + relations, err := NewPostgresRelationsTable(d.db) + if err != nil { + return nil, err + } // apply migrations which need multiple tables m := sqlutil.NewMigrator(d.db) @@ -129,6 +133,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) NotificationData: notificationData, Ignores: ignores, Presence: presence, + Relations: relations, } return &d, nil } diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index 937ced3a2..bf12203db 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -53,6 +53,7 @@ type Database struct { NotificationData tables.NotificationData Ignores tables.Ignores Presence tables.Presence + Relations tables.Relations } func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) { @@ -579,10 +580,40 @@ func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID s return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos) } -func (s *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) { - return s.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{ +func (d *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) { + return d.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{ gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomTopic, "m.room.message", }) } + +func (d *Database) UpdateRelations(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error { + var content gomatrixserverlib.RelationContent + if err := json.Unmarshal(event.Content(), &content); err != nil { + return fmt.Errorf("json.Unmarshal: %w", err) + } + switch { + case content.Relations == nil: + return nil + case content.Relations.EventID == "": + return nil + case content.Relations.RelationType == "": + return nil + case event.Type() == gomatrixserverlib.MRoomRedaction: + return nil + default: + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.Relations.InsertRelation( + ctx, txn, event.RoomID(), content.Relations.EventID, + event.EventID(), event.Type(), content.Relations.RelationType, + ) + }) + } +} + +func (d *Database) RedactRelations(ctx context.Context, roomID, redactedEventID string) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.Relations.DeleteRelation(ctx, txn, roomID, redactedEventID) + }) +} diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index d5b5b3121..cb61c1c26 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -589,3 +589,84 @@ func (d *DatabaseTransaction) PresenceAfter(ctx context.Context, after types.Str func (d *DatabaseTransaction) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) { return d.Presence.GetMaxPresenceID(ctx, d.txn) } + +func (d *DatabaseTransaction) MaxStreamPositionForRelations(ctx context.Context) (types.StreamPosition, error) { + id, err := d.Relations.SelectMaxRelationID(ctx, d.txn) + return types.StreamPosition(id), err +} + +func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) ( + events []types.StreamEvent, prevBatch, nextBatch string, err error, +) { + r := types.Range{ + From: from, + To: to, + Backwards: backwards, + } + + if r.Backwards && r.From == 0 { + // If we're working backwards (dir=b) and there's no ?from= specified then + // we will automatically want to work backwards from the current position, + // so find out what that is. + if r.From, err = d.MaxStreamPositionForRelations(ctx); err != nil { + return nil, "", "", fmt.Errorf("d.MaxStreamPositionForRelations: %w", err) + } + // The result normally isn't inclusive of the event *at* the ?from= + // position, so add 1 here so that we include the most recent relation. + r.From++ + } else if !r.Backwards && r.To == 0 { + // If we're working forwards (dir=f) and there's no ?to= specified then + // we will automatically want to work forwards towards the current position, + // so find out what that is. + if r.To, err = d.MaxStreamPositionForRelations(ctx); err != nil { + return nil, "", "", fmt.Errorf("d.MaxStreamPositionForRelations: %w", err) + } + } + + // First look up any relations from the database. We add one to the limit here + // so that we can tell if we're overflowing, as we will only set the "next_batch" + // in the response if we are. + relations, _, err := d.Relations.SelectRelationsInRange(ctx, d.txn, roomID, eventID, relType, eventType, r, limit+1) + if err != nil { + return nil, "", "", fmt.Errorf("d.Relations.SelectRelationsInRange: %w", err) + } + + // If we specified a relation type then just get those results, otherwise collate + // them from all of the returned relation types. + entries := []types.RelationEntry{} + if relType != "" { + entries = relations[relType] + } else { + for _, e := range relations { + entries = append(entries, e...) + } + } + + // If there were no entries returned, there were no relations, so stop at this point. + if len(entries) == 0 { + return nil, "", "", nil + } + + // Otherwise, let's try and work out what sensible prev_batch and next_batch values + // could be. We've requested an extra event by adding one to the limit already so + // that we can determine whether or not to provide a "next_batch", so trim off that + // event off the end if needs be. + if len(entries) > limit { + entries = entries[:len(entries)-1] + nextBatch = fmt.Sprintf("%d", entries[len(entries)-1].Position) + } + // TODO: set prevBatch? doesn't seem to affect the tests... + + // Extract all of the event IDs from the relation entries so that we can pull the + // events out of the database. Then go and fetch the events. + eventIDs := make([]string, 0, len(entries)) + for _, entry := range entries { + eventIDs = append(eventIDs, entry.EventID) + } + events, err = d.OutputEvents.SelectEvents(ctx, d.txn, eventIDs, nil, true) + if err != nil { + return nil, "", "", fmt.Errorf("d.OutputEvents.SelectEvents: %w", err) + } + + return events, prevBatch, nextBatch, nil +} diff --git a/syncapi/storage/sqlite3/relations_table.go b/syncapi/storage/sqlite3/relations_table.go new file mode 100644 index 000000000..7cbb5408f --- /dev/null +++ b/syncapi/storage/sqlite3/relations_table.go @@ -0,0 +1,163 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +const relationsSchema = ` +CREATE TABLE IF NOT EXISTS syncapi_relations ( + id BIGINT PRIMARY KEY, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + child_event_id TEXT NOT NULL, + child_event_type TEXT NOT NULL, + rel_type TEXT NOT NULL, + UNIQUE (room_id, event_id, child_event_id, rel_type) +); +` + +const insertRelationSQL = "" + + "INSERT INTO syncapi_relations (" + + " id, room_id, event_id, child_event_id, child_event_type, rel_type" + + ") VALUES ($1, $2, $3, $4, $5, $6) " + + " ON CONFLICT DO NOTHING" + +const deleteRelationSQL = "" + + "DELETE FROM syncapi_relations WHERE room_id = $1 AND child_event_id = $2" + +const selectRelationsInRangeAscSQL = "" + + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id > $5 AND id <= $6" + + " ORDER BY id ASC LIMIT $7" + +const selectRelationsInRangeDescSQL = "" + + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id >= $5 AND id < $6" + + " ORDER BY id DESC LIMIT $7" + +const selectMaxRelationIDSQL = "" + + "SELECT COALESCE(MAX(id), 0) FROM syncapi_relations" + +type relationsStatements struct { + streamIDStatements *StreamIDStatements + insertRelationStmt *sql.Stmt + selectRelationsInRangeAscStmt *sql.Stmt + selectRelationsInRangeDescStmt *sql.Stmt + deleteRelationStmt *sql.Stmt + selectMaxRelationIDStmt *sql.Stmt +} + +func NewSqliteRelationsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Relations, error) { + s := &relationsStatements{ + streamIDStatements: streamID, + } + _, err := db.Exec(relationsSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.insertRelationStmt, insertRelationSQL}, + {&s.selectRelationsInRangeAscStmt, selectRelationsInRangeAscSQL}, + {&s.selectRelationsInRangeDescStmt, selectRelationsInRangeDescSQL}, + {&s.deleteRelationStmt, deleteRelationSQL}, + {&s.selectMaxRelationIDStmt, selectMaxRelationIDSQL}, + }.Prepare(db) +} + +func (s *relationsStatements) InsertRelation( + ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string, +) (err error) { + var streamPos types.StreamPosition + if streamPos, err = s.streamIDStatements.nextRelationID(ctx, txn); err != nil { + return + } + _, err = sqlutil.TxStmt(txn, s.insertRelationStmt).ExecContext( + ctx, streamPos, roomID, eventID, childEventID, childEventType, relType, + ) + return +} + +func (s *relationsStatements) DeleteRelation( + ctx context.Context, txn *sql.Tx, roomID, childEventID string, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteRelationStmt) + _, err := stmt.ExecContext( + ctx, roomID, childEventID, + ) + return err +} + +// SelectRelationsInRange returns a map rel_type -> []child_event_id +func (s *relationsStatements) SelectRelationsInRange( + ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, + r types.Range, limit int, +) (map[string][]types.RelationEntry, types.StreamPosition, error) { + var lastPos types.StreamPosition + var stmt *sql.Stmt + if r.Backwards { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt) + } else { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt) + } + rows, err := stmt.QueryContext(ctx, roomID, eventID, relType, eventType, r.Low(), r.High(), limit) + if err != nil { + return nil, lastPos, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectRelationsInRange: rows.close() failed") + result := map[string][]types.RelationEntry{} + var ( + id types.StreamPosition + childEventID string + relationType string + ) + for rows.Next() { + if err = rows.Scan(&id, &childEventID, &relationType); err != nil { + return nil, lastPos, err + } + if id > lastPos { + lastPos = id + } + result[relationType] = append(result[relationType], types.RelationEntry{ + Position: id, + EventID: childEventID, + }) + } + if lastPos == 0 { + lastPos = r.To + } + return result, lastPos, rows.Err() +} + +func (s *relationsStatements) SelectMaxRelationID( + ctx context.Context, txn *sql.Tx, +) (id int64, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMaxRelationIDStmt) + err = stmt.QueryRowContext(ctx).Scan(&id) + return +} diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go index 1160a437e..a4bba508e 100644 --- a/syncapi/storage/sqlite3/stream_id_table.go +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -28,6 +28,8 @@ INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("presence", 0) ON CONFLICT DO NOTHING; INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("notification", 0) ON CONFLICT DO NOTHING; +INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("relation", 0) + ON CONFLICT DO NOTHING; ` const increaseStreamIDStmt = "" + @@ -86,3 +88,9 @@ func (s *StreamIDStatements) nextNotificationID(ctx context.Context, txn *sql.Tx err = increaseStmt.QueryRowContext(ctx, "notification").Scan(&pos) return } + +func (s *StreamIDStatements) nextRelationID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { + increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) + err = increaseStmt.QueryRowContext(ctx, "relation").Scan(&pos) + return +} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 0879030a6..510546909 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -123,6 +123,10 @@ func (d *SyncServerDatasource) prepare(ctx context.Context) (err error) { if err != nil { return err } + relations, err := NewSqliteRelationsTable(d.db, &d.streamID) + if err != nil { + return err + } // apply migrations which need multiple tables m := sqlutil.NewMigrator(d.db) @@ -153,6 +157,7 @@ func (d *SyncServerDatasource) prepare(ctx context.Context) (err error) { NotificationData: notificationData, Ignores: ignores, Presence: presence, + Relations: relations, } return nil } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 2fdc3cfbb..e48c050dd 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -206,3 +206,22 @@ type Presence interface { GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error) } + +type Relations interface { + // Inserts a relation which refers from the child event ID to the event ID in the given room. + // If the relation already exists then this function will do nothing and return no error. + InsertRelation(ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string) (err error) + // Deletes a relation which already exists as the result of an event redaction. If the relation + // does not exist then this function will do nothing and return no error. + DeleteRelation(ctx context.Context, txn *sql.Tx, roomID, childEventID string) error + // SelectRelationsInRange will return relations grouped by relation type within the given range. + // The map is relType -> []entry. If a relType parameter is specified then the results will only + // contain relations of that type, otherwise if "" is specified then all relations in the range + // will be returned, inclusive of the "to" position but excluding the "from" position. The stream + // position returned is the maximum position of the returned results. + SelectRelationsInRange(ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, r types.Range, limit int) (map[string][]types.RelationEntry, types.StreamPosition, error) + // SelectMaxRelationID returns the maximum ID of all relations, used to determine what the boundaries + // should be if there are no boundaries supplied (i.e. we want to work backwards but don't have a + // "from" or want to work forwards and don't have a "to"). + SelectMaxRelationID(ctx context.Context, txn *sql.Tx) (id int64, err error) +} diff --git a/syncapi/storage/tables/relations_test.go b/syncapi/storage/tables/relations_test.go new file mode 100644 index 000000000..46270e36d --- /dev/null +++ b/syncapi/storage/tables/relations_test.go @@ -0,0 +1,186 @@ +package tables_test + +import ( + "context" + "database/sql" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage/postgres" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/test" +) + +func newRelationsTable(t *testing.T, dbType test.DBType) (tables.Relations, *sql.DB, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + if err != nil { + t.Fatalf("failed to open db: %s", err) + } + + var tab tables.Relations + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresRelationsTable(db) + case test.DBTypeSQLite: + var stream sqlite3.StreamIDStatements + if err = stream.Prepare(db); err != nil { + t.Fatalf("failed to prepare stream stmts: %s", err) + } + tab, err = sqlite3.NewSqliteRelationsTable(db, &stream) + } + if err != nil { + t.Fatalf("failed to make new table: %s", err) + } + return tab, db, close +} + +func compareRelationsToExpected(t *testing.T, tab tables.Relations, r types.Range, expected []types.RelationEntry) { + ctx := context.Background() + relations, _, err := tab.SelectRelationsInRange(ctx, nil, roomID, "a", "", "", r, 50) + if err != nil { + t.Fatal(err) + } + if len(relations[relType]) != len(expected) { + t.Fatalf("incorrect number of values returned for range %v (got %d, want %d)", r, len(relations[relType]), len(expected)) + } + for i := 0; i < len(relations[relType]); i++ { + got := relations[relType][i] + want := expected[i] + if got != want { + t.Fatalf("range %v position %d should have been %q but got %q", r, i, got, want) + } + } +} + +const roomID = "!roomid:server" +const childType = "m.room.something" +const relType = "m.reaction" + +func TestRelationsTable(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, _, close := newRelationsTable(t, dbType) + defer close() + + // Insert some relations + for _, child := range []string{"b", "c", "d"} { + if err := tab.InsertRelation(ctx, nil, roomID, "a", child, childType, relType); err != nil { + t.Fatal(err) + } + } + + // Check the max position, we've inserted three things so it + // should be 3 + if max, err := tab.SelectMaxRelationID(ctx, nil); err != nil { + t.Fatal(err) + } else if max != 3 { + t.Fatalf("max position should have been 3 but got %d", max) + } + + // Query some ranges for "a" + for r, expected := range map[types.Range][]types.RelationEntry{ + {From: 0, To: 10, Backwards: false}: { + {Position: 1, EventID: "b"}, + {Position: 2, EventID: "c"}, + {Position: 3, EventID: "d"}, + }, + {From: 1, To: 2, Backwards: false}: { + {Position: 2, EventID: "c"}, + }, + {From: 1, To: 3, Backwards: false}: { + {Position: 2, EventID: "c"}, + {Position: 3, EventID: "d"}, + }, + {From: 10, To: 0, Backwards: true}: { + {Position: 3, EventID: "d"}, + {Position: 2, EventID: "c"}, + {Position: 1, EventID: "b"}, + }, + {From: 3, To: 1, Backwards: true}: { + {Position: 2, EventID: "c"}, + {Position: 1, EventID: "b"}, + }, + } { + compareRelationsToExpected(t, tab, r, expected) + } + + // Now delete one of the relations + if err := tab.DeleteRelation(ctx, nil, roomID, "c"); err != nil { + t.Fatal(err) + } + + // Query some more ranges for "a" + for r, expected := range map[types.Range][]types.RelationEntry{ + {From: 0, To: 10, Backwards: false}: { + {Position: 1, EventID: "b"}, + {Position: 3, EventID: "d"}, + }, + {From: 1, To: 2, Backwards: false}: {}, + {From: 1, To: 3, Backwards: false}: { + {Position: 3, EventID: "d"}, + }, + {From: 10, To: 0, Backwards: true}: { + {Position: 3, EventID: "d"}, + {Position: 1, EventID: "b"}, + }, + {From: 3, To: 1, Backwards: true}: { + {Position: 1, EventID: "b"}, + }, + } { + compareRelationsToExpected(t, tab, r, expected) + } + + // Insert some new relations + for _, child := range []string{"e", "f", "g", "h"} { + if err := tab.InsertRelation(ctx, nil, roomID, "a", child, childType, relType); err != nil { + t.Fatal(err) + } + } + + // Check the max position, we've inserted four things so it + // should now be 7 + if max, err := tab.SelectMaxRelationID(ctx, nil); err != nil { + t.Fatal(err) + } else if max != 7 { + t.Fatalf("max position should have been 3 but got %d", max) + } + + // Query last set of ranges for "a" + for r, expected := range map[types.Range][]types.RelationEntry{ + {From: 0, To: 10, Backwards: false}: { + {Position: 1, EventID: "b"}, + {Position: 3, EventID: "d"}, + {Position: 4, EventID: "e"}, + {Position: 5, EventID: "f"}, + {Position: 6, EventID: "g"}, + {Position: 7, EventID: "h"}, + }, + {From: 1, To: 2, Backwards: false}: {}, + {From: 1, To: 3, Backwards: false}: { + {Position: 3, EventID: "d"}, + }, + {From: 10, To: 0, Backwards: true}: { + {Position: 7, EventID: "h"}, + {Position: 6, EventID: "g"}, + {Position: 5, EventID: "f"}, + {Position: 4, EventID: "e"}, + {Position: 3, EventID: "d"}, + {Position: 1, EventID: "b"}, + }, + {From: 6, To: 3, Backwards: true}: { + {Position: 5, EventID: "f"}, + {Position: 4, EventID: "e"}, + {Position: 3, EventID: "d"}, + }, + } { + compareRelationsToExpected(t, tab, r, expected) + } + }) +} diff --git a/syncapi/types/types.go b/syncapi/types/types.go index b6d340f93..60a74a285 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -47,6 +47,14 @@ type StateDelta struct { // StreamPosition represents the offset in the sync stream a client is at. type StreamPosition int64 +func NewStreamPositionFromString(s string) (StreamPosition, error) { + n, err := strconv.Atoi(s) + if err != nil { + return 0, err + } + return StreamPosition(n), nil +} + // StreamEvent is the same as gomatrixserverlib.Event but also has the PDU stream position for this event. type StreamEvent struct { *gomatrixserverlib.HeaderedEvent @@ -599,3 +607,8 @@ type OutputSendToDeviceEvent struct { type IgnoredUsers struct { List map[string]interface{} `json:"ignored_users"` } + +type RelationEntry struct { + Position StreamPosition + EventID string +} From dcedd1b6bf1e890ff425bdf1fcd8a2e0850778b5 Mon Sep 17 00:00:00 2001 From: devonh Date: Thu, 13 Oct 2022 14:38:13 +0000 Subject: [PATCH 52/61] Federation backoff fixes and tests (#2792) This fixes some edge cases where federation queue backoffs and blacklisting weren't behaving as expected. It also adds new tests for the federation queues to ensure their behaviour continues to work correctly. --- federationapi/queue/destinationqueue.go | 10 + federationapi/queue/queue.go | 1 + federationapi/queue/queue_test.go | 422 ++++++++++++++++++++++++ federationapi/statistics/statistics.go | 8 +- go.mod | 2 +- 5 files changed, 441 insertions(+), 2 deletions(-) create mode 100644 federationapi/queue/queue_test.go diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 5cb8cae1f..4ae554ef3 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -75,6 +75,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re logrus.Errorf("attempt to send nil PDU with destination %q", oq.destination) return } + // Create a database entry that associates the given PDU NID with // this destination queue. We'll then be able to retrieve the PDU // later. @@ -108,6 +109,8 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re case oq.notify <- struct{}{}: default: } + } else { + oq.overflowed.Store(true) } } @@ -153,6 +156,8 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share case oq.notify <- struct{}{}: default: } + } else { + oq.overflowed.Store(true) } } @@ -335,6 +340,11 @@ func (oq *destinationQueue) backgroundSend() { // We failed to send the transaction. Mark it as a failure. oq.statistics.Failure() + // Queue up another attempt since the transaction failed. + select { + case oq.notify <- struct{}{}: + default: + } } else if transaction { // If we successfully sent the transaction then clear out // the pending events and EDUs, and wipe our transaction ID. diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 8245aa5bd..5d352eca6 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -332,6 +332,7 @@ func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) { if oqs.disabled { return } + oqs.statistics.ForServer(srv).RemoveBlacklist() if queue := oqs.getQueue(srv); queue != nil { queue.wakeQueueIfNeeded() } diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go new file mode 100644 index 000000000..8e4a675f4 --- /dev/null +++ b/federationapi/queue/queue_test.go @@ -0,0 +1,422 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + "go.uber.org/atomic" + "gotest.tools/v3/poll" + + "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/federationapi/storage/shared" + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +var dbMutex sync.Mutex + +type fakeDatabase struct { + storage.Database + pendingPDUServers map[gomatrixserverlib.ServerName]struct{} + pendingEDUServers map[gomatrixserverlib.ServerName]struct{} + blacklistedServers map[gomatrixserverlib.ServerName]struct{} + pendingPDUs map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent + pendingEDUs map[*shared.Receipt]*gomatrixserverlib.EDU + associatedPDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} + associatedEDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} +} + +func (d *fakeDatabase) StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal([]byte(js), &event); err == nil { + receipt := &shared.Receipt{} + d.pendingPDUs[receipt] = &event + return receipt, nil + } + + var edu gomatrixserverlib.EDU + if err := json.Unmarshal([]byte(js), &edu); err == nil { + receipt := &shared.Receipt{} + d.pendingEDUs[receipt] = &edu + return receipt, nil + } + + return nil, errors.New("Failed to determine type of json to store") +} + +func (d *fakeDatabase) GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + pdus = make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent) + if receipts, ok := d.associatedPDUs[serverName]; ok { + for receipt := range receipts { + if event, ok := d.pendingPDUs[receipt]; ok { + pdus[receipt] = event + } + } + } + return pdus, nil +} + +func (d *fakeDatabase) GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + edus = make(map[*shared.Receipt]*gomatrixserverlib.EDU) + if receipts, ok := d.associatedEDUs[serverName]; ok { + for receipt := range receipts { + if event, ok := d.pendingEDUs[receipt]; ok { + edus[receipt] = event + } + } + } + return edus, nil +} + +func (d *fakeDatabase) AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + if _, ok := d.pendingPDUs[receipt]; ok { + if _, ok := d.associatedPDUs[serverName]; !ok { + d.associatedPDUs[serverName] = make(map[*shared.Receipt]struct{}) + } + d.associatedPDUs[serverName][receipt] = struct{}{} + return nil + } else { + return errors.New("PDU doesn't exist") + } +} + +func (d *fakeDatabase) AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + if _, ok := d.pendingEDUs[receipt]; ok { + if _, ok := d.associatedEDUs[serverName]; !ok { + d.associatedEDUs[serverName] = make(map[*shared.Receipt]struct{}) + } + d.associatedEDUs[serverName][receipt] = struct{}{} + return nil + } else { + return errors.New("EDU doesn't exist") + } +} + +func (d *fakeDatabase) CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + if pdus, ok := d.associatedPDUs[serverName]; ok { + for _, receipt := range receipts { + delete(pdus, receipt) + } + } + + return nil +} + +func (d *fakeDatabase) CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + if edus, ok := d.associatedEDUs[serverName]; ok { + for _, receipt := range receipts { + delete(edus, receipt) + } + } + + return nil +} + +func (d *fakeDatabase) GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + var count int64 + if pdus, ok := d.associatedPDUs[serverName]; ok { + count = int64(len(pdus)) + } + return count, nil +} + +func (d *fakeDatabase) GetPendingEDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + var count int64 + if edus, ok := d.associatedEDUs[serverName]; ok { + count = int64(len(edus)) + } + return count, nil +} + +func (d *fakeDatabase) GetPendingPDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + servers := []gomatrixserverlib.ServerName{} + for server := range d.pendingPDUServers { + servers = append(servers, server) + } + return servers, nil +} + +func (d *fakeDatabase) GetPendingEDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + servers := []gomatrixserverlib.ServerName{} + for server := range d.pendingEDUServers { + servers = append(servers, server) + } + return servers, nil +} + +func (d *fakeDatabase) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + d.blacklistedServers[serverName] = struct{}{} + return nil +} + +func (d *fakeDatabase) RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + delete(d.blacklistedServers, serverName) + return nil +} + +func (d *fakeDatabase) RemoveAllServersFromBlacklist() error { + dbMutex.Lock() + defer dbMutex.Unlock() + + d.blacklistedServers = make(map[gomatrixserverlib.ServerName]struct{}) + return nil +} + +func (d *fakeDatabase) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + isBlacklisted := false + if _, ok := d.blacklistedServers[serverName]; ok { + isBlacklisted = true + } + + return isBlacklisted, nil +} + +type stubFederationRoomServerAPI struct { + rsapi.FederationRoomserverAPI +} + +func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error { + res.Banned = false + return nil +} + +type stubFederationClient struct { + api.FederationClient + shouldTxSucceed bool + txCount atomic.Uint32 +} + +func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) { + var result error + if !f.shouldTxSucceed { + result = fmt.Errorf("transaction failed") + } + + f.txCount.Add(1) + return gomatrixserverlib.RespSend{}, result +} + +func createDatabase() storage.Database { + return &fakeDatabase{ + pendingPDUServers: make(map[gomatrixserverlib.ServerName]struct{}), + pendingEDUServers: make(map[gomatrixserverlib.ServerName]struct{}), + blacklistedServers: make(map[gomatrixserverlib.ServerName]struct{}), + pendingPDUs: make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent), + pendingEDUs: make(map[*shared.Receipt]*gomatrixserverlib.EDU), + associatedPDUs: make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}), + } +} + +func mustCreateEvent(t *testing.T) *gomatrixserverlib.HeaderedEvent { + t.Helper() + content := `{"type":"m.room.message"}` + ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(content), false, gomatrixserverlib.RoomVersionV10) + if err != nil { + t.Fatalf("failed to create event: %v", err) + } + return ev.Headered(gomatrixserverlib.RoomVersionV10) +} + +func testSetup(failuresUntilBlacklist uint32, shouldTxSucceed bool) (storage.Database, *stubFederationClient, *OutgoingQueues) { + db := createDatabase() + + fc := &stubFederationClient{ + shouldTxSucceed: shouldTxSucceed, + txCount: *atomic.NewUint32(0), + } + rs := &stubFederationRoomServerAPI{} + stats := &statistics.Statistics{ + DB: db, + FailuresUntilBlacklist: failuresUntilBlacklist, + } + signingInfo := &SigningInfo{ + KeyID: "ed25519:auto", + PrivateKey: test.PrivateKeyA, + ServerName: "localhost", + } + queues := NewOutgoingQueues(db, process.NewProcessContext(), false, "localhost", fc, rs, stats, signingInfo) + + return db, fc, queues +} + +func TestSendTransactionOnSuccessRemovedFromDB(t *testing.T) { + ctx := context.Background() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues := testSetup(failuresUntilBlacklist, true) + + ev := mustCreateEvent(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() >= 1 { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for event to be removed from database") + } + return poll.Continue("waiting for more send attempts before checking database") + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendTransactionOnFailStoredInDB(t *testing.T) { + ctx := context.Background() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues := testSetup(failuresUntilBlacklist, false) + + ev := mustCreateEvent(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + // Wait for 2 backoff attempts to ensure there was adequate time to attempt sending + if fc.txCount.Load() >= 2 { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 1 { + return poll.Success() + } + return poll.Continue("waiting for event to be added to database") + } + return poll.Continue("waiting for more send attempts before checking database") + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendTransactionMultipleFailuresBlacklisted(t *testing.T) { + ctx := context.Background() + failuresUntilBlacklist := uint32(2) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues := testSetup(failuresUntilBlacklist, false) + + ev := mustCreateEvent(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() >= failuresUntilBlacklist { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + return poll.Success() + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database") + } + return poll.Continue("waiting for more send attempts before checking database") + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestRetryServerSendsSuccessfully(t *testing.T) { + ctx := context.Background() + failuresUntilBlacklist := uint32(1) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues := testSetup(failuresUntilBlacklist, false) + + ev := mustCreateEvent(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + checkBlacklisted := func(log poll.LogT) poll.Result { + if fc.txCount.Load() >= failuresUntilBlacklist { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + return poll.Success() + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database") + } + return poll.Continue("waiting for more send attempts before checking database") + } + poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) + + fc.shouldTxSucceed = true + db.RemoveServerFromBlacklist(destination) + queues.RetryServer(destination) + checkRetry := func(log poll.LogT) poll.Result { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for event to be removed from database") + } + poll.WaitOn(t, checkRetry, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go index db6d5c735..61a965791 100644 --- a/federationapi/statistics/statistics.go +++ b/federationapi/statistics/statistics.go @@ -95,8 +95,8 @@ func (s *ServerStatistics) cancel() { // we will unblacklist it. func (s *ServerStatistics) Success() { s.cancel() - s.successCounter.Inc() s.backoffCount.Store(0) + s.successCounter.Inc() if s.statistics.DB != nil { if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil { logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName) @@ -174,6 +174,12 @@ func (s *ServerStatistics) Blacklisted() bool { return s.blacklisted.Load() } +// RemoveBlacklist removes the blacklisted status from the server. +func (s *ServerStatistics) RemoveBlacklist() { + s.cancel() + s.backoffCount.Store(0) +} + // SuccessCount returns the number of successful requests. This is // usually useful in constructing transaction IDs. func (s *ServerStatistics) SuccessCount() uint32 { diff --git a/go.mod b/go.mod index eefad89e6..eeae9608f 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( golang.org/x/term v0.0.0-20220919170432-7a66f970e087 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 + gotest.tools/v3 v3.0.3 nhooyr.io/websocket v1.8.7 ) @@ -128,7 +129,6 @@ require ( gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gotest.tools/v3 v3.0.3 // indirect ) go 1.18 From f3be4b31850add1a33c932aa3fa7b0bb740554f2 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 13 Oct 2022 16:06:50 +0100 Subject: [PATCH 53/61] Revert "Federation backoff fixes and tests (#2792)" This reverts commit dcedd1b6bf1e890ff425bdf1fcd8a2e0850778b5. --- federationapi/queue/destinationqueue.go | 10 - federationapi/queue/queue.go | 1 - federationapi/queue/queue_test.go | 422 ------------------------ federationapi/statistics/statistics.go | 8 +- go.mod | 2 +- 5 files changed, 2 insertions(+), 441 deletions(-) delete mode 100644 federationapi/queue/queue_test.go diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 4ae554ef3..5cb8cae1f 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -75,7 +75,6 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re logrus.Errorf("attempt to send nil PDU with destination %q", oq.destination) return } - // Create a database entry that associates the given PDU NID with // this destination queue. We'll then be able to retrieve the PDU // later. @@ -109,8 +108,6 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re case oq.notify <- struct{}{}: default: } - } else { - oq.overflowed.Store(true) } } @@ -156,8 +153,6 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share case oq.notify <- struct{}{}: default: } - } else { - oq.overflowed.Store(true) } } @@ -340,11 +335,6 @@ func (oq *destinationQueue) backgroundSend() { // We failed to send the transaction. Mark it as a failure. oq.statistics.Failure() - // Queue up another attempt since the transaction failed. - select { - case oq.notify <- struct{}{}: - default: - } } else if transaction { // If we successfully sent the transaction then clear out // the pending events and EDUs, and wipe our transaction ID. diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 5d352eca6..8245aa5bd 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -332,7 +332,6 @@ func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) { if oqs.disabled { return } - oqs.statistics.ForServer(srv).RemoveBlacklist() if queue := oqs.getQueue(srv); queue != nil { queue.wakeQueueIfNeeded() } diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go deleted file mode 100644 index 8e4a675f4..000000000 --- a/federationapi/queue/queue_test.go +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright 2022 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package queue - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "testing" - "time" - - "go.uber.org/atomic" - "gotest.tools/v3/poll" - - "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/federationapi/statistics" - "github.com/matrix-org/dendrite/federationapi/storage" - "github.com/matrix-org/dendrite/federationapi/storage/shared" - rsapi "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/gomatrixserverlib" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" -) - -var dbMutex sync.Mutex - -type fakeDatabase struct { - storage.Database - pendingPDUServers map[gomatrixserverlib.ServerName]struct{} - pendingEDUServers map[gomatrixserverlib.ServerName]struct{} - blacklistedServers map[gomatrixserverlib.ServerName]struct{} - pendingPDUs map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent - pendingEDUs map[*shared.Receipt]*gomatrixserverlib.EDU - associatedPDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} - associatedEDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} -} - -func (d *fakeDatabase) StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - var event gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal([]byte(js), &event); err == nil { - receipt := &shared.Receipt{} - d.pendingPDUs[receipt] = &event - return receipt, nil - } - - var edu gomatrixserverlib.EDU - if err := json.Unmarshal([]byte(js), &edu); err == nil { - receipt := &shared.Receipt{} - d.pendingEDUs[receipt] = &edu - return receipt, nil - } - - return nil, errors.New("Failed to determine type of json to store") -} - -func (d *fakeDatabase) GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - pdus = make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent) - if receipts, ok := d.associatedPDUs[serverName]; ok { - for receipt := range receipts { - if event, ok := d.pendingPDUs[receipt]; ok { - pdus[receipt] = event - } - } - } - return pdus, nil -} - -func (d *fakeDatabase) GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - edus = make(map[*shared.Receipt]*gomatrixserverlib.EDU) - if receipts, ok := d.associatedEDUs[serverName]; ok { - for receipt := range receipts { - if event, ok := d.pendingEDUs[receipt]; ok { - edus[receipt] = event - } - } - } - return edus, nil -} - -func (d *fakeDatabase) AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - if _, ok := d.pendingPDUs[receipt]; ok { - if _, ok := d.associatedPDUs[serverName]; !ok { - d.associatedPDUs[serverName] = make(map[*shared.Receipt]struct{}) - } - d.associatedPDUs[serverName][receipt] = struct{}{} - return nil - } else { - return errors.New("PDU doesn't exist") - } -} - -func (d *fakeDatabase) AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - if _, ok := d.pendingEDUs[receipt]; ok { - if _, ok := d.associatedEDUs[serverName]; !ok { - d.associatedEDUs[serverName] = make(map[*shared.Receipt]struct{}) - } - d.associatedEDUs[serverName][receipt] = struct{}{} - return nil - } else { - return errors.New("EDU doesn't exist") - } -} - -func (d *fakeDatabase) CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - if pdus, ok := d.associatedPDUs[serverName]; ok { - for _, receipt := range receipts { - delete(pdus, receipt) - } - } - - return nil -} - -func (d *fakeDatabase) CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - if edus, ok := d.associatedEDUs[serverName]; ok { - for _, receipt := range receipts { - delete(edus, receipt) - } - } - - return nil -} - -func (d *fakeDatabase) GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - var count int64 - if pdus, ok := d.associatedPDUs[serverName]; ok { - count = int64(len(pdus)) - } - return count, nil -} - -func (d *fakeDatabase) GetPendingEDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - var count int64 - if edus, ok := d.associatedEDUs[serverName]; ok { - count = int64(len(edus)) - } - return count, nil -} - -func (d *fakeDatabase) GetPendingPDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - servers := []gomatrixserverlib.ServerName{} - for server := range d.pendingPDUServers { - servers = append(servers, server) - } - return servers, nil -} - -func (d *fakeDatabase) GetPendingEDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - servers := []gomatrixserverlib.ServerName{} - for server := range d.pendingEDUServers { - servers = append(servers, server) - } - return servers, nil -} - -func (d *fakeDatabase) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - d.blacklistedServers[serverName] = struct{}{} - return nil -} - -func (d *fakeDatabase) RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - delete(d.blacklistedServers, serverName) - return nil -} - -func (d *fakeDatabase) RemoveAllServersFromBlacklist() error { - dbMutex.Lock() - defer dbMutex.Unlock() - - d.blacklistedServers = make(map[gomatrixserverlib.ServerName]struct{}) - return nil -} - -func (d *fakeDatabase) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - isBlacklisted := false - if _, ok := d.blacklistedServers[serverName]; ok { - isBlacklisted = true - } - - return isBlacklisted, nil -} - -type stubFederationRoomServerAPI struct { - rsapi.FederationRoomserverAPI -} - -func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error { - res.Banned = false - return nil -} - -type stubFederationClient struct { - api.FederationClient - shouldTxSucceed bool - txCount atomic.Uint32 -} - -func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) { - var result error - if !f.shouldTxSucceed { - result = fmt.Errorf("transaction failed") - } - - f.txCount.Add(1) - return gomatrixserverlib.RespSend{}, result -} - -func createDatabase() storage.Database { - return &fakeDatabase{ - pendingPDUServers: make(map[gomatrixserverlib.ServerName]struct{}), - pendingEDUServers: make(map[gomatrixserverlib.ServerName]struct{}), - blacklistedServers: make(map[gomatrixserverlib.ServerName]struct{}), - pendingPDUs: make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent), - pendingEDUs: make(map[*shared.Receipt]*gomatrixserverlib.EDU), - associatedPDUs: make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}), - } -} - -func mustCreateEvent(t *testing.T) *gomatrixserverlib.HeaderedEvent { - t.Helper() - content := `{"type":"m.room.message"}` - ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(content), false, gomatrixserverlib.RoomVersionV10) - if err != nil { - t.Fatalf("failed to create event: %v", err) - } - return ev.Headered(gomatrixserverlib.RoomVersionV10) -} - -func testSetup(failuresUntilBlacklist uint32, shouldTxSucceed bool) (storage.Database, *stubFederationClient, *OutgoingQueues) { - db := createDatabase() - - fc := &stubFederationClient{ - shouldTxSucceed: shouldTxSucceed, - txCount: *atomic.NewUint32(0), - } - rs := &stubFederationRoomServerAPI{} - stats := &statistics.Statistics{ - DB: db, - FailuresUntilBlacklist: failuresUntilBlacklist, - } - signingInfo := &SigningInfo{ - KeyID: "ed25519:auto", - PrivateKey: test.PrivateKeyA, - ServerName: "localhost", - } - queues := NewOutgoingQueues(db, process.NewProcessContext(), false, "localhost", fc, rs, stats, signingInfo) - - return db, fc, queues -} - -func TestSendTransactionOnSuccessRemovedFromDB(t *testing.T) { - ctx := context.Background() - failuresUntilBlacklist := uint32(16) - destination := gomatrixserverlib.ServerName("remotehost") - db, fc, queues := testSetup(failuresUntilBlacklist, true) - - ev := mustCreateEvent(t) - err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) - assert.NoError(t, err) - - check := func(log poll.LogT) poll.Result { - if fc.txCount.Load() >= 1 { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 0 { - return poll.Success() - } - return poll.Continue("waiting for event to be removed from database") - } - return poll.Continue("waiting for more send attempts before checking database") - } - poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) -} - -func TestSendTransactionOnFailStoredInDB(t *testing.T) { - ctx := context.Background() - failuresUntilBlacklist := uint32(16) - destination := gomatrixserverlib.ServerName("remotehost") - db, fc, queues := testSetup(failuresUntilBlacklist, false) - - ev := mustCreateEvent(t) - err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) - assert.NoError(t, err) - - check := func(log poll.LogT) poll.Result { - // Wait for 2 backoff attempts to ensure there was adequate time to attempt sending - if fc.txCount.Load() >= 2 { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 1 { - return poll.Success() - } - return poll.Continue("waiting for event to be added to database") - } - return poll.Continue("waiting for more send attempts before checking database") - } - poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) -} - -func TestSendTransactionMultipleFailuresBlacklisted(t *testing.T) { - ctx := context.Background() - failuresUntilBlacklist := uint32(2) - destination := gomatrixserverlib.ServerName("remotehost") - db, fc, queues := testSetup(failuresUntilBlacklist, false) - - ev := mustCreateEvent(t) - err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) - assert.NoError(t, err) - - check := func(log poll.LogT) poll.Result { - if fc.txCount.Load() >= failuresUntilBlacklist { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 1 { - if val, _ := db.IsServerBlacklisted(destination); val { - return poll.Success() - } - return poll.Continue("waiting for server to be blacklisted") - } - return poll.Continue("waiting for event to be added to database") - } - return poll.Continue("waiting for more send attempts before checking database") - } - poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) -} - -func TestRetryServerSendsSuccessfully(t *testing.T) { - ctx := context.Background() - failuresUntilBlacklist := uint32(1) - destination := gomatrixserverlib.ServerName("remotehost") - db, fc, queues := testSetup(failuresUntilBlacklist, false) - - ev := mustCreateEvent(t) - err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) - assert.NoError(t, err) - - checkBlacklisted := func(log poll.LogT) poll.Result { - if fc.txCount.Load() >= failuresUntilBlacklist { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 1 { - if val, _ := db.IsServerBlacklisted(destination); val { - return poll.Success() - } - return poll.Continue("waiting for server to be blacklisted") - } - return poll.Continue("waiting for event to be added to database") - } - return poll.Continue("waiting for more send attempts before checking database") - } - poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) - - fc.shouldTxSucceed = true - db.RemoveServerFromBlacklist(destination) - queues.RetryServer(destination) - checkRetry := func(log poll.LogT) poll.Result { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 0 { - return poll.Success() - } - return poll.Continue("waiting for event to be removed from database") - } - poll.WaitOn(t, checkRetry, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) -} diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go index 61a965791..db6d5c735 100644 --- a/federationapi/statistics/statistics.go +++ b/federationapi/statistics/statistics.go @@ -95,8 +95,8 @@ func (s *ServerStatistics) cancel() { // we will unblacklist it. func (s *ServerStatistics) Success() { s.cancel() - s.backoffCount.Store(0) s.successCounter.Inc() + s.backoffCount.Store(0) if s.statistics.DB != nil { if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil { logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName) @@ -174,12 +174,6 @@ func (s *ServerStatistics) Blacklisted() bool { return s.blacklisted.Load() } -// RemoveBlacklist removes the blacklisted status from the server. -func (s *ServerStatistics) RemoveBlacklist() { - s.cancel() - s.backoffCount.Store(0) -} - // SuccessCount returns the number of successful requests. This is // usually useful in constructing transaction IDs. func (s *ServerStatistics) SuccessCount() uint32 { diff --git a/go.mod b/go.mod index eeae9608f..eefad89e6 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,6 @@ require ( golang.org/x/term v0.0.0-20220919170432-7a66f970e087 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 - gotest.tools/v3 v3.0.3 nhooyr.io/websocket v1.8.7 ) @@ -129,6 +128,7 @@ require ( gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools/v3 v3.0.3 // indirect ) go 1.18 From 088ad1dd21a06ad3c4ae2db2a73936ed3e0d809e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 14 Oct 2022 09:14:54 +0200 Subject: [PATCH 54/61] Fix `outliers whose auth_events are in a different room are correctly rejected` (#2791) Fixes `outliers whose auth_events are in a different room are correctly rejected`, by validating that auth events are all from the same room and not using rejected events for event auth. --- go.mod | 2 +- go.sum | 4 +- roomserver/internal/helpers/auth.go | 20 +++++- roomserver/internal/input/input_events.go | 31 ++++++--- .../internal/input/input_events_test.go | 63 +++++++++++++++++++ sytest-whitelist | 3 +- test/event.go | 7 +++ test/room.go | 9 ++- 8 files changed, 124 insertions(+), 15 deletions(-) create mode 100644 roomserver/internal/input/input_events_test.go diff --git a/go.mod b/go.mod index eefad89e6..23d5655d8 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64 + github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index 0d08ac692..a1069da37 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64 h1:QJmfAPC3P0ZHJzYD/QtbNc5EztKlK1ipRWP5SO/m4jw= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 h1:e5o68MWeU7wjTvvNKmVo655oCYesoNRoPeBb1Xfz54g= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= diff --git a/roomserver/internal/helpers/auth.go b/roomserver/internal/helpers/auth.go index 935a045df..03d8bca0b 100644 --- a/roomserver/internal/helpers/auth.go +++ b/roomserver/internal/helpers/auth.go @@ -19,10 +19,11 @@ import ( "fmt" "sort" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) // CheckForSoftFail returns true if the event should be soft-failed @@ -129,6 +130,12 @@ type authEvents struct { stateKeyNIDMap map[string]types.EventStateKeyNID state stateEntryMap events EventMap + valid bool +} + +// Valid verifies that all auth events are from the same room. +func (ae *authEvents) Valid() bool { + return ae.valid } // Create implements gomatrixserverlib.AuthEventProvider @@ -197,6 +204,7 @@ func loadAuthEvents( needed gomatrixserverlib.StateNeeded, state []types.StateEntry, ) (result authEvents, err error) { + result.valid = true // Look up the numeric IDs for the state keys needed for auth. var neededStateKeys []string neededStateKeys = append(neededStateKeys, needed.Member...) @@ -218,6 +226,16 @@ func loadAuthEvents( if result.events, err = db.Events(ctx, eventNIDs); err != nil { return } + roomID := "" + for _, ev := range result.events { + if roomID == "" { + roomID = ev.RoomID() + } + if ev.RoomID() != roomID { + result.valid = false + break + } + } return } diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index d1b6bc73e..60160e8e5 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -19,9 +19,16 @@ package input import ( "context" "database/sql" + "errors" "fmt" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + fedapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/eventutil" @@ -31,11 +38,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/internal/helpers" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/opentracing/opentracing-go" - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" ) // TODO: Does this value make sense? @@ -196,7 +198,7 @@ func (r *Inputer) processRoomEvent( isRejected := false authEvents := gomatrixserverlib.NewAuthEvents(nil) knownEvents := map[string]*types.Event{} - if err = r.fetchAuthEvents(ctx, logger, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil { + if err = r.fetchAuthEvents(ctx, logger, roomInfo, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil { return fmt.Errorf("r.fetchAuthEvents: %w", err) } @@ -336,7 +338,7 @@ func (r *Inputer) processRoomEvent( // doesn't have any associated state to store and we don't need to // notify anyone about it. if input.Kind == api.KindOutlier { - logger.Debug("Stored outlier") + logger.WithField("rejected", isRejected).Debug("Stored outlier") hooks.Run(hooks.KindNewEventPersisted, headered) return nil } @@ -536,6 +538,7 @@ func (r *Inputer) processStateBefore( func (r *Inputer) fetchAuthEvents( ctx context.Context, logger *logrus.Entry, + roomInfo *types.RoomInfo, event *gomatrixserverlib.HeaderedEvent, auth *gomatrixserverlib.AuthEvents, known map[string]*types.Event, @@ -557,9 +560,19 @@ func (r *Inputer) fetchAuthEvents( continue } ev := authEvents[0] + + isRejected := false + if roomInfo != nil { + isRejected, err = r.DB.IsEventRejected(ctx, roomInfo.RoomNID, ev.EventID()) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("r.DB.IsEventRejected failed: %w", err) + } + } known[authEventID] = &ev // don't take the pointer of the iterated event - if err = auth.AddEvent(ev.Event); err != nil { - return fmt.Errorf("auth.AddEvent: %w", err) + if !isRejected { + if err = auth.AddEvent(ev.Event); err != nil { + return fmt.Errorf("auth.AddEvent: %w", err) + } } } diff --git a/roomserver/internal/input/input_events_test.go b/roomserver/internal/input/input_events_test.go new file mode 100644 index 000000000..818e7715c --- /dev/null +++ b/roomserver/internal/input/input_events_test.go @@ -0,0 +1,63 @@ +package input + +import ( + "testing" + + "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/test" +) + +func Test_EventAuth(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + + // create two rooms, so we can craft "illegal" auth events + room1 := test.NewRoom(t, alice) + room2 := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat)) + + authEventIDs := make([]string, 0, 4) + authEvents := []*gomatrixserverlib.Event{} + + // Add the legal auth events from room2 + for _, x := range room2.Events() { + if x.Type() == gomatrixserverlib.MRoomCreate { + authEventIDs = append(authEventIDs, x.EventID()) + authEvents = append(authEvents, x.Event) + } + if x.Type() == gomatrixserverlib.MRoomPowerLevels { + authEventIDs = append(authEventIDs, x.EventID()) + authEvents = append(authEvents, x.Event) + } + if x.Type() == gomatrixserverlib.MRoomJoinRules { + authEventIDs = append(authEventIDs, x.EventID()) + authEvents = append(authEvents, x.Event) + } + } + + // Add the illegal auth event from room1 (rooms are different) + for _, x := range room1.Events() { + if x.Type() == gomatrixserverlib.MRoomMember { + authEventIDs = append(authEventIDs, x.EventID()) + authEvents = append(authEvents, x.Event) + } + } + + // Craft the illegal join event, with auth events from different rooms + ev := room2.CreateEvent(t, bob, "m.room.member", map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID), test.WithAuthIDs(authEventIDs)) + + // Add the auth events to the allower + allower := gomatrixserverlib.NewAuthEvents(nil) + for _, a := range authEvents { + if err := allower.AddEvent(a); err != nil { + t.Fatalf("allower.AddEvent failed: %v", err) + } + } + + // Finally check that the event is NOT allowed + if err := gomatrixserverlib.Allowed(ev.Event, &allower); err == nil { + t.Fatalf("event should not be allowed, but it was") + } +} diff --git a/sytest-whitelist b/sytest-whitelist index a3218ed70..2bd8b9403 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -743,4 +743,5 @@ User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave Existing members see new member's presence -Inbound federation can return missing events for joined visibility \ No newline at end of file +Inbound federation can return missing events for joined visibility +outliers whose auth_events are in a different room are correctly rejected \ No newline at end of file diff --git a/test/event.go b/test/event.go index 73fc656bd..0c7bf4355 100644 --- a/test/event.go +++ b/test/event.go @@ -30,6 +30,7 @@ type eventMods struct { unsigned interface{} keyID gomatrixserverlib.KeyID privKey ed25519.PrivateKey + authEvents []string } type eventModifier func(e *eventMods) @@ -52,6 +53,12 @@ func WithUnsigned(unsigned interface{}) eventModifier { } } +func WithAuthIDs(evs []string) eventModifier { + return func(e *eventMods) { + e.authEvents = evs + } +} + func WithKeyID(keyID gomatrixserverlib.KeyID) eventModifier { return func(e *eventMods) { e.keyID = keyID diff --git a/test/room.go b/test/room.go index 94eb51bbe..4328bf84f 100644 --- a/test/room.go +++ b/test/room.go @@ -21,8 +21,9 @@ import ( "testing" "time" - "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/internal/eventutil" ) type Preset int @@ -174,11 +175,17 @@ func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, conten if err != nil { t.Fatalf("CreateEvent[%s]: failed to StateNeededForEventBuilder: %s", eventType, err) } + refs, err := eventsNeeded.AuthEventReferences(&r.authEvents) if err != nil { t.Fatalf("CreateEvent[%s]: failed to AuthEventReferences: %s", eventType, err) } builder.AuthEvents = refs + + if len(mod.authEvents) > 0 { + builder.AuthEvents = mod.authEvents + } + ev, err := builder.Build( mod.originServerTS, mod.origin, mod.keyID, mod.privKey, r.Version, From fb44e33909660b5e37f2d422baf4ffa7ddc30b0a Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Fri, 14 Oct 2022 10:37:04 +0200 Subject: [PATCH 55/61] Relax test a bit --- userapi/storage/storage_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index ca7c1bfd2..1538a8138 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -192,19 +192,18 @@ func Test_Devices(t *testing.T) { newName := "new display name" err = db.UpdateDevice(ctx, localpart, deviceWithID.ID, &newName) assert.NoError(t, err, "unable to update device displayname") + updatedAfterTimestamp := time.Now().Unix() err = db.UpdateDeviceLastSeen(ctx, localpart, deviceWithID.ID, "127.0.0.1", "Element Web") assert.NoError(t, err, "unable to update device last seen") deviceWithID.DisplayName = newName deviceWithID.LastSeenIP = "127.0.0.1" - deviceWithID.LastSeenTS = int64(gomatrixserverlib.AsTimestamp(time.Now().Truncate(time.Second))) gotDevice, err = db.GetDeviceByID(ctx, localpart, deviceWithID.ID) assert.NoError(t, err, "unable to get device by id") assert.Equal(t, 2, len(devices)) assert.Equal(t, deviceWithID.DisplayName, gotDevice.DisplayName) assert.Equal(t, deviceWithID.LastSeenIP, gotDevice.LastSeenIP) - truncatedTime := gomatrixserverlib.Timestamp(gotDevice.LastSeenTS).Time().Truncate(time.Second) - assert.Equal(t, gomatrixserverlib.Timestamp(deviceWithID.LastSeenTS), gomatrixserverlib.AsTimestamp(truncatedTime)) + assert.Greater(t, gotDevice.LastSeenTS, updatedAfterTimestamp) // create one more device and remove the devices step by step newDeviceID := util.RandomString(16) From a8bc558a606266b2dc1d1c19cf0c052f3b733679 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 14 Oct 2022 10:38:12 +0200 Subject: [PATCH 56/61] Always add `UnreadNotifications` to joined room reponses (#2793) Fixes a minor bug, where we failed to add `UnreadNotifications` to the join response, if it wasn't in `GetUserUnreadNotificationCountsForRooms`. --- syncapi/streams/stream_notificationdata.go | 3 ++- syncapi/types/types.go | 8 +++++--- sytest-whitelist | 4 +++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go index 5a81fd09a..66ee0ded9 100644 --- a/syncapi/streams/stream_notificationdata.go +++ b/syncapi/streams/stream_notificationdata.go @@ -3,6 +3,7 @@ package streams import ( "context" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -53,7 +54,7 @@ func (p *NotificationDataStreamProvider) IncrementalSync( for roomID, jr := range req.Response.Rooms.Join { counts := countsByRoom[roomID] if counts == nil { - continue + counts = &eventutil.NotificationData{} } jr.UnreadNotifications = &types.UnreadNotifications{ HighlightCount: counts.UnreadHighlightCount, diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 60a74a285..57ce7b6ff 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -492,9 +492,11 @@ func (jr JoinResponse) MarshalJSON() ([]byte, error) { } } - if jr.UnreadNotifications != nil && - jr.UnreadNotifications.NotificationCount == 0 && jr.UnreadNotifications.HighlightCount == 0 { - a.UnreadNotifications = nil + if jr.UnreadNotifications != nil { + // if everything else is nil, also remove UnreadNotifications + if a.State == nil && a.Ephemeral == nil && a.AccountData == nil && a.Timeline == nil && a.Summary == nil { + a.UnreadNotifications = nil + } } return json.Marshal(a) } diff --git a/sytest-whitelist b/sytest-whitelist index 2bd8b9403..93d447d28 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -744,4 +744,6 @@ User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave Existing members see new member's presence Inbound federation can return missing events for joined visibility -outliers whose auth_events are in a different room are correctly rejected \ No newline at end of file +outliers whose auth_events are in a different room are correctly rejected +Messages that notify from another user increment notification_count +Messages that highlight from another user increment unread highlight count \ No newline at end of file From 82d1d434c559285963c3b59e3df04b75337a6a78 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 10:10:25 +0100 Subject: [PATCH 57/61] Update to NATS Server v2.9.3 and nats.go v1.18.0 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 23d5655d8..943760a00 100644 --- a/go.mod +++ b/go.mod @@ -26,8 +26,8 @@ require ( github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 - github.com/nats-io/nats-server/v2 v2.9.2 - github.com/nats-io/nats.go v1.17.0 + github.com/nats-io/nats-server/v2 v2.9.3 + github.com/nats-io/nats.go v1.18.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 diff --git a/go.sum b/go.sum index a1069da37..a8252ab77 100644 --- a/go.sum +++ b/go.sum @@ -422,10 +422,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= -github.com/nats-io/nats-server/v2 v2.9.2 h1:XNDgJgOYYaYlquLdbSHI3xssLipfKUOq3EmYIMNCOsE= -github.com/nats-io/nats-server/v2 v2.9.2/go.mod h1:4sq8wvrpbvSzL1n3ZfEYnH4qeUuIl5W990j3kw13rRk= -github.com/nats-io/nats.go v1.17.0 h1:1jp5BThsdGlN91hW0k3YEfJbfACjiOYtUiLXG0RL4IE= -github.com/nats-io/nats.go v1.17.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nats-server/v2 v2.9.3 h1:HrfzA7G9LNetKkm1z+jU/e9kuAe+E6uaBuuq9EB5sQQ= +github.com/nats-io/nats-server/v2 v2.9.3/go.mod h1:4sq8wvrpbvSzL1n3ZfEYnH4qeUuIl5W990j3kw13rRk= +github.com/nats-io/nats.go v1.18.0 h1:o480Ao6kuSSFyJO75rGTXCEPj7LGkY84C1Ye+Uhm4c0= +github.com/nats-io/nats.go v1.18.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= From f76969831ed2d0165b83b15ddda2560a774063c0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 10:59:32 +0100 Subject: [PATCH 58/61] Update direct dependencies (#2794) This updates a number of Dendrite's dependencies. --- go.mod | 51 +++++++++++------------ go.sum | 128 +++++++++++++++++++++++++++++++-------------------------- 2 files changed, 94 insertions(+), 85 deletions(-) diff --git a/go.mod b/go.mod index 943760a00..7bfd9b45f 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,10 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/blevesearch/bleve/v2 v2.3.4 github.com/codeclysm/extract v2.2.0+incompatible - github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d - github.com/docker/docker v20.10.18+incompatible + github.com/dgraph-io/ristretto v0.1.1 + github.com/docker/docker v20.10.19+incompatible github.com/docker/go-connections v0.4.0 - github.com/getsentry/sentry-go v0.13.0 + github.com/getsentry/sentry-go v0.14.0 github.com/gologme/log v1.3.0 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 @@ -21,9 +21,9 @@ require ( github.com/lib/pq v1.10.7 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 - github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 + github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 - github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c + github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 github.com/nats-io/nats-server/v2 v2.9.3 @@ -43,10 +43,10 @@ require ( github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c go.uber.org/atomic v1.10.0 - golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be + golang.org/x/crypto v0.0.0-20221012134737-56aed061732a golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 - golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 - golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1 + golang.org/x/mobile v0.0.0-20221012134814-c746ac228303 + golang.org/x/net v0.0.0-20221014081412-f15817d10f9b golang.org/x/term v0.0.0-20220919170432-7a66f970e087 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 @@ -54,14 +54,13 @@ require ( ) require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect github.com/RoaringBitmap/roaring v1.2.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.3.3 // indirect github.com/blevesearch/bleve_index_api v1.0.3 // indirect - github.com/blevesearch/geo v0.1.14 // indirect + github.com/blevesearch/geo v0.1.15 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect @@ -69,7 +68,7 @@ require ( github.com/blevesearch/segment v0.9.0 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.1 // indirect - github.com/blevesearch/vellum v1.0.8 // indirect + github.com/blevesearch/vellum v1.0.9 // indirect github.com/blevesearch/zapx/v11 v11.3.5 // indirect github.com/blevesearch/zapx/v12 v12.3.5 // indirect github.com/blevesearch/zapx/v13 v13.3.5 // indirect @@ -80,7 +79,7 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect @@ -92,13 +91,13 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect github.com/klauspost/compress v1.15.11 // indirect - github.com/kr/pretty v0.3.0 // indirect - github.com/lucas-clemente/quic-go v0.29.0 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/lucas-clemente/quic-go v0.29.2 // indirect + github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect + github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/minio/highwayhash v1.0.2 // indirect - github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect + github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect @@ -108,27 +107,27 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/onsi/ginkgo v1.16.5 // indirect - github.com/onsi/gomega v1.17.0 // indirect + github.com/onsi/gomega v1.22.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect + golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect - golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect + golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect + golang.org/x/text v0.3.8 // indirect golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gotest.tools/v3 v3.0.3 // indirect + gotest.tools/v3 v3.4.0 // indirect ) go 1.18 diff --git a/go.sum b/go.sum index a8252ab77..ed76397cc 100644 --- a/go.sum +++ b/go.sum @@ -42,7 +42,6 @@ github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf h1:kjPkmDHUTWUma github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -55,8 +54,8 @@ github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGh github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A= @@ -95,8 +94,8 @@ github.com/blevesearch/bleve/v2 v2.3.4/go.mod h1:Ot0zYum8XQRfPcwhae8bZmNyYubynso github.com/blevesearch/bleve_index_api v1.0.3 h1:DDSWaPXOZZJ2BB73ZTWjKxydAugjwywcqU+91AAqcAg= github.com/blevesearch/bleve_index_api v1.0.3/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= github.com/blevesearch/geo v0.1.13/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc= -github.com/blevesearch/geo v0.1.14 h1:TTDpJN6l9ck/cUYbXSn4aCElNls0Whe44rcQKsB7EfU= -github.com/blevesearch/geo v0.1.14/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc= +github.com/blevesearch/geo v0.1.15 h1:0NybEduqE5fduFRYiUKF0uqybAIFKXYjkBdXKYn7oA4= +github.com/blevesearch/geo v0.1.15/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc= github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= @@ -115,8 +114,9 @@ github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= -github.com/blevesearch/vellum v1.0.8 h1:iMGh4lfxza4BnWO/UJTMPlI3HsK9YawjPv+TteVa9ck= github.com/blevesearch/vellum v1.0.8/go.mod h1:+cpRi/tqq49xUYSQN2P7A5zNSNrS+MscLeeaZ3J46UA= +github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ= +github.com/blevesearch/vellum v1.0.9/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= github.com/blevesearch/zapx/v11 v11.3.5 h1:eBQWQ7huA+mzm0sAGnZDwgGGli7S45EO+N+ObFWssbI= github.com/blevesearch/zapx/v11 v11.3.5/go.mod h1:5UdIa/HRMdeRCiLQOyFESsnqBGiip7vQmYReA9toevU= github.com/blevesearch/zapx/v12 v12.3.5 h1:5pX2hU+R1aZihT7ac1dNWh1n4wqkIM9pZzWp0ANED9s= @@ -157,14 +157,14 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d h1:Wrc3UKTS+cffkOx0xRGFC+ZesNuTfn0ThvEC72N0krk= -github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.18+incompatible h1:SN84VYXTBNGn92T/QwIRPlum9zfemfitN7pbsp26WSc= -github.com/docker/docker v20.10.18+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.19+incompatible h1:lzEmjivyNHFHMNAFLXORMBXyGIhw/UP4DvJwvyKYq64= +github.com/docker/docker v20.10.19+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -184,20 +184,21 @@ github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBav github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= -github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70= +github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= +github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -210,12 +211,12 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -225,6 +226,7 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -353,59 +355,60 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= -github.com/lucas-clemente/quic-go v0.29.0 h1:Vw0mGTfmWqGzh4jx/kMymsIkFK6rErFVmg+t9RLrnZE= -github.com/lucas-clemente/quic-go v0.29.0/go.mod h1:CTcNfLYJS2UuRNB+zcNlgvkjBhxX6Hm3WUxxAQx2mgE= +github.com/lucas-clemente/quic-go v0.29.2 h1:O8Mt0O6LpvEW+wfC40vZdcw0DngwYzoxq5xULZNzSI8= +github.com/lucas-clemente/quic-go v0.29.2/go.mod h1:g6/h9YMmLuU54tL1gW25uIi3VlBp3uv+sBihplIuskE= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= -github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM= github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= +github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= -github.com/marten-seemann/qtls-go1-19 v0.1.0 h1:rLFKD/9mp/uq1SYGYuVZhm83wkmU95pK5df3GufyYYU= -github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= +github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= +github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= -github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= -github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= +github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= +github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 h1:e5o68MWeU7wjTvvNKmVo655oCYesoNRoPeBb1Xfz54g= github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= -github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk= -github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= +github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 h1:lzkSQvBv8TuqKJCPoVwOVvEnARTlua5rrNy/Qw2Vxeo= +github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -454,20 +457,23 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -503,8 +509,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -533,7 +539,6 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -565,8 +570,9 @@ github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -577,8 +583,8 @@ github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= @@ -625,8 +631,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20221012134737-56aed061732a h1:NmSIgad6KjE6VvHciPZuNRTKxGhlPfD6OA87W/PLkqg= +golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -641,8 +647,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b h1:SCE/18RnFsLrjydh/R/s5EVvHoZprqEQUuoxK8q2Pc4= -golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 h1:sBdrWpxhGDdTAYNqbgBLAR+ULAPPhfgncLr1X0lyWtg= +golang.org/x/exp v0.0.0-20221012211006-4de253d81b95/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -661,8 +667,9 @@ golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 h1:3vUV5x5+3LfQbgk7paCM6INOaJG9xXQbn79xoNkwfIk= golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= +golang.org/x/mobile v0.0.0-20221012134814-c746ac228303 h1:K4fp1rDuJBz0FCPAWzIJwnzwNEM7S6yobdZzMrZ/Zws= +golang.org/x/mobile v0.0.0-20221012134814-c746ac228303/go.mod h1:M32cGdzp91A8Ex9qQtyZinr19EYxzkFqDjW2oyHzTDQ= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -717,8 +724,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1 h1:TWZxd/th7FbRSMret2MVQdlI8uT49QEtwZdvJrxjEHU= -golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -741,6 +748,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -763,7 +771,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -787,11 +794,11 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -810,8 +817,10 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 h1:OK7RB6t2WQX54srQQYSXMW8dF5C6/8+oA/s5QBmmto4= +golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w= @@ -823,8 +832,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b h1:NXqSWXSRUSCaFuvitrWtU169I3876zRTalMRbfd6LL0= -golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -879,6 +888,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= @@ -1013,8 +1023,8 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From eac5678449f9134769344a6d7b3761e79eed1328 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 14:53:53 +0100 Subject: [PATCH 59/61] Update dependency now that it is fixed --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 7bfd9b45f..1303d4004 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 - github.com/kardianos/minwinsvc v1.0.0 + github.com/kardianos/minwinsvc v1.0.2 github.com/lib/pq v1.10.7 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 diff --git a/go.sum b/go.sum index ed76397cc..54055f2fd 100644 --- a/go.sum +++ b/go.sum @@ -343,8 +343,8 @@ github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/kardianos/minwinsvc v1.0.0 h1:+JfAi8IBJna0jY2dJGZqi7o15z13JelFIklJCAENALA= -github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc= +github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0= +github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -790,7 +790,6 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From cd8f7e125172629984f60546446e7c9d261b26c0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 15:14:29 +0100 Subject: [PATCH 60/61] Set inactivity threshold on durable consumers in the roomserver input API (#2795) This prevents us from holding onto durable consumers indefinitely for rooms that have long since turned inactive, since they do have a bit of a processing overhead in the NATS Server. If we clear up a consumer and then a room becomes active again, the consumer gets recreated as needed. The threshold is set to 24 hours for now, we can tweak it later if needs be. --- roomserver/internal/input/input.go | 34 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index c47793f0a..f5099ca11 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -89,6 +89,13 @@ type Inputer struct { Queryer *query.Queryer } +// If a room consumer is inactive for a while then we will allow NATS +// to clean it up. This stops us from holding onto durable consumers +// indefinitely for rooms that might no longer be active, since they do +// have an interest overhead in the NATS Server. If the room becomes +// active again then we'll recreate the consumer anyway. +const inactiveThreshold = time.Hour * 24 + type worker struct { phony.Inbox sync.Mutex @@ -125,11 +132,12 @@ func (r *Inputer) startWorkerForRoom(roomID string) { if _, err := w.r.JetStream.AddConsumer( r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent), &nats.ConsumerConfig{ - Durable: consumer, - AckPolicy: nats.AckAllPolicy, - DeliverPolicy: nats.DeliverAllPolicy, - FilterSubject: subject, - AckWait: MaximumMissingProcessingTime + (time.Second * 10), + Durable: consumer, + AckPolicy: nats.AckAllPolicy, + DeliverPolicy: nats.DeliverAllPolicy, + FilterSubject: subject, + AckWait: MaximumMissingProcessingTime + (time.Second * 10), + InactiveThreshold: inactiveThreshold, }, ); err != nil { logrus.WithError(err).Errorf("Failed to create consumer for room %q", w.roomID) @@ -145,6 +153,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) { nats.DeliverAll(), nats.AckWait(MaximumMissingProcessingTime+(time.Second*10)), nats.Bind(r.InputRoomEventTopic, consumer), + nats.InactiveThreshold(inactiveThreshold), ) if err != nil { logrus.WithError(err).Errorf("Failed to subscribe to stream for room %q", w.roomID) @@ -180,6 +189,21 @@ func (r *Inputer) Start() error { nats.ReplayInstant(), nats.BindStream(r.InputRoomEventTopic), ) + + // Make sure that the room consumers have the right config. + stream := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent) + for consumer := range r.JetStream.Consumers(stream) { + switch { + case consumer.Config.Durable == "": + continue // Ignore ephemeral consumers + case consumer.Config.InactiveThreshold != inactiveThreshold: + consumer.Config.InactiveThreshold = inactiveThreshold + if _, cerr := r.JetStream.UpdateConsumer(stream, &consumer.Config); cerr != nil { + logrus.WithError(cerr).Warnf("Failed to update inactive threshold on consumer %q", consumer.Name) + } + } + } + return err } From 81dbad39a3007a67b92f7abd669170a545a1f597 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 15:53:58 +0100 Subject: [PATCH 61/61] Dendrite 0.10.3 commit c6e18c18e93b54c006c6b4d0044aa53a0735a906 Author: Neil Alexander Date: Fri Oct 14 15:42:58 2022 +0100 Changelog and version bump --- CHANGES.md | 17 +++++++++++++++++ internal/version.go | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7552e41e5..eea2c3c7c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,22 @@ # Changelog +## Dendrite 0.10.3 (2022-10-14) + +### Features + +* Event relations are now tracked and support for the `/room/{roomID}/relations/...` client API endpoints have been added +* Support has been added for private read receipts +* The built-in NATS Server has been updated to version 2.9.3 + +### Fixes + +* The `unread_notifications` are now always populated in joined room responses +* The `/get_missing_events` federation API endpoint should now work correctly for rooms with `joined` and `invited` visibility settings, returning redacted events for events that other servers are not allowed to see +* The `/event` client API endpoint now applies history visibility correctly +* Read markers should now be updated much more reliably +* A rare bug in the sync API which could cause some `join` memberships to be incorrectly overwritten by other memberships when working out which rooms to populate has been fixed +* The federation API now correctly updates the joined hosts table during a state rewrite + ## Dendrite 0.10.2 (2022-10-07) ### Features diff --git a/internal/version.go b/internal/version.go index 56b83f852..c888748a8 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 10 - VersionPatch = 2 + VersionPatch = 3 VersionTag = "" // example: "rc1" )