From 534c29ab02e4566712bcc7eb9912b1a7a21e0ae4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jan 2021 12:58:48 +0000 Subject: [PATCH 1/6] Log event ID on consumer errors (fixes #1714) --- federationsender/consumers/roomserver.go | 1 + syncapi/consumers/roomserver.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index b53cb251b..513919c6f 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -102,6 +102,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { default: // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ + "event_id": ev.EventID(), "event": string(ev.JSON()), "add": output.NewRoomEvent.AddsStateEventIDs, "del": output.NewRoomEvent.RemovesStateEventIDs, diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 1d47b73a6..a8cc5f710 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -173,6 +173,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ + "event_id": ev.EventID(), "event": string(ev.JSON()), log.ErrorKey: err, "add": msg.AddsStateEventIDs, @@ -215,6 +216,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ + "event_id": ev.EventID(), "event": string(ev.JSON()), log.ErrorKey: err, }).Panicf("roomserver output log: write old event failure") @@ -276,6 +278,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ + "event_id": msg.Event.EventID(), "event": string(msg.Event.JSON()), "pdupos": pduPos, log.ErrorKey: err, From dd1e31bee7b96c0e7e1697b21f7f1119b5cc4622 Mon Sep 17 00:00:00 2001 From: sumitks866 <31448420+sumitks866@users.noreply.github.com> Date: Mon, 18 Jan 2021 18:39:28 +0530 Subject: [PATCH 2/6] Mention unstable features in /_matrix/client/versions (#1710) * Mention unstable features in /_matrix/client/versions * list enabled mscs * Don't update go.mod/go.sum * goimports Co-authored-by: Neil Alexander --- clientapi/routing/routing.go | 12 +++++++++--- setup/config/config.go | 1 + setup/config/config_clientapi.go | 2 ++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 8dbfc551d..7c320253d 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -62,13 +62,19 @@ func Setup( rateLimits := newRateLimits(&cfg.RateLimiting) userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg) + unstableFeatures := make(map[string]bool) + for _, msc := range cfg.MSCs.MSCs { + unstableFeatures["org.matrix."+msc] = true + } + publicAPIMux.Handle("/versions", httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { return util.JSONResponse{ Code: http.StatusOK, JSON: struct { - Versions []string `json:"versions"` - }{[]string{ + Versions []string `json:"versions"` + UnstableFeatures map[string]bool `json:"unstable_features"` + }{Versions: []string{ "r0.0.1", "r0.1.0", "r0.2.0", @@ -76,7 +82,7 @@ func Setup( "r0.4.0", "r0.5.0", "r0.6.1", - }}, + }, UnstableFeatures: unstableFeatures}, } }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/setup/config/config.go b/setup/config/config.go index b8b12d0c1..b91144078 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -344,6 +344,7 @@ func (c *Dendrite) Wiring() { c.ClientAPI.Derived = &c.Derived c.AppServiceAPI.Derived = &c.Derived + c.ClientAPI.MSCs = &c.MSCs } // Error returns a string detailing how many errors were contained within a diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 521154911..c7cb9c33e 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -37,6 +37,8 @@ type ClientAPI struct { // Rate-limiting options RateLimiting RateLimiting `yaml:"rate_limiting"` + + MSCs *MSCs `yaml:"mscs"` } func (c *ClientAPI) Defaults() { From 244ff0dccb64fbdf29b6cbf8a7e02a81b92c7330 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jan 2021 13:21:33 +0000 Subject: [PATCH 3/6] Don't create so many state snapshots when updating forward extremities (#1718) * Light-weight checking of state changes when updating forward extremities * Only do this for non-state events, since state events will always result in state change at extremities --- .../internal/input/input_latest_events.go | 40 +++++++++++++++++-- roomserver/state/state.go | 4 +- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index e5ec8c4d7..c9264a27d 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -100,7 +100,8 @@ type latestEventsUpdater struct { // The eventID of the event that was processed before this one. lastEventIDSent string // The latest events in the room after processing this event. - latest []types.StateAtEventAndReference + oldLatest []types.StateAtEventAndReference + latest []types.StateAtEventAndReference // The state entries removed from and added to the current state of the // room as a result of processing this event. They are sorted lists. removed []types.StateEntry @@ -123,10 +124,10 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // state snapshot from somewhere else, e.g. a federated room join, // then start with an empty set - none of the forward extremities // that we knew about before matter anymore. - oldLatest := []types.StateAtEventAndReference{} + u.oldLatest = []types.StateAtEventAndReference{} if !u.rewritesState { u.oldStateNID = u.updater.CurrentStateSnapshotNID() - oldLatest = u.updater.LatestEvents() + u.oldLatest = u.updater.LatestEvents() } // If the event has already been written to the output log then we @@ -140,7 +141,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // Work out what the latest events are. This will include the new // event if it is not already referenced. extremitiesChanged, err := u.calculateLatest( - oldLatest, u.event, + u.oldLatest, u.event, types.StateAtEventAndReference{ EventReference: u.event.EventReference(), StateAtEvent: u.stateAtEvent, @@ -200,6 +201,37 @@ func (u *latestEventsUpdater) latestState() error { var err error roomState := state.NewStateResolution(u.api.DB, *u.roomInfo) + // Work out if the state at the extremities has actually changed + // or not. If they haven't then we won't bother doing all of the + // hard work. + if u.event.StateKey() == nil { + stateChanged := false + oldStateNIDs := make([]types.StateSnapshotNID, 0, len(u.oldLatest)) + newStateNIDs := make([]types.StateSnapshotNID, 0, len(u.latest)) + for _, old := range u.oldLatest { + oldStateNIDs = append(oldStateNIDs, old.BeforeStateSnapshotNID) + } + for _, new := range u.latest { + newStateNIDs = append(newStateNIDs, new.BeforeStateSnapshotNID) + } + oldStateNIDs = state.UniqueStateSnapshotNIDs(oldStateNIDs) + newStateNIDs = state.UniqueStateSnapshotNIDs(newStateNIDs) + if len(oldStateNIDs) != len(newStateNIDs) { + stateChanged = true + } else { + for i := range oldStateNIDs { + if oldStateNIDs[i] != newStateNIDs[i] { + stateChanged = true + break + } + } + } + if !stateChanged { + u.newStateNID = u.oldStateNID + return nil + } + } + // Get a list of the current latest events. This may or may not // include the new event from the input path, depending on whether // it is a forward extremity or not. diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 87715af42..953276b24 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -116,7 +116,7 @@ func (v StateResolution) LoadCombinedStateAfterEvents( // Deduplicate the IDs before passing them to the database. // There could be duplicates because the events could be state events where // the snapshot of the room state before them was the same. - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, UniqueStateSnapshotNIDs(stateNIDs)) if err != nil { return nil, fmt.Errorf("v.db.StateBlockNIDs: %w", err) } @@ -1103,7 +1103,7 @@ func (s stateNIDSorter) Len() int { return len(s) } func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { +func UniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { return nids[:util.SortAndUnique(stateNIDSorter(nids))] } From 67c89b3c5ed8c4c945067ad640bf15de3c49a027 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jan 2021 13:37:49 +0000 Subject: [PATCH 4/6] Version 0.3.6 --- .github/workflows/docker-hub.yml | 11 ++++++----- CHANGES.md | 21 ++++++++++++++++++++- internal/version.go | 2 +- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 84745f7b2..2435fde2b 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -7,7 +7,8 @@ on: types: [published] env: - DOCKER_HUB_USER: matrixdotorg + DOCKER_NAMESPACE: matrixdotorg + DOCKER_HUB_USER: neilalexander PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 jobs: @@ -37,8 +38,8 @@ jobs: platforms: ${{ env.PLATFORMS }} push: true tags: | - ${{ env.DOCKER_HUB_USER }}/dendrite-monolith:latest - ${{ env.DOCKER_HUB_USER }}/dendrite-monolith:${{ env.RELEASE_VERSION }} + ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }} Polylith: runs-on: ubuntu-latest @@ -66,5 +67,5 @@ jobs: platforms: ${{ env.PLATFORMS }} push: true tags: | - ${{ env.DOCKER_HUB_USER }}/dendrite-polylith:latest - ${{ env.DOCKER_HUB_USER }}/dendrite-polylith:${{ env.RELEASE_VERSION }} + ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} diff --git a/CHANGES.md b/CHANGES.md index ea754a003..a91dea644 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,12 +1,31 @@ # Changelog +## Dendrite 0.3.6 (2021-01-18) + +### Features + +* Experimental support for MSC2946 (Spaces Summary) has been merged +* Send-to-device messages have been refactored and now take advantage of having their own stream position, making delivery more reliable +* Unstable features and MSCs are now listed in `/versions` (contributed by [sumitks866](https://github.com/sumitks866)) +* Well-known and DNS SRV record results for federated servers are now cached properly, improving outbound federation performance and reducing traffic + +### Fixes + +* Updating forward extremities will no longer result in so many unnecessary state snapshots, reducing on-going disk usage in the roomserver database +* Pagination tokens for `/messages` have been fixed, which should improve the reliability of scrollback/pagination +* Dendrite now avoids returning `null`s in fields of the `/sync` response, and omitting some fields altogether when not needed, which should fix sync issues with Element Android +* Requests for user device lists now time out quicker, which prevents federated `/send` requests from also timing out in many cases +* Empty push rules are no longer sent over and over again in `/sync` +* An integer overflow in the device list updater which could result in panics on 32-bit platforms has been fixed (contributed by [Lesterpig](https://github.com/Lesterpig)) +* Event IDs are now logged properly in federation sender and sync API consumer errors + ## Dendrite 0.3.5 (2021-01-11) ### Features * All `/sync` streams are now logically separate after a refactoring exercise -## Fixes +### Fixes * Event references are now deeply checked properly when calculating forward extremities, reducing the amount of forward extremities in most cases, which improves RAM utilisation and reduces the work done by state resolution * Sync no longer sends incorrect `next_batch` tokens with old stream positions, reducing flashbacks of old messages in clients diff --git a/internal/version.go b/internal/version.go index 2a65bfbfe..bb6d7038b 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 3 - VersionPatch = 5 + VersionPatch = 6 VersionTag = "" // example: "rc1" ) From f6151b6290b2d7ab23af1c67d3e9fb4c7a08a955 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jan 2021 14:00:47 +0000 Subject: [PATCH 5/6] Use dendritegithub for Docker Hub images --- .github/workflows/docker-hub.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 2435fde2b..0322866d7 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -8,7 +8,7 @@ on: env: DOCKER_NAMESPACE: matrixdotorg - DOCKER_HUB_USER: neilalexander + DOCKER_HUB_USER: dendritegithub PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 jobs: From ccfcb2d2808f4daa960921174a40b52b956d3a2a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jan 2021 15:09:54 +0000 Subject: [PATCH 6/6] Create bin dir --- build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.sh b/build.sh index 09ecb61ca..a49814084 100755 --- a/build.sh +++ b/build.sh @@ -17,6 +17,8 @@ else export FLAGS="" fi +mkdir -p bin + CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/... CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs