From 34221938ccb1f3a885ac9e5a36b79d3d74850d38 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 16:04:11 +0100 Subject: [PATCH 1/6] Version 0.8.2 (#2386) * Version 0.8.2 * Correct account data position mapping * Try that again * Don't duplicate wait-for-shutdowns --- CHANGES.md | 29 +++++++++++++++++++++++++++++ internal/version.go | 2 +- setup/base/base.go | 4 +--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 831a8969d..6278bcba4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,34 @@ # Changelog +## Dendrite 0.8.2 (2022-04-27) + +### Features + +* Lazy-loading has been added to the `/sync` endpoint, which should speed up syncs considerably +* Filtering has been added to the `/messages` endpoint +* The room summary now contains "heroes" (up to 5 users in the room) for clients to display when no room name is set +* The existing lazy-loading caches will now be used by `/messages` and `/context` so that member events will not be sent to clients more times than necessary +* The account data stream now uses the provided filters +* The built-in NATS Server has been updated to version 2.8.0 +* The `/state` and `/state_ids` endpoints will now return `M_NOT_FOUND` for rejected events +* Repeated calls to the `/redact` endpoint will now be idempotent when a transaction ID is given +* Dendrite should now be able to run as a Windows service under Service Control Manager + +### Fixes + +* Fictitious presence updates will no longer be created for users which have not sent us presence updates, which should speed up complete syncs considerably +* Uploading cross-signing device signatures should now be more reliable, fixing a number of bugs with cross-signing +* All account data should now be sent properly on a complete sync, which should eliminate problems with client settings or key backups appearing to be missing +* Account data will now be limited correctly on incremental syncs, returning the stream position of the most recent update rather than the latest stream position +* Account data will not be sent for parted rooms, which should reduce the number of left/forgotten rooms reappearing in clients as empty rooms +* The TURN username hash has been fixed which should help to resolve some problems when using TURN for voice calls (contributed by [fcwoknhenuxdfiyv](https://github.com/fcwoknhenuxdfiyv)) +* Push rules can no longer be modified using the account data endpoints +* Querying account availability should now work properly in polylith deployments +* A number of bugs with sync filters have been fixed +* A default sync filter will now be used if the request contains a filter ID that does not exist +* The `pushkey_ts` field is now using seconds instead of milliseconds +* A race condition when gracefully shutting down has been fixed, so JetStream should no longer cause the process to exit before other Dendrite components are finished shutting down + ## Dendrite 0.8.1 (2022-04-07) ### Fixes diff --git a/internal/version.go b/internal/version.go index 5227a03bf..2477bc9ac 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 8 - VersionPatch = 1 + VersionPatch = 2 VersionTag = "" // example: "rc1" ) diff --git a/setup/base/base.go b/setup/base/base.go index e67b034a3..7091c6ba5 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -469,14 +469,12 @@ func (b *BaseDendrite) SetupAndServeHTTP( } minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite) - <-b.ProcessContext.WaitForShutdown() + logrus.Infof("Stopping HTTP listeners") _ = internalServ.Shutdown(context.Background()) _ = externalServ.Shutdown(context.Background()) logrus.Infof("Stopped HTTP listeners") - - b.WaitForShutdown() } func (b *BaseDendrite) WaitForShutdown() { From 8d69e2f0b897dd1ecd99c7bd348ad8bfc8999c4e Mon Sep 17 00:00:00 2001 From: 0x1a8510f2 Date: Wed, 27 Apr 2022 20:19:46 +0100 Subject: [PATCH 2/6] Use Go 1.18 to build Docker images (#2391) Go 1.18 has now been released for a while and the CI already tests Dendrite with Go 1.18 so there should be no issues. Go 1.18 brings some performance improvements for ARM via the register calling convention so it makes sense to switch to it. --- build/docker/Dockerfile.monolith | 4 ++-- build/docker/Dockerfile.polylith | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith index 0d2a141ad..891a3a9e0 100644 --- a/build/docker/Dockerfile.monolith +++ b/build/docker/Dockerfile.monolith @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.17-alpine AS base +FROM docker.io/golang:1.18-alpine AS base RUN apk --update --no-cache add bash build-base @@ -23,4 +23,4 @@ COPY --from=base /build/bin/* /usr/bin/ VOLUME /etc/dendrite WORKDIR /etc/dendrite -ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith index c266fd480..ffdc35586 100644 --- a/build/docker/Dockerfile.polylith +++ b/build/docker/Dockerfile.polylith @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.17-alpine AS base +FROM docker.io/golang:1.18-alpine AS base RUN apk --update --no-cache add bash build-base @@ -23,4 +23,4 @@ COPY --from=base /build/bin/* /usr/bin/ VOLUME /etc/dendrite WORKDIR /etc/dendrite -ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] From 74259f296f225510e9fbb6c5aae191c3f86c729e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 27 Apr 2022 21:31:30 +0200 Subject: [PATCH 3/6] Fix #2390 (#2392) Fix duplicate heroes in `/sync` response. --- syncapi/storage/postgres/memberships_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 8c049977f..00223c57a 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -64,7 +64,7 @@ const selectMembershipCountSQL = "" + ") t WHERE t.membership = $3" const selectHeroesSQL = "" + - "SELECT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5" + "SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5" type membershipsStatements struct { upsertMembershipStmt *sql.Stmt From 2ff75b7c806829b211be0c310497728055e898cc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 Apr 2022 11:34:19 +0100 Subject: [PATCH 4/6] Ensure signature map exists (fixes #2393) (#2397) --- keyserver/internal/internal.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index e571c7e56..1677cf8e3 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -319,6 +319,9 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques // JSON, add the signatures and marshal it again, for some reason? for targetUserID, masterKey := range res.MasterKeys { + if masterKey.Signatures == nil { + masterKey.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } for targetKeyID := range masterKey.Keys { sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID) if err != nil { From 6deb10f3f61e4da0360a8910f6b5375a2d31f182 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 Apr 2022 11:45:56 +0100 Subject: [PATCH 5/6] Don't answer expensive federation requests for rooms we no longer belong to (#2398) This includes `/state`, `/state_ids`, `/get_missing_events` and `/backfill`. This should fix #2396. --- federationapi/routing/backfill.go | 6 ++++++ federationapi/routing/eventauth.go | 6 ++++++ federationapi/routing/missingevents.go | 6 ++++++ federationapi/routing/routing.go | 27 ++++++++++++++++++++++++++ federationapi/routing/state.go | 6 ++++++ 5 files changed, 51 insertions(+) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 31005209f..82f6cbabf 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -51,6 +51,12 @@ func Backfill( } } + // If we don't think we belong to this room then don't waste the effort + // responding to expensive requests for it. + if err := ErrorIfLocalServerNotInRoom(httpReq.Context(), rsAPI, roomID); err != nil { + return *err + } + // Check if all of the required parameters are there. eIDs, exists = httpReq.URL.Query()["v"] if !exists { diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go index 0a03a0cb4..e83cb8ad2 100644 --- a/federationapi/routing/eventauth.go +++ b/federationapi/routing/eventauth.go @@ -30,6 +30,12 @@ func GetEventAuth( roomID string, eventID string, ) util.JSONResponse { + // If we don't think we belong to this room then don't waste the effort + // responding to expensive requests for it. + if err := ErrorIfLocalServerNotInRoom(ctx, rsAPI, roomID); err != nil { + return *err + } + event, resErr := fetchEvent(ctx, rsAPI, eventID) if resErr != nil { return *resErr diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index dd3df7aa9..b826d69c4 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -45,6 +45,12 @@ func GetMissingEvents( } } + // If we don't think we belong to this room then don't waste the effort + // responding to expensive requests for it. + if err := ErrorIfLocalServerNotInRoom(httpReq.Context(), rsAPI, roomID); err != nil { + return *err + } + var eventsResponse api.QueryMissingEventsResponse if err := rsAPI.QueryMissingEvents( httpReq.Context(), &api.QueryMissingEventsRequest{ diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index a085ed780..6d24c8b40 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -15,6 +15,8 @@ package routing import ( + "context" + "fmt" "net/http" "github.com/gorilla/mux" @@ -24,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -491,3 +494,27 @@ func Setup( }), ).Methods(http.MethodGet) } + +func ErrorIfLocalServerNotInRoom( + ctx context.Context, + rsAPI api.RoomserverInternalAPI, + roomID string, +) *util.JSONResponse { + // Check if we think we're in this room. If we aren't then + // we won't waste CPU cycles serving this request. + joinedReq := &api.QueryServerJoinedToRoomRequest{ + RoomID: roomID, + } + joinedRes := &api.QueryServerJoinedToRoomResponse{} + if err := rsAPI.QueryServerJoinedToRoom(ctx, joinedReq, joinedRes); err != nil { + res := util.ErrorResponse(err) + return &res + } + if !joinedRes.IsInRoom { + return &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound(fmt.Sprintf("This server is not joined to room %s", roomID)), + } + } + return nil +} diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index a202c92c2..e2b67776a 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -101,6 +101,12 @@ func getState( roomID string, eventID string, ) (stateEvents, authEvents []*gomatrixserverlib.HeaderedEvent, errRes *util.JSONResponse) { + // If we don't think we belong to this room then don't waste the effort + // responding to expensive requests for it. + if err := ErrorIfLocalServerNotInRoom(ctx, rsAPI, roomID); err != nil { + return nil, nil, err + } + event, resErr := fetchEvent(ctx, rsAPI, eventID) if resErr != nil { return nil, nil, resErr From 65034d1f227de45e88d39ec5a3e83d854e840875 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 Apr 2022 11:46:15 +0100 Subject: [PATCH 6/6] Unlist test since it no longer seems to be flakey (hopefully?) --- sytest-blacklist | 1 - 1 file changed, 1 deletion(-) diff --git a/sytest-blacklist b/sytest-blacklist index 713a5b631..be0826eee 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -48,4 +48,3 @@ Notifications can be viewed with GET /notifications # More flakey If remote user leaves room we no longer receive device updates -Local device key changes get to remote servers