diff --git a/CHANGES.md b/CHANGES.md index 1ed87824a..ba14dd07a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,25 @@ # Changelog +## Dendrite 0.10.5 (2022-10-31) + +### Features + +* It is now possible to use hCaptcha instead of reCAPTCHA for protecting registration +* A new `auto_join_rooms` configuration option has been added for automatically joining new users to a set of rooms +* A new `/_dendrite/admin/downloadState/{serverName}/{roomID}` endpoint has been added, which allows a server administrator to attempt to repair a room with broken room state by downloading a state snapshot from another federated server in the room + +### Fixes + +* Querying cross-signing keys for users should now be considerably faster +* A bug in state resolution where some events were not correctly selected for third-party invites has been fixed +* A bug in state resolution which could result in `not in room` event rejections has been fixed +* When accepting a DM invite, it should now be possible to see messages that were sent before the invite was accepted +* Claiming remote E2EE one-time keys has been refactored and should be more reliable now +* Various fixes have been made to the `/members` endpoint, which may help with E2EE reliability and clients rendering memberships +* A race condition in the federation API destination queues has been fixed when associating queued events with remote server destinations +* A bug in the sync API where too many events were selected resulting in high CPU usage has been fixed +* Configuring the avatar URL for the Server Notices user should work correctly now + ## Dendrite 0.10.4 (2022-10-21) ### Features diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 69bca13be..9088f7716 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -191,3 +191,43 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien JSON: struct{}{}, } } + +func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + roomID, ok := vars["roomID"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting room ID."), + } + } + serverName, ok := vars["serverName"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting remote server name."), + } + } + res := &roomserverAPI.PerformAdminDownloadStateResponse{} + if err := rsAPI.PerformAdminDownloadState( + req.Context(), + &roomserverAPI.PerformAdminDownloadStateRequest{ + UserID: device.UserID, + RoomID: roomID, + ServerName: gomatrixserverlib.ServerName(serverName), + }, + res, + ); err != nil { + return jsonerror.InternalAPIError(req.Context(), err) + } + if err := res.Error; err != nil { + return err.JSONResponse() + } + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f1979928e..7ac7ccff8 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -174,6 +174,12 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) + dendriteAdminRouter.Handle("/admin/downloadState/{serverName}/{roomID}", + httputil.MakeAdminAPI("admin_download_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return AdminDownloadState(req, cfg, device, rsAPI) + }), + ).Methods(http.MethodGet, http.MethodOptions) + dendriteAdminRouter.Handle("/admin/fulltext/reindex", httputil.MakeAdminAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return AdminReindex(req, cfg, device, natsClient) diff --git a/go.mod b/go.mod index e749d7b49..e31af9bf1 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa + github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74 github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index e4b9597d3..1d922a2fc 100644 --- a/go.sum +++ b/go.sum @@ -423,8 +423,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa h1:S98DShDv3sn7O4n4HjtJOejypseYVpv1R/XPg+cDnfI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74 h1:I4LUlFqxZ72m3s9wIvUIV2FpprsxW28dO/0lAgepCZY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 h1:nAT5w41Q9uWTSnpKW55/hBwP91j2IFYPDRs0jJ8TyFI= github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= diff --git a/internal/version.go b/internal/version.go index 5d739a45d..7254ab102 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 10 - VersionPatch = 4 + VersionPatch = 5 VersionTag = "" // example: "rc1" ) diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 403bbe8be..a1373a62b 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -150,6 +150,7 @@ type ClientRoomserverAPI interface { PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse) error PerformAdminEvacuateRoom(ctx context.Context, req *PerformAdminEvacuateRoomRequest, res *PerformAdminEvacuateRoomResponse) error PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse) error + PerformAdminDownloadState(ctx context.Context, req *PerformAdminDownloadStateRequest, res *PerformAdminDownloadStateResponse) error PerformPeek(ctx context.Context, req *PerformPeekRequest, res *PerformPeekResponse) error PerformUnpeek(ctx context.Context, req *PerformUnpeekRequest, res *PerformUnpeekResponse) error PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index 8bef35379..342a3904c 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -131,6 +131,16 @@ func (t *RoomserverInternalAPITrace) PerformAdminEvacuateUser( return err } +func (t *RoomserverInternalAPITrace) PerformAdminDownloadState( + ctx context.Context, + req *PerformAdminDownloadStateRequest, + res *PerformAdminDownloadStateResponse, +) error { + err := t.Impl.PerformAdminDownloadState(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("PerformAdminDownloadState req=%+v res=%+v", js(req), js(res)) + return err +} + func (t *RoomserverInternalAPITrace) PerformInboundPeek( ctx context.Context, req *PerformInboundPeekRequest, diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 1442a4b09..2536a4bb5 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -237,3 +237,13 @@ type PerformAdminEvacuateUserResponse struct { Affected []string `json:"affected"` Error *PerformError } + +type PerformAdminDownloadStateRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + ServerName gomatrixserverlib.ServerName `json:"server_name"` +} + +type PerformAdminDownloadStateResponse struct { + Error *PerformError `json:"error,omitempty"` +} diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go index 6a6d51b0a..0406fc8f4 100644 --- a/roomserver/internal/perform/perform_admin.go +++ b/roomserver/internal/perform/perform_admin.go @@ -236,3 +236,145 @@ func (r *Admin) PerformAdminEvacuateUser( } return nil } + +func (r *Admin) PerformAdminDownloadState( + ctx context.Context, + req *api.PerformAdminDownloadStateRequest, + res *api.PerformAdminDownloadStateResponse, +) error { + roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("r.DB.RoomInfo: %s", err), + } + return nil + } + + if roomInfo == nil || roomInfo.IsStub() { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("room %q not found", req.RoomID), + } + return nil + } + + fwdExtremities, _, depth, err := r.DB.LatestEventIDs(ctx, roomInfo.RoomNID) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("r.DB.LatestEventIDs: %s", err), + } + return nil + } + + authEventMap := map[string]*gomatrixserverlib.Event{} + stateEventMap := map[string]*gomatrixserverlib.Event{} + + for _, fwdExtremity := range fwdExtremities { + var state gomatrixserverlib.RespState + state, err = r.Inputer.FSAPI.LookupState(ctx, req.ServerName, req.RoomID, fwdExtremity.EventID, roomInfo.RoomVersion) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("r.Inputer.FSAPI.LookupState (%q): %s", fwdExtremity.EventID, err), + } + return nil + } + for _, authEvent := range state.AuthEvents.UntrustedEvents(roomInfo.RoomVersion) { + if err = authEvent.VerifyEventSignatures(ctx, r.Inputer.KeyRing); err != nil { + continue + } + authEventMap[authEvent.EventID()] = authEvent + } + for _, stateEvent := range state.StateEvents.UntrustedEvents(roomInfo.RoomVersion) { + if err = stateEvent.VerifyEventSignatures(ctx, r.Inputer.KeyRing); err != nil { + continue + } + stateEventMap[stateEvent.EventID()] = stateEvent + } + } + + authEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(authEventMap)) + stateEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(stateEventMap)) + stateIDs := make([]string, 0, len(stateEventMap)) + + for _, authEvent := range authEventMap { + authEvents = append(authEvents, authEvent.Headered(roomInfo.RoomVersion)) + } + for _, stateEvent := range stateEventMap { + stateEvents = append(stateEvents, stateEvent.Headered(roomInfo.RoomVersion)) + stateIDs = append(stateIDs, stateEvent.EventID()) + } + + builder := &gomatrixserverlib.EventBuilder{ + Type: "org.matrix.dendrite.state_download", + Sender: req.UserID, + RoomID: req.RoomID, + Content: gomatrixserverlib.RawJSON("{}"), + } + + eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("gomatrixserverlib.StateNeededForEventBuilder: %s", err), + } + return nil + } + + queryRes := &api.QueryLatestEventsAndStateResponse{ + RoomExists: true, + RoomVersion: roomInfo.RoomVersion, + LatestEvents: fwdExtremities, + StateEvents: stateEvents, + Depth: depth, + } + + ev, err := eventutil.BuildEvent(ctx, builder, r.Cfg.Matrix, time.Now(), &eventsNeeded, queryRes) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("eventutil.BuildEvent: %s", err), + } + return nil + } + + inputReq := &api.InputRoomEventsRequest{ + Asynchronous: false, + } + inputRes := &api.InputRoomEventsResponse{} + + for _, authEvent := range append(authEvents, stateEvents...) { + inputReq.InputRoomEvents = append(inputReq.InputRoomEvents, api.InputRoomEvent{ + Kind: api.KindOutlier, + Event: authEvent, + }) + } + + inputReq.InputRoomEvents = append(inputReq.InputRoomEvents, api.InputRoomEvent{ + Kind: api.KindNew, + Event: ev, + Origin: r.Cfg.Matrix.ServerName, + HasState: true, + StateEventIDs: stateIDs, + SendAsServer: string(r.Cfg.Matrix.ServerName), + }) + + if err := r.Inputer.InputRoomEvents(ctx, inputReq, inputRes); err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("r.Inputer.InputRoomEvents: %s", err), + } + return nil + } + + if inputRes.ErrMsg != "" { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: inputRes.ErrMsg, + } + } + + return nil +} diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index a1dfc6aac..1bd1b3fb7 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -27,18 +27,19 @@ const ( RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" // Perform operations - RoomserverPerformInvitePath = "/roomserver/performInvite" - RoomserverPerformPeekPath = "/roomserver/performPeek" - RoomserverPerformUnpeekPath = "/roomserver/performUnpeek" - RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade" - RoomserverPerformJoinPath = "/roomserver/performJoin" - RoomserverPerformLeavePath = "/roomserver/performLeave" - RoomserverPerformBackfillPath = "/roomserver/performBackfill" - RoomserverPerformPublishPath = "/roomserver/performPublish" - RoomserverPerformInboundPeekPath = "/roomserver/performInboundPeek" - RoomserverPerformForgetPath = "/roomserver/performForget" - RoomserverPerformAdminEvacuateRoomPath = "/roomserver/performAdminEvacuateRoom" - RoomserverPerformAdminEvacuateUserPath = "/roomserver/performAdminEvacuateUser" + RoomserverPerformInvitePath = "/roomserver/performInvite" + RoomserverPerformPeekPath = "/roomserver/performPeek" + RoomserverPerformUnpeekPath = "/roomserver/performUnpeek" + RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade" + RoomserverPerformJoinPath = "/roomserver/performJoin" + RoomserverPerformLeavePath = "/roomserver/performLeave" + RoomserverPerformBackfillPath = "/roomserver/performBackfill" + RoomserverPerformPublishPath = "/roomserver/performPublish" + RoomserverPerformInboundPeekPath = "/roomserver/performInboundPeek" + RoomserverPerformForgetPath = "/roomserver/performForget" + RoomserverPerformAdminEvacuateRoomPath = "/roomserver/performAdminEvacuateRoom" + RoomserverPerformAdminEvacuateUserPath = "/roomserver/performAdminEvacuateUser" + RoomserverPerformAdminDownloadStatePath = "/roomserver/performAdminDownloadState" // Query operations RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" @@ -261,6 +262,17 @@ func (h *httpRoomserverInternalAPI) PerformAdminEvacuateRoom( ) } +func (h *httpRoomserverInternalAPI) PerformAdminDownloadState( + ctx context.Context, + request *api.PerformAdminDownloadStateRequest, + response *api.PerformAdminDownloadStateResponse, +) error { + return httputil.CallInternalRPCAPI( + "PerformAdminDownloadState", h.roomserverURL+RoomserverPerformAdminDownloadStatePath, + h.httpClient, ctx, request, response, + ) +} + func (h *httpRoomserverInternalAPI) PerformAdminEvacuateUser( ctx context.Context, request *api.PerformAdminEvacuateUserRequest, diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 3b688174a..4d37e90b5 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -65,6 +65,11 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateUser", r.PerformAdminEvacuateUser), ) + internalAPIMux.Handle( + RoomserverPerformAdminDownloadStatePath, + httputil.MakeInternalRPCAPI("RoomserverPerformAdminDownloadState", r.PerformAdminDownloadState), + ) + internalAPIMux.Handle( RoomserverQueryPublishedRoomsPath, httputil.MakeInternalRPCAPI("RoomserverQueryPublishedRooms", r.QueryPublishedRooms), diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 3ab0f4ed4..90f401481 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -251,8 +251,10 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err) } - recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents) - delta.StateEvents = removeDuplicates(delta.StateEvents, recentEvents) // roll back + recentEvents := gomatrixserverlib.HeaderedReverseTopologicalOrdering( + snapshot.StreamEventsToEvents(device, recentStreamEvents), + gomatrixserverlib.TopologicalOrderByPrevEvents, + ) prevBatch, err := snapshot.GetBackwardTopologyPos(ctx, recentStreamEvents) if err != nil { return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err) @@ -263,10 +265,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( return r.To, nil } - // Sort the events so that we can pick out the latest events from both sections. - recentEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(recentEvents, gomatrixserverlib.TopologicalOrderByPrevEvents) - delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(delta.StateEvents, gomatrixserverlib.TopologicalOrderByAuthEvents) - // Work out what the highest stream position is for all of the events in this // room that were returned. latestPosition := r.To @@ -314,6 +312,14 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( limited = true } + // Now that we've filtered the timeline, work out which state events are still + // left. Anything that appears in the filtered timeline will be removed from the + // "state" section and kept in "timeline". + delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering( + removeDuplicates(delta.StateEvents, recentEvents), + gomatrixserverlib.TopologicalOrderByAuthEvents, + ) + if len(delta.StateEvents) > 0 { updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID()) } @@ -507,7 +513,6 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( // transaction IDs for complete syncs, but we do it anyway because Sytest demands it for: // "Can sync a room with a message with a transaction id" - which does a complete sync to check. recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents) - stateEvents = removeDuplicates(stateEvents, recentEvents) events := recentEvents // Only apply history visibility checks if the response is for joined rooms @@ -521,7 +526,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. limited = limited && len(events) == len(recentEvents) - + stateEvents = removeDuplicates(stateEvents, recentEvents) if stateFilter.LazyLoadMembers { if err != nil { return nil, err diff --git a/sytest-blacklist b/sytest-blacklist index 14edf398a..e2859dcb6 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -40,4 +40,9 @@ Accesing an AS-hosted room alias asks the AS server Guest users can join guest_access rooms # This will fail in HTTP API mode, so blacklisted for now -If a device list update goes missing, the server resyncs on the next one \ No newline at end of file + +If a device list update goes missing, the server resyncs on the next one + +# Might be a bug in the test because leaves do appear :-( + +Leaves are present in non-gapped incremental syncs diff --git a/sytest-whitelist b/sytest-whitelist index 6e4500d06..28235b772 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -699,7 +699,7 @@ We do send redundant membership state across incremental syncs if asked Rejecting invite over federation doesn't break incremental /sync Gapped incremental syncs include all state changes Old leaves are present in gapped incremental syncs -Leaves are present in non-gapped incremental syncs +#Leaves are present in non-gapped incremental syncs Members from the gap are included in gappy incr LL sync Presence can be set from sync /state returns M_NOT_FOUND for a rejected message event