Merge pull request #53 from HereNotThere/sync-upstream

Sync upstream changes

CHANGES.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## Dendrite 0.10.5 (2022-10-31)
+
+### Features
+
+* It is now possible to use hCaptcha instead of reCAPTCHA for protecting registration
+* A new `auto_join_rooms` configuration option has been added for automatically joining new users to a set of rooms
+* A new `/_dendrite/admin/downloadState/{serverName}/{roomID}` endpoint has been added, which allows a server administrator to attempt to repair a room with broken room state by downloading a state snapshot from another federated server in the room
+
+### Fixes
+
+* Querying cross-signing keys for users should now be considerably faster
+* A bug in state resolution where some events were not correctly selected for third-party invites has been fixed
+* A bug in state resolution which could result in `not in room` event rejections has been fixed
+* When accepting a DM invite, it should now be possible to see messages that were sent before the invite was accepted
+* Claiming remote E2EE one-time keys has been refactored and should be more reliable now
+* Various fixes have been made to the `/members` endpoint, which may help with E2EE reliability and clients rendering memberships
+* A race condition in the federation API destination queues has been fixed when associating queued events with remote server destinations
+* A bug in the sync API where too many events were selected resulting in high CPU usage has been fixed
+* Configuring the avatar URL for the Server Notices user should work correctly now
+
 ## Dendrite 0.10.4 (2022-10-21)
 
 ### Features

clientapi/routing/admin.go

@@ -191,3 +191,43 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien
 		JSON: struct{}{},
 	}
 }
+
+func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	roomID, ok := vars["roomID"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting room ID."),
+		}
+	}
+	serverName, ok := vars["serverName"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting remote server name."),
+		}
+	}
+	res := &roomserverAPI.PerformAdminDownloadStateResponse{}
+	if err := rsAPI.PerformAdminDownloadState(
+		req.Context(),
+		&roomserverAPI.PerformAdminDownloadStateRequest{
+			UserID:     device.UserID,
+			RoomID:     roomID,
+			ServerName: gomatrixserverlib.ServerName(serverName),
+		},
+		res,
+	); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{},
+	}
+}

clientapi/routing/routing.go

@@ -174,6 +174,12 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/downloadState/{serverName}/{roomID}",
+		httputil.MakeAdminAPI("admin_download_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminDownloadState(req, cfg, device, rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	dendriteAdminRouter.Handle("/admin/fulltext/reindex",
 		httputil.MakeAdminAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return AdminReindex(req, cfg, device, natsClient)

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74
 	github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15

go.sum

@@ -423,8 +423,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa h1:S98DShDv3sn7O4n4HjtJOejypseYVpv1R/XPg+cDnfI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74 h1:I4LUlFqxZ72m3s9wIvUIV2FpprsxW28dO/0lAgepCZY=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 h1:nAT5w41Q9uWTSnpKW55/hBwP91j2IFYPDRs0jJ8TyFI=
 github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 10
-	VersionPatch = 4
+	VersionPatch = 5
 	VersionTag   = "" // example: "rc1"
 )
 

roomserver/api/api.go

@@ -150,6 +150,7 @@ type ClientRoomserverAPI interface {
 	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse) error
 	PerformAdminEvacuateRoom(ctx context.Context, req *PerformAdminEvacuateRoomRequest, res *PerformAdminEvacuateRoomResponse) error
 	PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse) error
+	PerformAdminDownloadState(ctx context.Context, req *PerformAdminDownloadStateRequest, res *PerformAdminDownloadStateResponse) error
 	PerformPeek(ctx context.Context, req *PerformPeekRequest, res *PerformPeekResponse) error
 	PerformUnpeek(ctx context.Context, req *PerformUnpeekRequest, res *PerformUnpeekResponse) error
 	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error

roomserver/api/api_trace.go

@@ -131,6 +131,16 @@ func (t *RoomserverInternalAPITrace) PerformAdminEvacuateUser(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) PerformAdminDownloadState(
+	ctx context.Context,
+	req *PerformAdminDownloadStateRequest,
+	res *PerformAdminDownloadStateResponse,
+) error {
+	err := t.Impl.PerformAdminDownloadState(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformAdminDownloadState req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
 func (t *RoomserverInternalAPITrace) PerformInboundPeek(
 	ctx context.Context,
 	req *PerformInboundPeekRequest,

roomserver/api/perform.go

@@ -237,3 +237,13 @@ type PerformAdminEvacuateUserResponse struct {
 	Affected []string `json:"affected"`
 	Error    *PerformError
 }
+
+type PerformAdminDownloadStateRequest struct {
+	RoomID     string                       `json:"room_id"`
+	UserID     string                       `json:"user_id"`
+	ServerName gomatrixserverlib.ServerName `json:"server_name"`
+}
+
+type PerformAdminDownloadStateResponse struct {
+	Error *PerformError `json:"error,omitempty"`
+}

roomserver/internal/perform/perform_admin.go

@@ -236,3 +236,145 @@ func (r *Admin) PerformAdminEvacuateUser(
 	}
 	return nil
 }
+
+func (r *Admin) PerformAdminDownloadState(
+	ctx context.Context,
+	req *api.PerformAdminDownloadStateRequest,
+	res *api.PerformAdminDownloadStateResponse,
+) error {
+	roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.DB.RoomInfo: %s", err),
+		}
+		return nil
+	}
+
+	if roomInfo == nil || roomInfo.IsStub() {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("room %q not found", req.RoomID),
+		}
+		return nil
+	}
+
+	fwdExtremities, _, depth, err := r.DB.LatestEventIDs(ctx, roomInfo.RoomNID)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.DB.LatestEventIDs: %s", err),
+		}
+		return nil
+	}
+
+	authEventMap := map[string]*gomatrixserverlib.Event{}
+	stateEventMap := map[string]*gomatrixserverlib.Event{}
+
+	for _, fwdExtremity := range fwdExtremities {
+		var state gomatrixserverlib.RespState
+		state, err = r.Inputer.FSAPI.LookupState(ctx, req.ServerName, req.RoomID, fwdExtremity.EventID, roomInfo.RoomVersion)
+		if err != nil {
+			res.Error = &api.PerformError{
+				Code: api.PerformErrorBadRequest,
+				Msg:  fmt.Sprintf("r.Inputer.FSAPI.LookupState (%q): %s", fwdExtremity.EventID, err),
+			}
+			return nil
+		}
+		for _, authEvent := range state.AuthEvents.UntrustedEvents(roomInfo.RoomVersion) {
+			if err = authEvent.VerifyEventSignatures(ctx, r.Inputer.KeyRing); err != nil {
+				continue
+			}
+			authEventMap[authEvent.EventID()] = authEvent
+		}
+		for _, stateEvent := range state.StateEvents.UntrustedEvents(roomInfo.RoomVersion) {
+			if err = stateEvent.VerifyEventSignatures(ctx, r.Inputer.KeyRing); err != nil {
+				continue
+			}
+			stateEventMap[stateEvent.EventID()] = stateEvent
+		}
+	}
+
+	authEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(authEventMap))
+	stateEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(stateEventMap))
+	stateIDs := make([]string, 0, len(stateEventMap))
+
+	for _, authEvent := range authEventMap {
+		authEvents = append(authEvents, authEvent.Headered(roomInfo.RoomVersion))
+	}
+	for _, stateEvent := range stateEventMap {
+		stateEvents = append(stateEvents, stateEvent.Headered(roomInfo.RoomVersion))
+		stateIDs = append(stateIDs, stateEvent.EventID())
+	}
+
+	builder := &gomatrixserverlib.EventBuilder{
+		Type:    "org.matrix.dendrite.state_download",
+		Sender:  req.UserID,
+		RoomID:  req.RoomID,
+		Content: gomatrixserverlib.RawJSON("{}"),
+	}
+
+	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("gomatrixserverlib.StateNeededForEventBuilder: %s", err),
+		}
+		return nil
+	}
+
+	queryRes := &api.QueryLatestEventsAndStateResponse{
+		RoomExists:   true,
+		RoomVersion:  roomInfo.RoomVersion,
+		LatestEvents: fwdExtremities,
+		StateEvents:  stateEvents,
+		Depth:        depth,
+	}
+
+	ev, err := eventutil.BuildEvent(ctx, builder, r.Cfg.Matrix, time.Now(), &eventsNeeded, queryRes)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("eventutil.BuildEvent: %s", err),
+		}
+		return nil
+	}
+
+	inputReq := &api.InputRoomEventsRequest{
+		Asynchronous: false,
+	}
+	inputRes := &api.InputRoomEventsResponse{}
+
+	for _, authEvent := range append(authEvents, stateEvents...) {
+		inputReq.InputRoomEvents = append(inputReq.InputRoomEvents, api.InputRoomEvent{
+			Kind:  api.KindOutlier,
+			Event: authEvent,
+		})
+	}
+
+	inputReq.InputRoomEvents = append(inputReq.InputRoomEvents, api.InputRoomEvent{
+		Kind:          api.KindNew,
+		Event:         ev,
+		Origin:        r.Cfg.Matrix.ServerName,
+		HasState:      true,
+		StateEventIDs: stateIDs,
+		SendAsServer:  string(r.Cfg.Matrix.ServerName),
+	})
+
+	if err := r.Inputer.InputRoomEvents(ctx, inputReq, inputRes); err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.Inputer.InputRoomEvents: %s", err),
+		}
+		return nil
+	}
+
+	if inputRes.ErrMsg != "" {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  inputRes.ErrMsg,
+		}
+	}
+
+	return nil
+}

roomserver/inthttp/client.go

@@ -27,18 +27,19 @@ const (
 	RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents"
 
 	// Perform operations
-	RoomserverPerformInvitePath            = "/roomserver/performInvite"
-	RoomserverPerformPeekPath              = "/roomserver/performPeek"
-	RoomserverPerformUnpeekPath            = "/roomserver/performUnpeek"
-	RoomserverPerformRoomUpgradePath       = "/roomserver/performRoomUpgrade"
-	RoomserverPerformJoinPath              = "/roomserver/performJoin"
-	RoomserverPerformLeavePath             = "/roomserver/performLeave"
-	RoomserverPerformBackfillPath          = "/roomserver/performBackfill"
-	RoomserverPerformPublishPath           = "/roomserver/performPublish"
-	RoomserverPerformInboundPeekPath       = "/roomserver/performInboundPeek"
-	RoomserverPerformForgetPath            = "/roomserver/performForget"
-	RoomserverPerformAdminEvacuateRoomPath = "/roomserver/performAdminEvacuateRoom"
-	RoomserverPerformAdminEvacuateUserPath = "/roomserver/performAdminEvacuateUser"
+	RoomserverPerformInvitePath             = "/roomserver/performInvite"
+	RoomserverPerformPeekPath               = "/roomserver/performPeek"
+	RoomserverPerformUnpeekPath             = "/roomserver/performUnpeek"
+	RoomserverPerformRoomUpgradePath        = "/roomserver/performRoomUpgrade"
+	RoomserverPerformJoinPath               = "/roomserver/performJoin"
+	RoomserverPerformLeavePath              = "/roomserver/performLeave"
+	RoomserverPerformBackfillPath           = "/roomserver/performBackfill"
+	RoomserverPerformPublishPath            = "/roomserver/performPublish"
+	RoomserverPerformInboundPeekPath        = "/roomserver/performInboundPeek"
+	RoomserverPerformForgetPath             = "/roomserver/performForget"
+	RoomserverPerformAdminEvacuateRoomPath  = "/roomserver/performAdminEvacuateRoom"
+	RoomserverPerformAdminEvacuateUserPath  = "/roomserver/performAdminEvacuateUser"
+	RoomserverPerformAdminDownloadStatePath = "/roomserver/performAdminDownloadState"
 
 	// Query operations
 	RoomserverQueryLatestEventsAndStatePath    = "/roomserver/queryLatestEventsAndState"
@@ -261,6 +262,17 @@ func (h *httpRoomserverInternalAPI) PerformAdminEvacuateRoom(
 	)
 }
 
+func (h *httpRoomserverInternalAPI) PerformAdminDownloadState(
+	ctx context.Context,
+	request *api.PerformAdminDownloadStateRequest,
+	response *api.PerformAdminDownloadStateResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformAdminDownloadState", h.roomserverURL+RoomserverPerformAdminDownloadStatePath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
 func (h *httpRoomserverInternalAPI) PerformAdminEvacuateUser(
 	ctx context.Context,
 	request *api.PerformAdminEvacuateUserRequest,

roomserver/inthttp/server.go

@@ -65,6 +65,11 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 		httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateUser", r.PerformAdminEvacuateUser),
 	)
 
+	internalAPIMux.Handle(
+		RoomserverPerformAdminDownloadStatePath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformAdminDownloadState", r.PerformAdminDownloadState),
+	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryPublishedRoomsPath,
 		httputil.MakeInternalRPCAPI("RoomserverQueryPublishedRooms", r.QueryPublishedRooms),

syncapi/streams/stream_pdu.go

@@ -251,8 +251,10 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		}
 		return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err)
 	}
-	recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents)
-	delta.StateEvents = removeDuplicates(delta.StateEvents, recentEvents) // roll back
+	recentEvents := gomatrixserverlib.HeaderedReverseTopologicalOrdering(
+		snapshot.StreamEventsToEvents(device, recentStreamEvents),
+		gomatrixserverlib.TopologicalOrderByPrevEvents,
+	)
 	prevBatch, err := snapshot.GetBackwardTopologyPos(ctx, recentStreamEvents)
 	if err != nil {
 		return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err)
@@ -263,10 +265,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		return r.To, nil
 	}
 
-	// Sort the events so that we can pick out the latest events from both sections.
-	recentEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(recentEvents, gomatrixserverlib.TopologicalOrderByPrevEvents)
-	delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(delta.StateEvents, gomatrixserverlib.TopologicalOrderByAuthEvents)
-
 	// Work out what the highest stream position is for all of the events in this
 	// room that were returned.
 	latestPosition := r.To
@@ -314,6 +312,14 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		limited = true
 	}
 
+	// Now that we've filtered the timeline, work out which state events are still
+	// left. Anything that appears in the filtered timeline will be removed from the
+	// "state" section and kept in "timeline".
+	delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(
+		removeDuplicates(delta.StateEvents, recentEvents),
+		gomatrixserverlib.TopologicalOrderByAuthEvents,
+	)
+
 	if len(delta.StateEvents) > 0 {
 		updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID())
 	}
@@ -507,7 +513,6 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	// transaction IDs for complete syncs, but we do it anyway because Sytest demands it for:
 	// "Can sync a room with a message with a transaction id" - which does a complete sync to check.
 	recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents)
-	stateEvents = removeDuplicates(stateEvents, recentEvents)
 
 	events := recentEvents
 	// Only apply history visibility checks if the response is for joined rooms
@@ -521,7 +526,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	// If we are limited by the filter AND the history visibility filter
 	// didn't "remove" events, return that the response is limited.
 	limited = limited && len(events) == len(recentEvents)
-
+	stateEvents = removeDuplicates(stateEvents, recentEvents)
 	if stateFilter.LazyLoadMembers {
 		if err != nil {
 			return nil, err

sytest-blacklist

@@ -40,4 +40,9 @@ Accesing an AS-hosted room alias asks the AS server
 Guest users can join guest_access rooms
 
 # This will fail in HTTP API mode, so blacklisted for now
-If a device list update goes missing, the server resyncs on the next one
+
+If a device list update goes missing, the server resyncs on the next one
+
+# Might be a bug in the test because leaves do appear :-(
+
+Leaves are present in non-gapped incremental syncs

sytest-whitelist

@@ -699,7 +699,7 @@ We do send redundant membership state across incremental syncs if asked
 Rejecting invite over federation doesn't break incremental /sync
 Gapped incremental syncs include all state changes
 Old leaves are present in gapped incremental syncs
-Leaves are present in non-gapped incremental syncs
+#Leaves are present in non-gapped incremental syncs
 Members from the gap are included in gappy incr LL sync
 Presence can be set from sync
 /state returns M_NOT_FOUND for a rejected message event