Merge branch 'main' into s7evink/optimizepresence

CHANGES.md

@@ -1,5 +1,34 @@
 # Changelog
 
+## Dendrite 0.8.2 (2022-04-27)
+
+### Features
+
+* Lazy-loading has been added to the `/sync` endpoint, which should speed up syncs considerably
+* Filtering has been added to the `/messages` endpoint
+* The room summary now contains "heroes" (up to 5 users in the room) for clients to display when no room name is set
+* The existing lazy-loading caches will now be used by `/messages` and `/context` so that member events will not be sent to clients more times than necessary
+* The account data stream now uses the provided filters
+* The built-in NATS Server has been updated to version 2.8.0
+* The `/state` and `/state_ids` endpoints will now return `M_NOT_FOUND` for rejected events
+* Repeated calls to the `/redact` endpoint will now be idempotent when a transaction ID is given
+* Dendrite should now be able to run as a Windows service under Service Control Manager
+
+### Fixes
+
+* Fictitious presence updates will no longer be created for users which have not sent us presence updates, which should speed up complete syncs considerably
+* Uploading cross-signing device signatures should now be more reliable, fixing a number of bugs with cross-signing
+* All account data should now be sent properly on a complete sync, which should eliminate problems with client settings or key backups appearing to be missing
+* Account data will now be limited correctly on incremental syncs, returning the stream position of the most recent update rather than the latest stream position
+* Account data will not be sent for parted rooms, which should reduce the number of left/forgotten rooms reappearing in clients as empty rooms
+* The TURN username hash has been fixed which should help to resolve some problems when using TURN for voice calls (contributed by [fcwoknhenuxdfiyv](https://github.com/fcwoknhenuxdfiyv))
+* Push rules can no longer be modified using the account data endpoints
+* Querying account availability should now work properly in polylith deployments
+* A number of bugs with sync filters have been fixed
+* A default sync filter will now be used if the request contains a filter ID that does not exist
+* The `pushkey_ts` field is now using seconds instead of milliseconds
+* A race condition when gracefully shutting down has been fixed, so JetStream should no longer cause the process to exit before other Dendrite components are finished shutting down
+
 ## Dendrite 0.8.1 (2022-04-07)
 
 ### Fixes

build/docker/Dockerfile.monolith

@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.17-alpine AS base
+FROM docker.io/golang:1.18-alpine AS base
 
 RUN apk --update --no-cache add bash build-base
 
@@ -23,4 +23,4 @@ COPY --from=base /build/bin/* /usr/bin/
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
 
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]

build/docker/Dockerfile.polylith

@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.17-alpine AS base
+FROM docker.io/golang:1.18-alpine AS base
 
 RUN apk --update --no-cache add bash build-base
 
@@ -23,4 +23,4 @@ COPY --from=base /build/bin/* /usr/bin/
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
 
-ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
+ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]

federationapi/routing/backfill.go

@@ -51,6 +51,12 @@ func Backfill(
 		}
 	}
 
+	// If we don't think we belong to this room then don't waste the effort
+	// responding to expensive requests for it.
+	if err := ErrorIfLocalServerNotInRoom(httpReq.Context(), rsAPI, roomID); err != nil {
+		return *err
+	}
+
 	// Check if all of the required parameters are there.
 	eIDs, exists = httpReq.URL.Query()["v"]
 	if !exists {

federationapi/routing/eventauth.go

@@ -30,6 +30,12 @@ func GetEventAuth(
 	roomID string,
 	eventID string,
 ) util.JSONResponse {
+	// If we don't think we belong to this room then don't waste the effort
+	// responding to expensive requests for it.
+	if err := ErrorIfLocalServerNotInRoom(ctx, rsAPI, roomID); err != nil {
+		return *err
+	}
+
 	event, resErr := fetchEvent(ctx, rsAPI, eventID)
 	if resErr != nil {
 		return *resErr

federationapi/routing/missingevents.go

@@ -45,6 +45,12 @@ func GetMissingEvents(
 		}
 	}
 
+	// If we don't think we belong to this room then don't waste the effort
+	// responding to expensive requests for it.
+	if err := ErrorIfLocalServerNotInRoom(httpReq.Context(), rsAPI, roomID); err != nil {
+		return *err
+	}
+
 	var eventsResponse api.QueryMissingEventsResponse
 	if err := rsAPI.QueryMissingEvents(
 		httpReq.Context(), &api.QueryMissingEventsRequest{

federationapi/routing/routing.go

@@ -15,6 +15,8 @@
 package routing
 
 import (
+	"context"
+	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -24,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -491,3 +494,27 @@ func Setup(
 		}),
 	).Methods(http.MethodGet)
 }
+
+func ErrorIfLocalServerNotInRoom(
+	ctx context.Context,
+	rsAPI api.RoomserverInternalAPI,
+	roomID string,
+) *util.JSONResponse {
+	// Check if we think we're in this room. If we aren't then
+	// we won't waste CPU cycles serving this request.
+	joinedReq := &api.QueryServerJoinedToRoomRequest{
+		RoomID: roomID,
+	}
+	joinedRes := &api.QueryServerJoinedToRoomResponse{}
+	if err := rsAPI.QueryServerJoinedToRoom(ctx, joinedReq, joinedRes); err != nil {
+		res := util.ErrorResponse(err)
+		return &res
+	}
+	if !joinedRes.IsInRoom {
+		return &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound(fmt.Sprintf("This server is not joined to room %s", roomID)),
+		}
+	}
+	return nil
+}

federationapi/routing/state.go

@@ -101,6 +101,12 @@ func getState(
 	roomID string,
 	eventID string,
 ) (stateEvents, authEvents []*gomatrixserverlib.HeaderedEvent, errRes *util.JSONResponse) {
+	// If we don't think we belong to this room then don't waste the effort
+	// responding to expensive requests for it.
+	if err := ErrorIfLocalServerNotInRoom(ctx, rsAPI, roomID); err != nil {
+		return nil, nil, err
+	}
+
 	event, resErr := fetchEvent(ctx, rsAPI, eventID)
 	if resErr != nil {
 		return nil, nil, resErr

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 1
+	VersionPatch = 2
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/internal/internal.go

@@ -319,6 +319,9 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 	// JSON, add the signatures and marshal it again, for some reason?
 
 	for targetUserID, masterKey := range res.MasterKeys {
+		if masterKey.Signatures == nil {
+			masterKey.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{}
+		}
 		for targetKeyID := range masterKey.Keys {
 			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID)
 			if err != nil {

setup/base/base.go

@@ -469,14 +469,12 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 	}
 
 	minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite)
-
 	<-b.ProcessContext.WaitForShutdown()
+
 	logrus.Infof("Stopping HTTP listeners")
 	_ = internalServ.Shutdown(context.Background())
 	_ = externalServ.Shutdown(context.Background())
 	logrus.Infof("Stopped HTTP listeners")
-
-	b.WaitForShutdown()
 }
 
 func (b *BaseDendrite) WaitForShutdown() {

syncapi/storage/postgres/memberships_table.go

@@ -64,7 +64,7 @@ const selectMembershipCountSQL = "" +
 	") t WHERE t.membership = $3"
 
 const selectHeroesSQL = "" +
-	"SELECT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5"
+	"SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5"
 
 type membershipsStatements struct {
 	upsertMembershipStmt      *sql.Stmt

sytest-blacklist

@@ -48,4 +48,3 @@ Notifications can be viewed with GET /notifications
 # More flakey
 
 If remote user leaves room we no longer receive device updates
-Local device key changes get to remote servers