Limit recent events when going backwards (#2840)

If we're going backwards, we were selecting potentially thousands of
events, which in turn were fed to history visibility checks, resulting
in bad sync performance.

syncapi/streams/stream_pdu.go

@@ -216,6 +216,9 @@ func (p *PDUStreamProvider) IncrementalSync(
 	return newPos
 }
 
+// Limit the recent events to X when going backwards
+const recentEventBackwardsLimit = 100
+
 // nolint:gocyclo
 func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	ctx context.Context,
@@ -229,9 +232,15 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 ) (types.StreamPosition, error) {
 
 	originalLimit := eventFilter.Limit
-	if r.Backwards {
+	// If we're going backwards, grep at least X events, this is mostly to satisfy Sytest
-		eventFilter.Limit = int(r.From - r.To)
+	if r.Backwards && originalLimit < recentEventBackwardsLimit {
+		eventFilter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way
+		diff := r.From - r.To
+		if diff > 0 && diff < recentEventBackwardsLimit {
+			eventFilter.Limit = int(diff)
 		}
+	}
+
 	recentStreamEvents, limited, err := snapshot.RecentEvents(
 		ctx, delta.RoomID, r,
 		eventFilter, true, true,