diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index bd9ec042f..b01157443 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -178,7 +178,10 @@ func (p *PDUStreamProvider) IncrementalSync(
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
 			return to
 		}
-		if pos > newPos {
+		switch {
+		case r.Backwards && pos < newPos:
+			fallthrough
+		case !r.Backwards && pos > newPos:
 			newPos = pos
 		}
 	}