diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index f1c5450fd..8c2b8fc89 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
+	"math"
 	"strconv"
 	"time"
 
@@ -73,43 +74,28 @@ func (p *InviteStreamProvider) IncrementalSync(
 		return to
 	}
 	for roomID := range retiredInvites {
-		if req.Response.Rooms.Invite[roomID] != nil {
-			continue
-		}
-		if req.Response.Rooms.Join[roomID] != nil {
+		membership, _, err := snapshot.SelectMembershipForUser(ctx, roomID, req.Device.UserID, math.MaxInt64)
+		// Skip if the user is an existing member of the room.
+		// Otherwise, the NewLeaveResponse will eject the user from the room unintentionally
+		if membership == gomatrixserverlib.Join ||
+			err != nil {
 			continue
 		}
 
-		joinedUsers, err := snapshot.AllJoinedUsersInRoom(ctx, []string{roomID})
-		if err != nil {
-			continue
-		}
-
-		if !contains(joinedUsers[roomID], req.Device.UserID) {
-			lr := types.NewLeaveResponse()
-			h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...))
-			lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{
-				// fake event ID which muxes in the to position
-				EventID:        "$" + base64.RawURLEncoding.EncodeToString(h[:]),
-				OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
-				RoomID:         roomID,
-				Sender:         req.Device.UserID,
-				StateKey:       &req.Device.UserID,
-				Type:           "m.room.member",
-				Content:        gomatrixserverlib.RawJSON(`{"membership":"leave"}`),
-			})
-			req.Response.Rooms.Leave[roomID] = lr
-		}
+		lr := types.NewLeaveResponse()
+		h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...))
+		lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{
+			// fake event ID which muxes in the to position
+			EventID:        "$" + base64.RawURLEncoding.EncodeToString(h[:]),
+			OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
+			RoomID:         roomID,
+			Sender:         req.Device.UserID,
+			StateKey:       &req.Device.UserID,
+			Type:           "m.room.member",
+			Content:        gomatrixserverlib.RawJSON(`{"membership":"leave"}`),
+		})
+		req.Response.Rooms.Leave[roomID] = lr
 	}
 
 	return maxID
 }
-
-func contains(values []string, findVal string) bool {
-	for _, v := range values {
-		if v == findVal {
-			return true
-		}
-	}
-	return false
-}