diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 700f25c10..569e6df48 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -74,27 +74,43 @@ func (p *InviteStreamProvider) IncrementalSync(
 		return to
 	}
 	for roomID := range retiredInvites {
-		if _, ok := req.Response.Rooms.Invite[roomID]; ok {
+		if req.Response.Rooms.Invite[roomID] != nil {
 			continue
 		}
-		if _, ok := req.Response.Rooms.Join[roomID]; ok {
+		if req.Response.Rooms.Join[roomID] != nil {
 			continue
 		}
-		lr := types.NewLeaveResponse()
-		h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...))
-		lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{
-			// fake event ID which muxes in the to position
-			EventID:        "$" + base64.RawURLEncoding.EncodeToString(h[:]),
-			OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
-			RoomID:         roomID,
-			Sender:         req.Device.UserID,
-			StateKey:       &req.Device.UserID,
-			Type:           "m.room.member",
-			Content:        gomatrixserverlib.RawJSON(`{"membership":"leave"}`),
-		})
-		req.Response.Rooms.Leave[roomID] = lr
 
+		joinedUsers, err := snapshot.AllJoinedUsersInRoom(ctx, []string{roomID})
+		if err != nil {
+			continue
+		}
+
+		if !contains(joinedUsers[roomID], req.Device.UserID) {
+			lr := types.NewLeaveResponse()
+			h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...))
+			lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{
+				// fake event ID which muxes in the to position
+				EventID:        "$" + base64.RawURLEncoding.EncodeToString(h[:]),
+				OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
+				RoomID:         roomID,
+				Sender:         req.Device.UserID,
+				StateKey:       &req.Device.UserID,
+				Type:           "m.room.member",
+				Content:        gomatrixserverlib.RawJSON(`{"membership":"leave"}`),
+			})
+			req.Response.Rooms.Leave[roomID] = lr
+		}
 	}
 
 	return maxID
 }
+
+func contains(values []string, findVal string) bool {
+	for _, v := range values {
+		if v == findVal {
+			return true
+		}
+	}
+	return false
+}