diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go
index e1fca34bd..592c91b01 100644
--- a/syncapi/notifier/notifier.go
+++ b/syncapi/notifier/notifier.go
@@ -258,6 +258,17 @@ func (n *Notifier) SharedUsers(userID string) (sharedUsers []string) {
 	return sharedUsers
 }
 
+func (n *Notifier) IsSharedUser(userA, userB string) bool {
+	for _, users := range n.roomIDToJoinedUsers {
+		_, okA := users[userA]
+		_, okB := users[userB]
+		if okA && okB {
+			return true
+		}
+	}
+	return false
+}
+
 // GetListener returns a UserStreamListener that can be used to wait for
 // updates for a user. Must be closed.
 // notify for anything before sincePos
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index 7b4cf982e..6d5ec54bb 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -64,17 +64,6 @@ func (p *PresenceStreamProvider) IncrementalSync(
 		return to
 	}
 
-	// get all joined users
-	// TODO: SharedUsers might get out of sync
-	sharedUsers := p.notifier.SharedUsers(req.Device.UserID)
-
-	// convert array to a map for easier checking if a user exists
-	sharedUsersMap := make(map[string]bool, len(sharedUsers))
-	sharedUsersMap[req.Device.UserID] = true
-	for i := range sharedUsers {
-		sharedUsersMap[sharedUsers[i]] = true
-	}
-
 	// add newly joined rooms user presences
 	newlyJoined := joinedRooms(req.Response, req.Device.UserID)
 	if len(newlyJoined) > 0 {
@@ -87,7 +76,6 @@ func (p *PresenceStreamProvider) IncrementalSync(
 		for _, roomID := range newlyJoined {
 			roomUsers := p.notifier.JoinedUsers(roomID)
 			for i := range roomUsers {
-				sharedUsersMap[roomUsers[i]] = true
 				// we already got a presence from this user
 				if _, ok := presences[roomUsers[i]]; ok {
 					continue
@@ -108,7 +96,7 @@ func (p *PresenceStreamProvider) IncrementalSync(
 	for i := range presences {
 		presence := presences[i]
 		// Ignore users we don't share a room with
-		if !sharedUsersMap[presence.UserID] {
+		if req.Device.UserID != presence.UserID && !p.notifier.IsSharedUser(req.Device.UserID, presence.UserID) {
 			continue
 		}
 		cacheKey := req.Device.UserID + req.Device.ID + presence.UserID