diff --git a/internal/caching/cache_lazy_load_members.go b/internal/caching/cache_lazy_load_members.go
index 903575430..71a317624 100644
--- a/internal/caching/cache_lazy_load_members.go
+++ b/internal/caching/cache_lazy_load_members.go
@@ -8,41 +8,56 @@ import (
 )
 
 const (
-	LazyLoadCacheName       = "lazy_load_members"
-	LazyLoadCacheMaxEntries = 128
-	LazyLoadCacheMutable    = true
-	LazyLoadCacheMaxAge     = time.Minute * 30
+	LazyLoadCacheName           = "lazy_load_members"
+	LazyLoadCacheMaxEntries     = 128
+	LazyLoadCacheMaxUserEntries = 128
+	LazyLoadCacheMutable        = true
+	LazyLoadCacheMaxAge         = time.Minute * 30
 )
 
 type LazyLoadCache struct {
-	// Mapping from userID/deviceID to InMemoryLRUCachePartition
-	userCaches map[string]*InMemoryLRUCachePartition
+	// InMemoryLRUCachePartition containing other InMemoryLRUCachePartitions
+	// with the actual cached members
+	userCaches *InMemoryLRUCachePartition
 }
 
 // NewLazyLoadCache creates a new LazyLoadCache.
-func NewLazyLoadCache() *LazyLoadCache {
-	return &LazyLoadCache{
-		userCaches: make(map[string]*InMemoryLRUCachePartition),
-	}
-}
-
-func (c *LazyLoadCache) lazyLoadCacheForUser(device *userapi.Device) (*InMemoryLRUCachePartition, error) {
-	cacheName := fmt.Sprintf("%s/%s", device.UserID, device.ID)
-	cache, ok := c.userCaches[cacheName]
-	if ok {
-		return cache, nil
-	}
+func NewLazyLoadCache() (*LazyLoadCache, error) {
 	cache, err := NewInMemoryLRUCachePartition(
 		LazyLoadCacheName,
 		LazyLoadCacheMutable,
 		LazyLoadCacheMaxEntries,
 		LazyLoadCacheMaxAge,
+		true,
+	)
+	if err != nil {
+		return nil, err
+	}
+	go cacheCleaner(cache)
+	return &LazyLoadCache{
+		userCaches: cache,
+	}, nil
+}
+
+func (c *LazyLoadCache) lazyLoadCacheForUser(device *userapi.Device) (*InMemoryLRUCachePartition, error) {
+	cacheName := fmt.Sprintf("%s/%s", device.UserID, device.ID)
+	userCache, ok := c.userCaches.Get(cacheName)
+	if ok && userCache != nil {
+		if cache, ok := userCache.(*InMemoryLRUCachePartition); ok {
+			return cache, nil
+		}
+	}
+	cache, err := NewInMemoryLRUCachePartition(
+		LazyLoadCacheName,
+		LazyLoadCacheMutable,
+		LazyLoadCacheMaxUserEntries,
+		LazyLoadCacheMaxAge,
 		false,
 	)
 	if err != nil {
 		return nil, err
 	}
-	c.userCaches[cacheName] = cache
+	c.userCaches.Set(cacheName, cache)
 	go cacheCleaner(cache)
 	return cache, nil
 }
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index 90c2b57dc..2f9165d91 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -57,7 +57,10 @@ func AddPublicRoutes(
 	}
 
 	eduCache := caching.NewTypingCache()
-	lazyLoadCache := caching.NewLazyLoadCache()
+	lazyLoadCache, err := caching.NewLazyLoadCache()
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to create lazy loading cache")
+	}
 	notifier := notifier.NewNotifier()
 	streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, lazyLoadCache, notifier)
 	notifier.SetCurrentPosition(streams.Latest(context.Background()))