diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index c3b34ab5d..efa583331 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -42,7 +42,7 @@ func main() {
 
 	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
 
-	cache, err := caching.NewInMemoryLRUCache(&cfg.Global)
+	cache, err := caching.NewInMemoryLRUCache(true)
 	if err != nil {
 		panic(err)
 	}
diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go
index 71995cae2..ff3f06b6e 100644
--- a/cmd/roomserver-integration-tests/main.go
+++ b/cmd/roomserver-integration-tests/main.go
@@ -257,7 +257,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 		panic(err)
 	}
 
-	cache, err := caching.NewInMemoryLRUCache(&cfg.Global)
+	cache, err := caching.NewInMemoryLRUCache(false)
 	if err != nil {
 		panic(err)
 	}
diff --git a/dendrite-config.yaml b/dendrite-config.yaml
index eca4f8bf0..ccdb32432 100644
--- a/dendrite-config.yaml
+++ b/dendrite-config.yaml
@@ -103,11 +103,6 @@ global:
       username: metrics
       password: metrics
 
-  # Advanced options for tuning in-memory caches. Do not change these unless
-  # you know what you are doing.
-  # caches:
-  #   federationsender_cache_size: 128
-
 # Configuration for the Appservice API.
 app_service_api:
   internal_api:
diff --git a/internal/caching/cache_federationevents.go b/internal/caching/cache_federationevents.go
index 6ab00c4a7..a48c11fd2 100644
--- a/internal/caching/cache_federationevents.go
+++ b/internal/caching/cache_federationevents.go
@@ -7,8 +7,9 @@ import (
 )
 
 const (
-	FederationSenderCacheName    = "federation_event"
-	FederationSenderCacheMutable = true // to allow use of Unset only
+	FederationEventCacheName       = "federation_event"
+	FederationEventCacheMaxEntries = 256
+	FederationEventCacheMutable    = true // to allow use of Unset only
 )
 
 // FederationSenderCache contains the subset of functions needed for
diff --git a/internal/caching/impl_inmemorylru.go b/internal/caching/impl_inmemorylru.go
index 7e4b8a488..f05e8f3c6 100644
--- a/internal/caching/impl_inmemorylru.go
+++ b/internal/caching/impl_inmemorylru.go
@@ -4,17 +4,16 @@ import (
 	"fmt"
 
 	lru "github.com/hashicorp/golang-lru"
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
-func NewInMemoryLRUCache(cfg *config.Global) (*Caches, error) {
+func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) {
 	roomVersions, err := NewInMemoryLRUCachePartition(
 		RoomVersionCacheName,
 		RoomVersionCacheMutable,
 		RoomVersionCacheMaxEntries,
-		cfg.Metrics.Enabled,
+		enablePrometheus,
 	)
 	if err != nil {
 		return nil, err
@@ -23,7 +22,7 @@ func NewInMemoryLRUCache(cfg *config.Global) (*Caches, error) {
 		ServerKeyCacheName,
 		ServerKeyCacheMutable,
 		ServerKeyCacheMaxEntries,
-		cfg.Metrics.Enabled,
+		enablePrometheus,
 	)
 	if err != nil {
 		return nil, err
@@ -32,7 +31,7 @@ func NewInMemoryLRUCache(cfg *config.Global) (*Caches, error) {
 		RoomServerStateKeyNIDsCacheName,
 		RoomServerStateKeyNIDsCacheMutable,
 		RoomServerStateKeyNIDsCacheMaxEntries,
-		cfg.Metrics.Enabled,
+		enablePrometheus,
 	)
 	if err != nil {
 		return nil, err
@@ -41,7 +40,7 @@ func NewInMemoryLRUCache(cfg *config.Global) (*Caches, error) {
 		RoomServerEventTypeNIDsCacheName,
 		RoomServerEventTypeNIDsCacheMutable,
 		RoomServerEventTypeNIDsCacheMaxEntries,
-		cfg.Metrics.Enabled,
+		enablePrometheus,
 	)
 	if err != nil {
 		return nil, err
@@ -50,7 +49,7 @@ func NewInMemoryLRUCache(cfg *config.Global) (*Caches, error) {
 		RoomServerRoomNIDsCacheName,
 		RoomServerRoomNIDsCacheMutable,
 		RoomServerRoomNIDsCacheMaxEntries,
-		cfg.Metrics.Enabled,
+		enablePrometheus,
 	)
 	if err != nil {
 		return nil, err
@@ -59,16 +58,16 @@ func NewInMemoryLRUCache(cfg *config.Global) (*Caches, error) {
 		RoomServerRoomIDsCacheName,
 		RoomServerRoomIDsCacheMutable,
 		RoomServerRoomIDsCacheMaxEntries,
-		cfg.Metrics.Enabled,
+		enablePrometheus,
 	)
 	if err != nil {
 		return nil, err
 	}
 	federationEvents, err := NewInMemoryLRUCachePartition(
-		FederationSenderCacheName,
-		FederationSenderCacheMutable,
-		cfg.Caches.FederationSenderEventCacheSize,
-		cfg.Metrics.Enabled,
+		FederationEventCacheName,
+		FederationEventCacheMutable,
+		FederationEventCacheMaxEntries,
+		enablePrometheus,
 	)
 	if err != nil {
 		return nil, err
diff --git a/setup/base.go b/setup/base.go
index 79cfae6d1..acbf2d35f 100644
--- a/setup/base.go
+++ b/setup/base.go
@@ -106,7 +106,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 		logrus.WithError(err).Panicf("failed to start opentracing")
 	}
 
-	cache, err := caching.NewInMemoryLRUCache(&cfg.Global)
+	cache, err := caching.NewInMemoryLRUCache(true)
 	if err != nil {
 		logrus.WithError(err).Warnf("Failed to create cache")
 	}
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index ef2b09113..956522176 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -48,9 +48,6 @@ type Global struct {
 
 	// Metrics configuration
 	Metrics Metrics `yaml:"metrics"`
-
-	// Cache configuration
-	Caches Caches `yaml:"caches"`
 }
 
 func (c *Global) Defaults() {
@@ -143,14 +140,3 @@ func (c DatabaseOptions) MaxOpenConns() int {
 func (c DatabaseOptions) ConnMaxLifetime() time.Duration {
 	return time.Duration(c.ConnMaxLifetimeSeconds) * time.Second
 }
-
-type Caches struct {
-	FederationSenderEventCacheSize int `yaml:"federationsender_cache_size"`
-}
-
-func (c *Caches) Defaults() {
-	c.FederationSenderEventCacheSize = 128
-}
-
-func (c *Caches) Verify(configErrs *ConfigErrors, isMonolith bool) {
-}