diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go
index 0d6de1560..8931830f3 100644
--- a/federationapi/routing/keys.go
+++ b/federationapi/routing/keys.go
@@ -160,7 +160,7 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver
 	for _, oldVerifyKey := range cfg.Matrix.OldVerifyKeys {
 		keys.OldVerifyKeys[oldVerifyKey.KeyID] = gomatrixserverlib.OldVerifyKey{
 			VerifyKey: gomatrixserverlib.VerifyKey{
-				Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PublicKey),
+				Key: oldVerifyKey.PublicKey,
 			},
 			ExpiredTS: oldVerifyKey.ExpiredAt,
 		}
diff --git a/setup/config/config.go b/setup/config/config.go
index 5a005460f..df5e44585 100644
--- a/setup/config/config.go
+++ b/setup/config/config.go
@@ -263,7 +263,7 @@ func loadConfig(
 
 			key.KeyID = keyID
 			key.PrivateKey = privateKey
-			key.PublicKey = privateKey.Public().(ed25519.PublicKey)
+			key.PublicKey = privateKey.Public().(gomatrixserverlib.Base64Bytes)
 		}
 	}
 
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index d1e9fd468..17fff741f 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -128,7 +128,7 @@ type OldVerifyKeys struct {
 	PrivateKey ed25519.PrivateKey `yaml:"-"`
 
 	// The public key, in case only that part is known.
-	PublicKey ed25519.PublicKey `yaml:"public_key"`
+	PublicKey gomatrixserverlib.Base64Bytes `yaml:"public_key"`
 
 	// The key ID of the private key.
 	KeyID gomatrixserverlib.KeyID `yaml:"-"`