diff --git a/go.mod b/go.mod
index bbac53eca..7d06b987c 100644
--- a/go.mod
+++ b/go.mod
@@ -18,7 +18,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200521101409-e5f0907c8b33
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible
diff --git a/go.sum b/go.sum
index d487f42a3..606cd7f2b 100644
--- a/go.sum
+++ b/go.sum
@@ -358,6 +358,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b h1:nAmSc1KvQOumoRTz/LD68KyrB6Q5/6q7CmQ5Bswc2nM=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200521101409-e5f0907c8b33 h1:b0bbec/+QDYlrIT2/JQuezcASrJt278PjgfHPtRDzVk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200521101409-e5f0907c8b33/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
diff --git a/serverkeyapi/api/http.go b/serverkeyapi/api/http.go
index b35d858f8..cc5576074 100644
--- a/serverkeyapi/api/http.go
+++ b/serverkeyapi/api/http.go
@@ -18,7 +18,7 @@ const (
 )
 
 type InputPublicKeysRequest struct {
-	Keys map[gomatrixserverlib.ServerName]map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyLookupResult `json:"keys"`
+	Keys map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"keys"`
 }
 
 type InputPublicKeysResponse struct {
@@ -37,11 +37,11 @@ func (h *httpServerKeyInternalAPI) InputPublicKeys(
 }
 
 type QueryPublicKeysRequest struct {
-	Requests map[gomatrixserverlib.ServerName]map[gomatrixserverlib.KeyID]gomatrixserverlib.Timestamp `json:"requests"`
+	Requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp `json:"requests"`
 }
 
 type QueryPublicKeysResponse struct {
-	Results map[gomatrixserverlib.ServerName]map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyLookupResult `json:"results"`
+	Results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"results"`
 }
 
 func (h *httpServerKeyInternalAPI) QueryPublicKeys(
diff --git a/serverkeyapi/api/satisfy.go b/serverkeyapi/api/satisfy.go
index 32f8aeab9..1a5dbc168 100644
--- a/serverkeyapi/api/satisfy.go
+++ b/serverkeyapi/api/satisfy.go
@@ -17,10 +17,8 @@ func (s *httpServerKeyInternalAPI) StoreKeys(
 	request := InputPublicKeysRequest{}
 	response := InputPublicKeysResponse{}
 	for req, res := range results {
-		if _, ok := request.Keys[req.ServerName]; !ok {
-			request.Keys[req.ServerName] = map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyLookupResult{}
-		}
-		request.Keys[req.ServerName][req.KeyID] = res
+		request.Keys[req] = res
+		s.immutableCache.StoreServerKey(req, res)
 	}
 	return s.InputPublicKeys(ctx, &request, &response)
 }
@@ -29,27 +27,22 @@ func (s *httpServerKeyInternalAPI) FetchKeys(
 	ctx context.Context,
 	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
 ) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) {
+	result := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
 	request := QueryPublicKeysRequest{}
 	response := QueryPublicKeysResponse{}
 	for req, ts := range requests {
-		if _, ok := request.Requests[req.ServerName]; !ok {
-			request.Requests[req.ServerName] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Timestamp{}
+		if res, ok := s.immutableCache.GetServerKey(req); ok {
+			result[req] = res
+			continue
 		}
-		request.Requests[req.ServerName][req.KeyID] = ts
+		request.Requests[req] = ts
 	}
 	err := s.QueryPublicKeys(ctx, &request, &response)
 	if err != nil {
 		return nil, err
 	}
-	result := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
-	for serverName, byServerName := range response.Results {
-		for keyID, res := range byServerName {
-			key := gomatrixserverlib.PublicKeyLookupRequest{
-				ServerName: serverName,
-				KeyID:      keyID,
-			}
-			result[key] = res
-		}
+	for req, res := range response.Results {
+		result[req] = res
 	}
 	return result, nil
 }
diff --git a/serverkeyapi/internal/http.go b/serverkeyapi/internal/http.go
index 1e857b372..60141e6bb 100644
--- a/serverkeyapi/internal/http.go
+++ b/serverkeyapi/internal/http.go
@@ -13,33 +13,28 @@ import (
 func (s *ServerKeyAPI) SetupHTTP(servMux *http.ServeMux) {
 	servMux.Handle(api.ServerKeyQueryPublicKeyPath,
 		common.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse {
+			result := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
 			request := api.QueryPublicKeysRequest{}
-			response := api.QueryPublicKeysResponse{
-				Results: map[gomatrixserverlib.ServerName]map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyLookupResult{},
-			}
+			response := api.QueryPublicKeysResponse{}
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
 				return util.MessageResponse(http.StatusBadRequest, err.Error())
 			}
 			lookup := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp)
-			for serverName, byServerName := range request.Requests {
-				for keyID, timestamp := range byServerName {
-					key := gomatrixserverlib.PublicKeyLookupRequest{
-						ServerName: serverName,
-						KeyID:      keyID,
-					}
-					lookup[key] = timestamp
+			for req, timestamp := range request.Requests {
+				if res, ok := s.ImmutableCache.GetServerKey(req); ok {
+					result[req] = res
+					continue
 				}
+				lookup[req] = timestamp
 			}
 			keys, err := s.DB.FetchKeys(req.Context(), lookup)
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
 			for req, res := range keys {
-				if _, ok := response.Results[req.ServerName]; !ok {
-					response.Results[req.ServerName] = map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyLookupResult{}
-				}
-				response.Results[req.ServerName][req.KeyID] = res
+				result[req] = res
 			}
+			response.Results = result
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
@@ -51,14 +46,9 @@ func (s *ServerKeyAPI) SetupHTTP(servMux *http.ServeMux) {
 				return util.MessageResponse(http.StatusBadRequest, err.Error())
 			}
 			store := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult)
-			for serverName, byServerName := range request.Keys {
-				for keyID, keyResult := range byServerName {
-					key := gomatrixserverlib.PublicKeyLookupRequest{
-						ServerName: serverName,
-						KeyID:      keyID,
-					}
-					store[key] = keyResult
-				}
+			for req, res := range request.Keys {
+				store[req] = res
+				s.ImmutableCache.StoreServerKey(req, res)
 			}
 			if err := s.DB.StoreKeys(req.Context(), store); err != nil {
 				return util.ErrorResponse(err)