diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 89621aa87..49ef03054 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -472,7 +472,9 @@ func (a *KeyInternalAPI) queryRemoteKeys(
 		close(resultCh)
 	}()
 
-	for result := range resultCh {
+	processResult := func(result *gomatrixserverlib.RespQueryKeys) {
+		respMu.Lock()
+		defer respMu.Unlock()
 		for userID, nest := range result.DeviceKeys {
 			res.DeviceKeys[userID] = make(map[string]json.RawMessage)
 			for deviceID, deviceKey := range nest {
@@ -495,6 +497,10 @@ func (a *KeyInternalAPI) queryRemoteKeys(
 		// TODO: do we want to persist these somewhere now
 		// that we have fetched them?
 	}
+
+	for result := range resultCh {
+		processResult(result)
+	}
 }
 
 func (a *KeyInternalAPI) queryRemoteKeysOnServer(