diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index c3c7a382a..304b67b23 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -363,7 +363,6 @@ func (u *DeviceListUpdater) worker(ch chan gomatrixserverlib.ServerName) {
 			// Don't retry a server that we're already waiting for.
 			continue
 		}
-		retriesMu.Unlock()
 		waitTime, shouldRetry := u.processServer(serverName)
 		if shouldRetry {
 			retriesMu.Lock()