Clean up periodically, we hit a race condition otherwise

federationsender/queue/destinationqueue.go

@@ -159,6 +159,10 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share
 // then we will interrupt the backoff, causing any federation
 // requests to retry.
 func (oq *destinationQueue) wakeQueueIfNeeded() {
+	// If the destination is blacklisted then do nothing.
+	if _, blacklisted := oq.statistics.BackoffInfo(); blacklisted {
+		return
+	}
 	// If we are backing off then interrupt the backoff.
 	if oq.backingOff.CAS(true, false) {
 		oq.interruptBackoff <- true

federationsender/queue/queue.go

@@ -101,6 +101,7 @@ func NewOutgoingQueues(
 		signing:    signing,
 		queues:     map[gomatrixserverlib.ServerName]*destinationQueue{},
 	}
+	queues.clearQueues()
 	// Look up which servers we have pending items for and then rehydrate those queues.
 	if !disabled {
 		time.AfterFunc(time.Second*5, func() {
@@ -167,14 +168,29 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d
 			signing:          oqs.signing,
 		}
 		oqs.queues[destination] = oq
-		oq.wakeQueueIfNeeded()
 	}
 	return oq
 }
 
+func (oqs *OutgoingQueues) clearQueues() {
+	oqs.queuesMutex.Lock()
+	queues := oqs.queues
+	oqs.queuesMutex.Unlock()
+	for _, q := range queues {
+		oqs.clearQueue(q)
+	}
+	time.AfterFunc(time.Minute, oqs.clearQueues)
+}
+
 func (oqs *OutgoingQueues) clearQueue(oq *destinationQueue) {
 	oqs.queuesMutex.Lock()
 	defer oqs.queuesMutex.Unlock()
+	switch {
+	case oq.running.Load():
+		return
+	case oq.backingOff.Load():
+		return
+	}
 
 	close(oq.notify)
 	close(oq.interruptBackoff)