From a360f231e8f0d388dd717cf51286eb4ba56e792b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 7 Jul 2020 13:47:28 +0100 Subject: [PATCH 1/3] Add a bit more logging to the fedsender --- federationsender/queue/destinationqueue.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 70e50d2e8..1ad45254c 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -65,6 +65,7 @@ type destinationQueue struct { func (oq *destinationQueue) sendEvent(nid int64) { if oq.statistics.Blacklisted() { // If the destination is blacklisted then drop the event. + log.Infof("%s is blacklisted; dropping event", oq.destination) return } oq.wakeQueueIfNeeded() @@ -214,6 +215,7 @@ func (oq *destinationQueue) backgroundSend() { // backoff duration to complete first, or until explicitly // told to retry. if backoff, duration := oq.statistics.BackoffDuration(); backoff { + log.WithField("duration", duration).Infof("Backing off %s", oq.destination) oq.backingOff.Store(true) select { case <-time.After(duration): @@ -336,6 +338,7 @@ func (oq *destinationQueue) nextTransaction( // If we didn't get anything from the database and there are no // pending EDUs then there's nothing to do - stop here. if len(pdus) == 0 && len(pendingEDUs) == 0 { + log.Warnf("no pdus/edus for nextTransaction for destination %q", oq.destination) return false, nil } From 51a7f4bb7c9dd81a1f9bc54792aa212837fb267a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 7 Jul 2020 14:27:45 +0100 Subject: [PATCH 2/3] bugfix: continue sending PDUs if ones are added whilst sending another PDU Without this, the queue goes back to sleep on `<-oq.notifyPDUs` which won't fire because `pendingPDUs` is already > 0. This should fix a flakey sytest. --- federationsender/queue/destinationqueue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 1ad45254c..ab779a5d3 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -225,7 +225,7 @@ func (oq *destinationQueue) backgroundSend() { } // If we have pending PDUs or EDUs then construct a transaction. - if oq.pendingPDUs.Load() > 0 || len(oq.pendingEDUs) > 0 { + for oq.pendingPDUs.Load() > 0 || len(oq.pendingEDUs) > 0 { // Try sending the next transaction and see what happens. transaction, terr := oq.nextTransaction(oq.pendingEDUs) if terr != nil { From 9035e169b7844215eafe2e32359eed3f908adeb8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 7 Jul 2020 14:30:57 +0100 Subject: [PATCH 3/3] Break if no txn is sent --- federationsender/queue/destinationqueue.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index ab779a5d3..accd39f4d 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -250,6 +250,8 @@ func (oq *destinationQueue) backgroundSend() { oq.statistics.Success() // Clean up the in-memory buffers. oq.cleanPendingEDUs() + } else { + break } }