a16db1c408
* Improve federation sender performance and behaviour, add backoff * Tweaks * Tweaks * Tweaks * Take copies of events before passing to destination queues * Don't accidentally drop queued messages * Don't take copies again * Tidy up a bit * Break out statistics (tracked component-wide), report success and failures from Perform actions * Fix comment, use atomic add * Improve logic a bit, don't block on wakeup, move idle check * Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc * Dedupe destinations, fix other bug hopefully * Dispatch sends again * Federation sender to ignore invites that are destined locally * Loopback invite events * Remodel a bit with channels * Linter * Only loopback invite event if we know the room * We should tell other resident servers about the invite if we know about the room * Correct invite signing * Fix invite loopback * Check HTTP response codes, push new invites to front of queue * Review comments
123 lines
4 KiB
Go
123 lines
4 KiB
Go
package types
|
|
|
|
import (
|
|
"math"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/matrix-org/gomatrixserverlib"
|
|
"go.uber.org/atomic"
|
|
)
|
|
|
|
const (
|
|
// How many times should we tolerate consecutive failures before we
|
|
// just blacklist the host altogether? Bear in mind that the backoff
|
|
// is exponential, so the max time here to attempt is 2**failures.
|
|
FailuresUntilBlacklist = 16 // 16 equates to roughly 18 hours.
|
|
)
|
|
|
|
// Statistics contains information about all of the remote federated
|
|
// hosts that we have interacted with. It is basically a threadsafe
|
|
// wrapper.
|
|
type Statistics struct {
|
|
servers map[gomatrixserverlib.ServerName]*ServerStatistics
|
|
mutex sync.RWMutex
|
|
}
|
|
|
|
// ForServer returns server statistics for the given server name. If it
|
|
// does not exist, it will create empty statistics and return those.
|
|
func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerStatistics {
|
|
// If the map hasn't been initialised yet then do that.
|
|
if s.servers == nil {
|
|
s.mutex.Lock()
|
|
s.servers = make(map[gomatrixserverlib.ServerName]*ServerStatistics)
|
|
s.mutex.Unlock()
|
|
}
|
|
// Look up if we have statistics for this server already.
|
|
s.mutex.RLock()
|
|
server, found := s.servers[serverName]
|
|
s.mutex.RUnlock()
|
|
// If we don't, then make one.
|
|
if !found {
|
|
s.mutex.Lock()
|
|
server = &ServerStatistics{}
|
|
s.servers[serverName] = server
|
|
s.mutex.Unlock()
|
|
}
|
|
return server
|
|
}
|
|
|
|
// ServerStatistics contains information about our interactions with a
|
|
// remote federated host, e.g. how many times we were successful, how
|
|
// many times we failed etc. It also manages the backoff time and black-
|
|
// listing a remote host if it remains uncooperative.
|
|
type ServerStatistics struct {
|
|
blacklisted atomic.Bool // is the remote side dead?
|
|
backoffUntil atomic.Value // time.Time to wait until before sending requests
|
|
failCounter atomic.Uint32 // how many times have we failed?
|
|
successCounter atomic.Uint32 // how many times have we succeeded?
|
|
}
|
|
|
|
// Success updates the server statistics with a new successful
|
|
// attempt, which increases the sent counter and resets the idle and
|
|
// failure counters. If a host was blacklisted at this point then
|
|
// we will unblacklist it.
|
|
func (s *ServerStatistics) Success() {
|
|
s.successCounter.Add(1)
|
|
s.failCounter.Store(0)
|
|
s.blacklisted.Store(false)
|
|
}
|
|
|
|
// Failure marks a failure and works out when to backoff until. It
|
|
// returns true if the worker should give up altogether because of
|
|
// too many consecutive failures. At this point the host is marked
|
|
// as blacklisted.
|
|
func (s *ServerStatistics) Failure() bool {
|
|
// Increase the fail counter.
|
|
failCounter := s.failCounter.Add(1)
|
|
|
|
// Check that we haven't failed more times than is acceptable.
|
|
if failCounter >= FailuresUntilBlacklist {
|
|
// We've exceeded the maximum amount of times we're willing
|
|
// to back off, which is probably in the region of hours by
|
|
// now. Mark the host as blacklisted and tell the caller to
|
|
// give up.
|
|
s.blacklisted.Store(true)
|
|
return true
|
|
}
|
|
|
|
// We're still under the threshold so work out the exponential
|
|
// backoff based on how many times we have failed already. The
|
|
// worker goroutine will wait until this time before processing
|
|
// anything from the queue.
|
|
backoffSeconds := time.Second * time.Duration(math.Exp2(float64(failCounter)))
|
|
s.backoffUntil.Store(
|
|
time.Now().Add(backoffSeconds),
|
|
)
|
|
return false
|
|
}
|
|
|
|
// BackoffDuration returns both a bool stating whether to wait,
|
|
// and then if true, a duration to wait for.
|
|
func (s *ServerStatistics) BackoffDuration() (bool, time.Duration) {
|
|
backoff, until := false, time.Second
|
|
if b, ok := s.backoffUntil.Load().(time.Time); ok {
|
|
if b.After(time.Now()) {
|
|
backoff, until = true, time.Until(b)
|
|
}
|
|
}
|
|
return backoff, until
|
|
}
|
|
|
|
// Blacklisted returns true if the server is blacklisted and false
|
|
// otherwise.
|
|
func (s *ServerStatistics) Blacklisted() bool {
|
|
return s.blacklisted.Load()
|
|
}
|
|
|
|
// SuccessCount returns the number of successful requests. This is
|
|
// usually useful in constructing transaction IDs.
|
|
func (s *ServerStatistics) SuccessCount() uint32 {
|
|
return s.successCounter.Load()
|
|
}
|