From e8dd37d533275233b7bebeeb70b996e166cf2d3b Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 8 Apr 2022 12:24:40 +0200 Subject: [PATCH] Add metrics for internal API requests (#2310) * Add response size and requests total to internal handler * Move MustRegister calls to New* funcs * Move MustRegister back to init * Init at some place, minimize changes --- clientapi/routing/register.go | 5 ---- clientapi/routing/routing.go | 3 +++ clientapi/routing/sendevent.go | 4 ---- federationapi/routing/routing.go | 5 ++++ federationapi/routing/send.go | 6 ----- internal/httputil/httpapi.go | 28 ++++++++++++++++++++--- roomserver/internal/input/input.go | 5 +--- roomserver/internal/input/input_events.go | 4 ---- syncapi/sync/requestpool.go | 9 +++----- 9 files changed, 37 insertions(+), 32 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 7d84f2494..8253f3155 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -64,11 +64,6 @@ const ( sessionIDLength = 24 ) -func init() { - // Register prometheus metrics. They must be registered to be exposed. - prometheus.MustRegister(amtRegUsers) -} - // sessionsDict keeps track of completed auth stages for each session. // It shouldn't be passed by value because it contains a mutex. type sessionsDict struct { diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 32e83187f..37d825b80 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -37,6 +37,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/nats-io/nats.go" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -60,6 +61,8 @@ func Setup( extRoomsProvider api.ExtraPublicRoomsProvider, mscCfg *config.MSCs, natsClient *nats.Conn, ) { + prometheus.MustRegister(amtRegUsers, sendEventDuration) + rateLimits := httputil.NewRateLimits(&cfg.RateLimiting) userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index c5884e80b..1211fa72d 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -46,10 +46,6 @@ var ( userRoomSendMutexes sync.Map // (roomID+userID) -> mutex. mutexes to ensure correct ordering of sendEvents ) -func init() { - prometheus.MustRegister(sendEventDuration) -} - var sendEventDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "dendrite", diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 9e5cdb28b..a085ed780 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -29,6 +29,7 @@ import ( userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -53,6 +54,10 @@ func Setup( servers federationAPI.ServersInRoomProvider, producer *producers.SyncAPIProducer, ) { + prometheus.MustRegister( + pduCountTotal, eduCountTotal, + ) + v2keysmux := keyMux.PathPrefix("/v2").Subrouter() v1fedmux := fedMux.PathPrefix("/v1").Subrouter() v2fedmux := fedMux.PathPrefix("/v2").Subrouter() diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index c3351388c..f2b902b6f 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -74,12 +74,6 @@ var ( ) ) -func init() { - prometheus.MustRegister( - pduCountTotal, eduCountTotal, - ) -} - var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse // Send implements /_matrix/federation/v1/send/{txnID} diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index 1a37a1eec..5fcacd2ad 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -169,8 +169,9 @@ func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request) return promhttp.InstrumentHandlerCounter( promauto.NewCounterVec( prometheus.CounterOpts{ - Name: metricsName, - Help: "Total number of http requests for HTML resources", + Name: metricsName, + Help: "Total number of http requests for HTML resources", + Namespace: "dendrite", }, []string{"code"}, ), @@ -201,7 +202,28 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse h.ServeHTTP(w, req) } - return http.HandlerFunc(withSpan) + return promhttp.InstrumentHandlerCounter( + promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: metricsName + "_requests_total", + Help: "Total number of internal API calls", + Namespace: "dendrite", + }, + []string{"code"}, + ), + promhttp.InstrumentHandlerResponseSize( + promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "dendrite", + Name: metricsName + "_response_size_bytes", + Help: "A histogram of response sizes for requests.", + Buckets: []float64{200, 500, 900, 1500, 5000, 15000, 50000, 100000}, + }, + []string{}, + ), + http.HandlerFunc(withSpan), + ), + ) } // MakeFedAPI makes an http.Handler that checks matrix federation authentication. diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index 6a8ae6d00..1fea6ef06 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -167,6 +167,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) { // will look to see if we have a worker for that room which has its // own consumer. If we don't, we'll start one. func (r *Inputer) Start() error { + prometheus.MustRegister(roomserverInputBackpressure, processRoomEventDuration) _, err := r.JetStream.Subscribe( "", // This is blank because we specified it in BindStream. func(m *nats.Msg) { @@ -421,10 +422,6 @@ func (r *Inputer) WriteOutputEvents(roomID string, updates []api.OutputEvent) er return nil } -func init() { - prometheus.MustRegister(roomserverInputBackpressure) -} - var roomserverInputBackpressure = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "dendrite", diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 1acbf0443..3ab9ba4f0 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -37,10 +37,6 @@ import ( "github.com/sirupsen/logrus" ) -func init() { - prometheus.MustRegister(processRoomEventDuration) -} - // TODO: Does this value make sense? const MaximumMissingProcessingTime = time.Minute * 2 diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 6fc08bd92..703340997 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -67,6 +67,9 @@ func NewRequestPool( streams *streams.Streams, notifier *notifier.Notifier, producer PresencePublisher, ) *RequestPool { + prometheus.MustRegister( + activeSyncRequests, waitingSyncRequests, + ) rp := &RequestPool{ db: db, cfg: cfg, @@ -183,12 +186,6 @@ func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) rp.lastseen.Store(device.UserID+device.ID, time.Now()) } -func init() { - prometheus.MustRegister( - activeSyncRequests, waitingSyncRequests, - ) -} - var activeSyncRequests = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "dendrite",