Move phone home stats to util package

cmd/dendrite-polylith-multi/main.go

@@ -31,7 +31,7 @@ import (
 type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite)
 
 func main() {
-	cfg := setup.ParseFlags(true)
+	cfg := setup.ParseFlags(false)
 
 	component := ""
 	if flag.NFlag() > 0 {

setup/config/config.go

@@ -79,6 +79,8 @@ type Dendrite struct {
 
 	// Any information derived from the configuration options for later use.
 	Derived Derived `yaml:"-"`
+
+	IsMonolith bool `yaml:"-"`
 }
 
 // TODO: Kill Derived
@@ -211,6 +213,7 @@ func loadConfig(
 ) (*Dendrite, error) {
 	var c Dendrite
 	c.Defaults(false)
+	c.IsMonolith = monolithic
 
 	var err error
 	if err = yaml.Unmarshal(configData, &c); err != nil {

userapi/userapi.go

@@ -15,19 +15,10 @@
 package userapi
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"math"
-	"net/http"
-	"runtime"
-	"syscall"
 	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/internal/pushgateway"
-	version "github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
@@ -39,7 +30,7 @@ import (
 	"github.com/matrix-org/dendrite/userapi/inthttp"
 	"github.com/matrix-org/dendrite/userapi/producers"
 	"github.com/matrix-org/dendrite/userapi/storage"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/dendrite/userapi/util"
 	"github.com/sirupsen/logrus"
 )
 
@@ -101,201 +92,10 @@ func NewInternalAPI(
 	}
 	time.AfterFunc(time.Minute, cleanOldNotifs)
 
+	if base.Cfg.Global.ReportStats {
+		go util.StartPhoneHomeCollector(time.Now(), base.Cfg, db)
+	}
+
 	return userAPI
 }
 
-type phoneHomeStats struct {
-	prevData   timestampToRUUsage
-	stats      map[string]interface{}
-	serverName gomatrixserverlib.ServerName
-	startTime  time.Time
-	cfg        *config.Dendrite
-	db         storage.Database
-	isMonolith bool
-	client     *http.Client
-}
-
-type timestampToRUUsage struct {
-	timestamp int64
-	usage     syscall.Rusage
-}
-
-func startPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, userDB storage.Database, isMonolith bool) {
-
-	p := phoneHomeStats{
-		startTime:  startTime,
-		serverName: cfg.Global.ServerName,
-		cfg:        cfg,
-		db:         userDB,
-		isMonolith: isMonolith,
-		client: &http.Client{
-			Timeout: time.Second * 30,
-		},
-	}
-
-	// start initial run after 5min
-	time.AfterFunc(time.Second*1, func() {
-		p.collect()
-	})
-
-	// run every 3 hours
-	ticker := time.NewTicker(time.Second * 10)
-
-	for {
-		select {
-		case <-ticker.C:
-			p.collect()
-		}
-	}
-}
-
-func (p *phoneHomeStats) collect() {
-	p.stats = make(map[string]interface{})
-	// general information
-	p.stats["homeserver"] = p.serverName
-	p.stats["monolith"] = p.isMonolith
-	p.stats["version"] = version.VersionString()
-	p.stats["timestamp"] = time.Now().Unix()
-	p.stats["go_version"] = runtime.Version()
-	p.stats["go_arch"] = runtime.GOARCH
-	p.stats["go_os"] = runtime.GOOS
-	p.stats["num_cpu"] = runtime.NumCPU()
-	p.stats["num_go_routine"] = runtime.NumGoroutine()
-	p.stats["uptime_seconds"] = math.Floor(time.Now().Sub(p.startTime).Seconds())
-
-	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*1)
-	defer cancel()
-
-	// cpu and memory usage information
-	err := getMemoryStats(p)
-	if err != nil {
-		logrus.WithError(err).Error("unable to get memory/cpu stats")
-		return
-	}
-
-	// configuration information
-	p.stats["federation_disabled"] = p.cfg.Global.DisableFederation
-	p.stats["nats_embedded"] = true
-	p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory
-	if len(p.cfg.Global.JetStream.Addresses) > 0 {
-		p.stats["nats_embedded"] = false
-		p.stats["nats_in_memory"] = false // probably
-	}
-	if len(p.cfg.Logging) > 0 {
-		p.stats["log_level"] = p.cfg.Logging[0].Level
-	} else {
-		p.stats["log_level"] = "info"
-	}
-
-	// database configuration
-	db, err := sqlutil.Open(&p.cfg.SyncAPI.Database)
-	if err != nil {
-		logrus.WithError(err).Error("unable to database")
-		return
-	}
-	defer db.Close()
-
-	dbVersion := "unknown"
-	dbEngine := "unknown"
-	switch {
-	case p.cfg.SyncAPI.Database.ConnectionString.IsSQLite():
-		dbEngine = "SQLite"
-		row := db.QueryRow("select sqlite_version();")
-		if err := row.Scan(&dbVersion); err != nil {
-			logrus.WithError(err).Error("unable to query version")
-			return
-		}
-	case p.cfg.SyncAPI.Database.ConnectionString.IsPostgres():
-		dbEngine = "Postgres"
-		row := db.QueryRow("SHOW server_version;")
-		if err := row.Scan(&dbVersion); err != nil {
-			logrus.WithError(err).Error("unable to query version")
-			return
-		}
-	}
-	p.stats["database_engine"] = dbEngine
-	p.stats["database_server_version"] = dbVersion
-
-	// message and room stats
-	// TODO: Find a solution to actually set these values
-	p.stats["total_room_count"] = 0
-	p.stats["daily_messages"] = 0
-	p.stats["daily_sent_messages"] = 0
-	p.stats["daily_e2ee_messages"] = 0
-	p.stats["daily_sent_e2ee_messages"] = 0
-
-	count, err := p.db.AllUsers(ctx)
-	if err != nil {
-		logrus.WithError(err).Error("unable to query AllUsers")
-		return
-	}
-	p.stats["total_users"] = count
-
-	count, err = p.db.NonBridgedUsers(ctx)
-	if err != nil {
-		logrus.WithError(err).Error("unable to query NonBridgedUsers")
-		return
-	}
-	p.stats["total_nonbridged_users"] = count
-
-	count, err = p.db.DailyUsers(ctx)
-	if err != nil {
-		logrus.WithError(err).Error("unable to query DailyUsers")
-		return
-	}
-	p.stats["daily_active_users"] = count
-
-	count, err = p.db.MonthlyUsers(ctx)
-	if err != nil {
-		logrus.WithError(err).Error("unable to query MonthlyUsers")
-		return
-	}
-	p.stats["monthly_active_users"] = count
-
-	res, err := p.db.RegisteredUserByType(ctx)
-	if err != nil {
-		logrus.WithError(err).Error("unable to query RegisteredUserByType")
-		return
-	}
-	for t, c := range res {
-		p.stats["daily_user_type_"+t] = c
-	}
-
-	res, err = p.db.R30Users(ctx)
-	if err != nil {
-		logrus.WithError(err).Error("unable to query R30Users")
-		return
-	}
-	for t, c := range res {
-		p.stats["r30_users_"+t] = c
-	}
-	res, err = p.db.R30UsersV2(ctx)
-	if err != nil {
-		logrus.WithError(err).Error("unable to query R30UsersV2")
-		return
-	}
-	for t, c := range res {
-		p.stats["r30v2_users_"+t] = c
-	}
-
-	output := bytes.Buffer{}
-	if err = json.NewEncoder(&output).Encode(p.stats); err != nil {
-		logrus.WithError(err).Error("unable to encode stats")
-		return
-	}
-
-	logrus.Infof("Reporting stats to %s: %s", p.cfg.Global.ReportStatsEndpoint, output.String())
-
-	request, err := http.NewRequest("POST", p.cfg.Global.ReportStatsEndpoint, &output)
-	if err != nil {
-		logrus.WithError(err).Error("unable to create phone home stats request")
-		return
-	}
-	request.Header.Set("User-Agent", "Dendrite/"+version.VersionString())
-
-	_, err = p.client.Do(request)
-	if err != nil {
-		logrus.WithError(err).Error("unable to send phone home stats")
-		return
-	}
-}

userapi/util/phonehomestats.go

@@ -0,0 +1,229 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"math"
+	"net/http"
+	"runtime"
+	"syscall"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/userapi/storage"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+type phoneHomeStats struct {
+	prevData   timestampToRUUsage
+	stats      map[string]interface{}
+	serverName gomatrixserverlib.ServerName
+	startTime  time.Time
+	cfg        *config.Dendrite
+	db         storage.Database
+	isMonolith bool
+	client     *http.Client
+}
+
+type timestampToRUUsage struct {
+	timestamp int64
+	usage     syscall.Rusage
+}
+
+func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, userDB storage.Database) {
+
+	p := phoneHomeStats{
+		startTime:  startTime,
+		serverName: cfg.Global.ServerName,
+		cfg:        cfg,
+		db:         userDB,
+		isMonolith: cfg.IsMonolith,
+		client: &http.Client{
+			Timeout: time.Second * 30,
+		},
+	}
+
+	// start initial run after 5min
+	time.AfterFunc(time.Minute * 5, func() {
+		p.collect()
+	})
+
+	// run every 3 hours
+	ticker := time.NewTicker(time.Hour * 3)
+
+	for {
+		select {
+		case <-ticker.C:
+			p.collect()
+		}
+	}
+}
+
+func (p *phoneHomeStats) collect() {
+	p.stats = make(map[string]interface{})
+	// general information
+	p.stats["homeserver"] = p.serverName
+	p.stats["monolith"] = p.isMonolith
+	p.stats["version"] = internal.VersionString()
+	p.stats["timestamp"] = time.Now().Unix()
+	p.stats["go_version"] = runtime.Version()
+	p.stats["go_arch"] = runtime.GOARCH
+	p.stats["go_os"] = runtime.GOOS
+	p.stats["num_cpu"] = runtime.NumCPU()
+	p.stats["num_go_routine"] = runtime.NumGoroutine()
+	p.stats["uptime_seconds"] = math.Floor(time.Now().Sub(p.startTime).Seconds())
+
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*1)
+	defer cancel()
+
+	// cpu and memory usage information
+	err := getMemoryStats(p)
+	if err != nil {
+		logrus.WithError(err).Error("unable to get memory/cpu stats")
+		return
+	}
+
+	// configuration information
+	p.stats["federation_disabled"] = p.cfg.Global.DisableFederation
+	p.stats["nats_embedded"] = true
+	p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory
+	if len(p.cfg.Global.JetStream.Addresses) > 0 {
+		p.stats["nats_embedded"] = false
+		p.stats["nats_in_memory"] = false // probably
+	}
+	if len(p.cfg.Logging) > 0 {
+		p.stats["log_level"] = p.cfg.Logging[0].Level
+	} else {
+		p.stats["log_level"] = "info"
+	}
+
+	// database configuration
+	db, err := sqlutil.Open(&p.cfg.UserAPI.AccountDatabase)
+	if err != nil {
+		logrus.WithError(err).Error("unable to connecto to database")
+		return
+	}
+	defer db.Close()
+
+	dbVersion := "unknown"
+	dbEngine := "unknown"
+	switch {
+	case p.cfg.UserAPI.AccountDatabase.ConnectionString.IsSQLite():
+		dbEngine = "SQLite"
+		row := db.QueryRow("select sqlite_version();")
+		if err := row.Scan(&dbVersion); err != nil {
+			logrus.WithError(err).Error("unable to query version")
+			return
+		}
+	case p.cfg.UserAPI.AccountDatabase.ConnectionString.IsPostgres():
+		dbEngine = "Postgres"
+		row := db.QueryRow("SHOW server_version;")
+		if err := row.Scan(&dbVersion); err != nil {
+			logrus.WithError(err).Error("unable to query version")
+			return
+		}
+	}
+	p.stats["database_engine"] = dbEngine
+	p.stats["database_server_version"] = dbVersion
+
+	// message and room stats
+	// TODO: Find a solution to actually set these values
+	p.stats["total_room_count"] = 0
+	p.stats["daily_messages"] = 0
+	p.stats["daily_sent_messages"] = 0
+	p.stats["daily_e2ee_messages"] = 0
+	p.stats["daily_sent_e2ee_messages"] = 0
+
+	count, err := p.db.AllUsers(ctx)
+	if err != nil {
+		logrus.WithError(err).Error("unable to query AllUsers")
+		return
+	}
+	p.stats["total_users"] = count
+
+	count, err = p.db.NonBridgedUsers(ctx)
+	if err != nil {
+		logrus.WithError(err).Error("unable to query NonBridgedUsers")
+		return
+	}
+	p.stats["total_nonbridged_users"] = count
+
+	count, err = p.db.DailyUsers(ctx)
+	if err != nil {
+		logrus.WithError(err).Error("unable to query DailyUsers")
+		return
+	}
+	p.stats["daily_active_users"] = count
+
+	count, err = p.db.MonthlyUsers(ctx)
+	if err != nil {
+		logrus.WithError(err).Error("unable to query MonthlyUsers")
+		return
+	}
+	p.stats["monthly_active_users"] = count
+
+	res, err := p.db.RegisteredUserByType(ctx)
+	if err != nil {
+		logrus.WithError(err).Error("unable to query RegisteredUserByType")
+		return
+	}
+	for t, c := range res {
+		p.stats["daily_user_type_"+t] = c
+	}
+
+	res, err = p.db.R30Users(ctx)
+	if err != nil {
+		logrus.WithError(err).Error("unable to query R30Users")
+		return
+	}
+	for t, c := range res {
+		p.stats["r30_users_"+t] = c
+	}
+	res, err = p.db.R30UsersV2(ctx)
+	if err != nil {
+		logrus.WithError(err).Error("unable to query R30UsersV2")
+		return
+	}
+	for t, c := range res {
+		p.stats["r30v2_users_"+t] = c
+	}
+
+	output := bytes.Buffer{}
+	if err = json.NewEncoder(&output).Encode(p.stats); err != nil {
+		logrus.WithError(err).Error("unable to encode stats")
+		return
+	}
+
+	logrus.Infof("Reporting stats to %s: %s", p.cfg.Global.ReportStatsEndpoint, output.String())
+
+	request, err := http.NewRequest("POST", p.cfg.Global.ReportStatsEndpoint, &output)
+	if err != nil {
+		logrus.WithError(err).Error("unable to create phone home stats request")
+		return
+	}
+	request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString())
+
+	_, err = p.client.Do(request)
+	if err != nil {
+		logrus.WithError(err).Warn("unable to send phone home stats")
+		return
+	}
+}

userapi/util/stats.go

@@ -15,7 +15,7 @@
 //go:build !wasm
 // +build !wasm
 
-package userapi
+package util
 
 import (
 	"math"
@@ -38,7 +38,6 @@ func getMemoryStats(p *phoneHomeStats) error {
 	usedCPUTime := (newUsage.Utime.Sec + newUsage.Stime.Sec) - (oldUsage.usage.Utime.Sec + oldUsage.usage.Stime.Sec)
 
 	if usedCPUTime == 0 || newData.timestamp == oldUsage.timestamp {
-		logrus.Info("setting cpu average to 0")
 		p.stats["cpu_average"] = 0
 	} else {
 		p.stats["cpu_average"] = math.Floor(float64(usedCPUTime / (newData.timestamp - oldUsage.timestamp) * 100))

userapi/util/stats_wasm.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package userapi
+package util
 
 // stub, since WASM doesn't support syscall.Getrusage
 func getMemoryStats(p *phoneHomeStats) error {