mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-01 03:03:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Move phone home stats to util package

Browse source
This commit is contained in:
Till Faelligen 2022-03-04 08:48:10 +01:00
parent 623df231ee
commit 8500f82149
6 changed files with 240 additions and 209 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/dendrite-polylith-multi/main.go
View file

@ -31,7 +31,7 @@ import (
type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite)
func main() {
cfg := setup.ParseFlags(true)
cfg := setup.ParseFlags(false)
component := ""
if flag.NFlag() > 0 {

3
setup/config/config.go
View file

@ -79,6 +79,8 @@ type Dendrite struct {
// Any information derived from the configuration options for later use.
Derived Derived `yaml:"-"`
IsMonolith bool `yaml:"-"`
}
// TODO: Kill Derived
@ -211,6 +213,7 @@ func loadConfig(
) (*Dendrite, error) {
var c Dendrite
c.Defaults(false)
c.IsMonolith = monolithic
var err error
if err = yaml.Unmarshal(configData, &c); err != nil {

210
userapi/userapi.go
View file

@ -15,19 +15,10 @@
package userapi
import (
"bytes"
"context"
"encoding/json"
"math"
"net/http"
"runtime"
"syscall"
"time"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/internal/pushgateway"
version "github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
keyapi "github.com/matrix-org/dendrite/keyserver/api"
rsapi "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/base"
@ -39,7 +30,7 @@ import (
"github.com/matrix-org/dendrite/userapi/inthttp"
"github.com/matrix-org/dendrite/userapi/producers"
"github.com/matrix-org/dendrite/userapi/storage"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/userapi/util"
"github.com/sirupsen/logrus"
)
@ -101,201 +92,10 @@ func NewInternalAPI(
}
time.AfterFunc(time.Minute, cleanOldNotifs)
if base.Cfg.Global.ReportStats {
go util.StartPhoneHomeCollector(time.Now(), base.Cfg, db)
}
return userAPI
}
type phoneHomeStats struct {
prevData timestampToRUUsage
stats map[string]interface{}
serverName gomatrixserverlib.ServerName
startTime time.Time
cfg *config.Dendrite
db storage.Database
isMonolith bool
client *http.Client
}
type timestampToRUUsage struct {
timestamp int64
usage syscall.Rusage
}
func startPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, userDB storage.Database, isMonolith bool) {
p := phoneHomeStats{
startTime: startTime,
serverName: cfg.Global.ServerName,
cfg: cfg,
db: userDB,
isMonolith: isMonolith,
client: &http.Client{
Timeout: time.Second * 30,
},
}
// start initial run after 5min
time.AfterFunc(time.Second*1, func() {
p.collect()
})
// run every 3 hours
ticker := time.NewTicker(time.Second * 10)
for {
select {
case <-ticker.C:
p.collect()
}
}
}
func (p *phoneHomeStats) collect() {
p.stats = make(map[string]interface{})
// general information
p.stats["homeserver"] = p.serverName
p.stats["monolith"] = p.isMonolith
p.stats["version"] = version.VersionString()
p.stats["timestamp"] = time.Now().Unix()
p.stats["go_version"] = runtime.Version()
p.stats["go_arch"] = runtime.GOARCH
p.stats["go_os"] = runtime.GOOS
p.stats["num_cpu"] = runtime.NumCPU()
p.stats["num_go_routine"] = runtime.NumGoroutine()
p.stats["uptime_seconds"] = math.Floor(time.Now().Sub(p.startTime).Seconds())
ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*1)
defer cancel()
// cpu and memory usage information
err := getMemoryStats(p)
if err != nil {
logrus.WithError(err).Error("unable to get memory/cpu stats")
return
}
// configuration information
p.stats["federation_disabled"] = p.cfg.Global.DisableFederation
p.stats["nats_embedded"] = true
p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory
if len(p.cfg.Global.JetStream.Addresses) > 0 {
p.stats["nats_embedded"] = false
p.stats["nats_in_memory"] = false // probably
}
if len(p.cfg.Logging) > 0 {
p.stats["log_level"] = p.cfg.Logging[0].Level
} else {
p.stats["log_level"] = "info"
}
// database configuration
db, err := sqlutil.Open(&p.cfg.SyncAPI.Database)
if err != nil {
logrus.WithError(err).Error("unable to database")
return
}
defer db.Close()
dbVersion := "unknown"
dbEngine := "unknown"
switch {
case p.cfg.SyncAPI.Database.ConnectionString.IsSQLite():
dbEngine = "SQLite"
row := db.QueryRow("select sqlite_version();")
if err := row.Scan(&dbVersion); err != nil {
logrus.WithError(err).Error("unable to query version")
return
}
case p.cfg.SyncAPI.Database.ConnectionString.IsPostgres():
dbEngine = "Postgres"
row := db.QueryRow("SHOW server_version;")
if err := row.Scan(&dbVersion); err != nil {
logrus.WithError(err).Error("unable to query version")
return
}
}
p.stats["database_engine"] = dbEngine
p.stats["database_server_version"] = dbVersion
// message and room stats
// TODO: Find a solution to actually set these values
p.stats["total_room_count"] = 0
p.stats["daily_messages"] = 0
p.stats["daily_sent_messages"] = 0
p.stats["daily_e2ee_messages"] = 0
p.stats["daily_sent_e2ee_messages"] = 0
count, err := p.db.AllUsers(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query AllUsers")
return
}
p.stats["total_users"] = count
count, err = p.db.NonBridgedUsers(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query NonBridgedUsers")
return
}
p.stats["total_nonbridged_users"] = count
count, err = p.db.DailyUsers(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query DailyUsers")
return
}
p.stats["daily_active_users"] = count
count, err = p.db.MonthlyUsers(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query MonthlyUsers")
return
}
p.stats["monthly_active_users"] = count
res, err := p.db.RegisteredUserByType(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query RegisteredUserByType")
return
}
for t, c := range res {
p.stats["daily_user_type_"+t] = c
}
res, err = p.db.R30Users(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query R30Users")
return
}
for t, c := range res {
p.stats["r30_users_"+t] = c
}
res, err = p.db.R30UsersV2(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query R30UsersV2")
return
}
for t, c := range res {
p.stats["r30v2_users_"+t] = c
}
output := bytes.Buffer{}
if err = json.NewEncoder(&output).Encode(p.stats); err != nil {
logrus.WithError(err).Error("unable to encode stats")
return
}
logrus.Infof("Reporting stats to %s: %s", p.cfg.Global.ReportStatsEndpoint, output.String())
request, err := http.NewRequest("POST", p.cfg.Global.ReportStatsEndpoint, &output)
if err != nil {
logrus.WithError(err).Error("unable to create phone home stats request")
return
}
request.Header.Set("User-Agent", "Dendrite/"+version.VersionString())
_, err = p.client.Do(request)
if err != nil {
logrus.WithError(err).Error("unable to send phone home stats")
return
}
}

229
userapi/util/phonehomestats.go Normal file
View file

@ -0,0 +1,229 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"bytes"
"context"
"encoding/json"
"math"
"net/http"
"runtime"
"syscall"
"time"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/storage"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
)
type phoneHomeStats struct {
prevData timestampToRUUsage
stats map[string]interface{}
serverName gomatrixserverlib.ServerName
startTime time.Time
cfg *config.Dendrite
db storage.Database
isMonolith bool
client *http.Client
}
type timestampToRUUsage struct {
timestamp int64
usage syscall.Rusage
}
func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, userDB storage.Database) {
p := phoneHomeStats{
startTime: startTime,
serverName: cfg.Global.ServerName,
cfg: cfg,
db: userDB,
isMonolith: cfg.IsMonolith,
client: &http.Client{
Timeout: time.Second * 30,
},
}
// start initial run after 5min
time.AfterFunc(time.Minute * 5, func() {
p.collect()
})
// run every 3 hours
ticker := time.NewTicker(time.Hour * 3)
for {
select {
case <-ticker.C:
p.collect()
}
}
}
func (p *phoneHomeStats) collect() {
p.stats = make(map[string]interface{})
// general information
p.stats["homeserver"] = p.serverName
p.stats["monolith"] = p.isMonolith
p.stats["version"] = internal.VersionString()
p.stats["timestamp"] = time.Now().Unix()
p.stats["go_version"] = runtime.Version()
p.stats["go_arch"] = runtime.GOARCH
p.stats["go_os"] = runtime.GOOS
p.stats["num_cpu"] = runtime.NumCPU()
p.stats["num_go_routine"] = runtime.NumGoroutine()
p.stats["uptime_seconds"] = math.Floor(time.Now().Sub(p.startTime).Seconds())
ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*1)
defer cancel()
// cpu and memory usage information
err := getMemoryStats(p)
if err != nil {
logrus.WithError(err).Error("unable to get memory/cpu stats")
return
}
// configuration information
p.stats["federation_disabled"] = p.cfg.Global.DisableFederation
p.stats["nats_embedded"] = true
p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory
if len(p.cfg.Global.JetStream.Addresses) > 0 {
p.stats["nats_embedded"] = false
p.stats["nats_in_memory"] = false // probably
}
if len(p.cfg.Logging) > 0 {
p.stats["log_level"] = p.cfg.Logging[0].Level
} else {
p.stats["log_level"] = "info"
}
// database configuration
db, err := sqlutil.Open(&p.cfg.UserAPI.AccountDatabase)
if err != nil {
logrus.WithError(err).Error("unable to connecto to database")
return
}
defer db.Close()
dbVersion := "unknown"
dbEngine := "unknown"
switch {
case p.cfg.UserAPI.AccountDatabase.ConnectionString.IsSQLite():
dbEngine = "SQLite"
row := db.QueryRow("select sqlite_version();")
if err := row.Scan(&dbVersion); err != nil {
logrus.WithError(err).Error("unable to query version")
return
}
case p.cfg.UserAPI.AccountDatabase.ConnectionString.IsPostgres():
dbEngine = "Postgres"
row := db.QueryRow("SHOW server_version;")
if err := row.Scan(&dbVersion); err != nil {
logrus.WithError(err).Error("unable to query version")
return
}
}
p.stats["database_engine"] = dbEngine
p.stats["database_server_version"] = dbVersion
// message and room stats
// TODO: Find a solution to actually set these values
p.stats["total_room_count"] = 0
p.stats["daily_messages"] = 0
p.stats["daily_sent_messages"] = 0
p.stats["daily_e2ee_messages"] = 0
p.stats["daily_sent_e2ee_messages"] = 0
count, err := p.db.AllUsers(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query AllUsers")
return
}
p.stats["total_users"] = count
count, err = p.db.NonBridgedUsers(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query NonBridgedUsers")
return
}
p.stats["total_nonbridged_users"] = count
count, err = p.db.DailyUsers(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query DailyUsers")
return
}
p.stats["daily_active_users"] = count
count, err = p.db.MonthlyUsers(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query MonthlyUsers")
return
}
p.stats["monthly_active_users"] = count
res, err := p.db.RegisteredUserByType(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query RegisteredUserByType")
return
}
for t, c := range res {
p.stats["daily_user_type_"+t] = c
}
res, err = p.db.R30Users(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query R30Users")
return
}
for t, c := range res {
p.stats["r30_users_"+t] = c
}
res, err = p.db.R30UsersV2(ctx)
if err != nil {
logrus.WithError(err).Error("unable to query R30UsersV2")
return
}
for t, c := range res {
p.stats["r30v2_users_"+t] = c
}
output := bytes.Buffer{}
if err = json.NewEncoder(&output).Encode(p.stats); err != nil {
logrus.WithError(err).Error("unable to encode stats")
return
}
logrus.Infof("Reporting stats to %s: %s", p.cfg.Global.ReportStatsEndpoint, output.String())
request, err := http.NewRequest("POST", p.cfg.Global.ReportStatsEndpoint, &output)
if err != nil {
logrus.WithError(err).Error("unable to create phone home stats request")
return
}
request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString())
_, err = p.client.Do(request)
if err != nil {
logrus.WithError(err).Warn("unable to send phone home stats")
return
}
}

3
userapi/stats.go → userapi/util/stats.go
View file

@ -15,7 +15,7 @@
//go:build !wasm
// +build !wasm
package userapi
package util
import (
"math"
@ -38,7 +38,6 @@ func getMemoryStats(p *phoneHomeStats) error {
usedCPUTime := (newUsage.Utime.Sec + newUsage.Stime.Sec) - (oldUsage.usage.Utime.Sec + oldUsage.usage.Stime.Sec)
if usedCPUTime == 0 || newData.timestamp == oldUsage.timestamp {
logrus.Info("setting cpu average to 0")
p.stats["cpu_average"] = 0
} else {
p.stats["cpu_average"] = math.Floor(float64(usedCPUTime / (newData.timestamp - oldUsage.timestamp) * 100))

2
userapi/stats_wasm.go → userapi/util/stats_wasm.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package userapi
package util
// stub, since WASM doesn't support syscall.Getrusage
func getMemoryStats(p *phoneHomeStats) error {