diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go index edfe6cdb0..bfc8a3359 100644 --- a/cmd/dendrite-polylith-multi/main.go +++ b/cmd/dendrite-polylith-multi/main.go @@ -31,7 +31,7 @@ import ( type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite) func main() { - cfg := setup.ParseFlags(true) + cfg := setup.ParseFlags(false) component := "" if flag.NFlag() > 0 { diff --git a/setup/config/config.go b/setup/config/config.go index eb371a54b..14ce32df6 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -79,6 +79,8 @@ type Dendrite struct { // Any information derived from the configuration options for later use. Derived Derived `yaml:"-"` + + IsMonolith bool `yaml:"-"` } // TODO: Kill Derived @@ -211,6 +213,7 @@ func loadConfig( ) (*Dendrite, error) { var c Dendrite c.Defaults(false) + c.IsMonolith = monolithic var err error if err = yaml.Unmarshal(configData, &c); err != nil { diff --git a/userapi/userapi.go b/userapi/userapi.go index 1963621ae..81e37b95c 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -15,19 +15,10 @@ package userapi import ( - "bytes" - "context" - "encoding/json" - "math" - "net/http" - "runtime" - "syscall" "time" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/internal/pushgateway" - version "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/sqlutil" keyapi "github.com/matrix-org/dendrite/keyserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" @@ -39,7 +30,7 @@ import ( "github.com/matrix-org/dendrite/userapi/inthttp" "github.com/matrix-org/dendrite/userapi/producers" "github.com/matrix-org/dendrite/userapi/storage" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/userapi/util" "github.com/sirupsen/logrus" ) @@ -101,201 +92,10 @@ func NewInternalAPI( } time.AfterFunc(time.Minute, cleanOldNotifs) + if base.Cfg.Global.ReportStats { + go util.StartPhoneHomeCollector(time.Now(), base.Cfg, db) + } + return userAPI } -type phoneHomeStats struct { - prevData timestampToRUUsage - stats map[string]interface{} - serverName gomatrixserverlib.ServerName - startTime time.Time - cfg *config.Dendrite - db storage.Database - isMonolith bool - client *http.Client -} - -type timestampToRUUsage struct { - timestamp int64 - usage syscall.Rusage -} - -func startPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, userDB storage.Database, isMonolith bool) { - - p := phoneHomeStats{ - startTime: startTime, - serverName: cfg.Global.ServerName, - cfg: cfg, - db: userDB, - isMonolith: isMonolith, - client: &http.Client{ - Timeout: time.Second * 30, - }, - } - - // start initial run after 5min - time.AfterFunc(time.Second*1, func() { - p.collect() - }) - - // run every 3 hours - ticker := time.NewTicker(time.Second * 10) - - for { - select { - case <-ticker.C: - p.collect() - } - } -} - -func (p *phoneHomeStats) collect() { - p.stats = make(map[string]interface{}) - // general information - p.stats["homeserver"] = p.serverName - p.stats["monolith"] = p.isMonolith - p.stats["version"] = version.VersionString() - p.stats["timestamp"] = time.Now().Unix() - p.stats["go_version"] = runtime.Version() - p.stats["go_arch"] = runtime.GOARCH - p.stats["go_os"] = runtime.GOOS - p.stats["num_cpu"] = runtime.NumCPU() - p.stats["num_go_routine"] = runtime.NumGoroutine() - p.stats["uptime_seconds"] = math.Floor(time.Now().Sub(p.startTime).Seconds()) - - ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*1) - defer cancel() - - // cpu and memory usage information - err := getMemoryStats(p) - if err != nil { - logrus.WithError(err).Error("unable to get memory/cpu stats") - return - } - - // configuration information - p.stats["federation_disabled"] = p.cfg.Global.DisableFederation - p.stats["nats_embedded"] = true - p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory - if len(p.cfg.Global.JetStream.Addresses) > 0 { - p.stats["nats_embedded"] = false - p.stats["nats_in_memory"] = false // probably - } - if len(p.cfg.Logging) > 0 { - p.stats["log_level"] = p.cfg.Logging[0].Level - } else { - p.stats["log_level"] = "info" - } - - // database configuration - db, err := sqlutil.Open(&p.cfg.SyncAPI.Database) - if err != nil { - logrus.WithError(err).Error("unable to database") - return - } - defer db.Close() - - dbVersion := "unknown" - dbEngine := "unknown" - switch { - case p.cfg.SyncAPI.Database.ConnectionString.IsSQLite(): - dbEngine = "SQLite" - row := db.QueryRow("select sqlite_version();") - if err := row.Scan(&dbVersion); err != nil { - logrus.WithError(err).Error("unable to query version") - return - } - case p.cfg.SyncAPI.Database.ConnectionString.IsPostgres(): - dbEngine = "Postgres" - row := db.QueryRow("SHOW server_version;") - if err := row.Scan(&dbVersion); err != nil { - logrus.WithError(err).Error("unable to query version") - return - } - } - p.stats["database_engine"] = dbEngine - p.stats["database_server_version"] = dbVersion - - // message and room stats - // TODO: Find a solution to actually set these values - p.stats["total_room_count"] = 0 - p.stats["daily_messages"] = 0 - p.stats["daily_sent_messages"] = 0 - p.stats["daily_e2ee_messages"] = 0 - p.stats["daily_sent_e2ee_messages"] = 0 - - count, err := p.db.AllUsers(ctx) - if err != nil { - logrus.WithError(err).Error("unable to query AllUsers") - return - } - p.stats["total_users"] = count - - count, err = p.db.NonBridgedUsers(ctx) - if err != nil { - logrus.WithError(err).Error("unable to query NonBridgedUsers") - return - } - p.stats["total_nonbridged_users"] = count - - count, err = p.db.DailyUsers(ctx) - if err != nil { - logrus.WithError(err).Error("unable to query DailyUsers") - return - } - p.stats["daily_active_users"] = count - - count, err = p.db.MonthlyUsers(ctx) - if err != nil { - logrus.WithError(err).Error("unable to query MonthlyUsers") - return - } - p.stats["monthly_active_users"] = count - - res, err := p.db.RegisteredUserByType(ctx) - if err != nil { - logrus.WithError(err).Error("unable to query RegisteredUserByType") - return - } - for t, c := range res { - p.stats["daily_user_type_"+t] = c - } - - res, err = p.db.R30Users(ctx) - if err != nil { - logrus.WithError(err).Error("unable to query R30Users") - return - } - for t, c := range res { - p.stats["r30_users_"+t] = c - } - res, err = p.db.R30UsersV2(ctx) - if err != nil { - logrus.WithError(err).Error("unable to query R30UsersV2") - return - } - for t, c := range res { - p.stats["r30v2_users_"+t] = c - } - - output := bytes.Buffer{} - if err = json.NewEncoder(&output).Encode(p.stats); err != nil { - logrus.WithError(err).Error("unable to encode stats") - return - } - - logrus.Infof("Reporting stats to %s: %s", p.cfg.Global.ReportStatsEndpoint, output.String()) - - request, err := http.NewRequest("POST", p.cfg.Global.ReportStatsEndpoint, &output) - if err != nil { - logrus.WithError(err).Error("unable to create phone home stats request") - return - } - request.Header.Set("User-Agent", "Dendrite/"+version.VersionString()) - - _, err = p.client.Do(request) - if err != nil { - logrus.WithError(err).Error("unable to send phone home stats") - return - } -} \ No newline at end of file diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go new file mode 100644 index 000000000..b46aca4be --- /dev/null +++ b/userapi/util/phonehomestats.go @@ -0,0 +1,229 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "bytes" + "context" + "encoding/json" + "math" + "net/http" + "runtime" + "syscall" + "time" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/userapi/storage" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +type phoneHomeStats struct { + prevData timestampToRUUsage + stats map[string]interface{} + serverName gomatrixserverlib.ServerName + startTime time.Time + cfg *config.Dendrite + db storage.Database + isMonolith bool + client *http.Client +} + +type timestampToRUUsage struct { + timestamp int64 + usage syscall.Rusage +} + +func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, userDB storage.Database) { + + p := phoneHomeStats{ + startTime: startTime, + serverName: cfg.Global.ServerName, + cfg: cfg, + db: userDB, + isMonolith: cfg.IsMonolith, + client: &http.Client{ + Timeout: time.Second * 30, + }, + } + + // start initial run after 5min + time.AfterFunc(time.Minute * 5, func() { + p.collect() + }) + + // run every 3 hours + ticker := time.NewTicker(time.Hour * 3) + + for { + select { + case <-ticker.C: + p.collect() + } + } +} + +func (p *phoneHomeStats) collect() { + p.stats = make(map[string]interface{}) + // general information + p.stats["homeserver"] = p.serverName + p.stats["monolith"] = p.isMonolith + p.stats["version"] = internal.VersionString() + p.stats["timestamp"] = time.Now().Unix() + p.stats["go_version"] = runtime.Version() + p.stats["go_arch"] = runtime.GOARCH + p.stats["go_os"] = runtime.GOOS + p.stats["num_cpu"] = runtime.NumCPU() + p.stats["num_go_routine"] = runtime.NumGoroutine() + p.stats["uptime_seconds"] = math.Floor(time.Now().Sub(p.startTime).Seconds()) + + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*1) + defer cancel() + + // cpu and memory usage information + err := getMemoryStats(p) + if err != nil { + logrus.WithError(err).Error("unable to get memory/cpu stats") + return + } + + // configuration information + p.stats["federation_disabled"] = p.cfg.Global.DisableFederation + p.stats["nats_embedded"] = true + p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory + if len(p.cfg.Global.JetStream.Addresses) > 0 { + p.stats["nats_embedded"] = false + p.stats["nats_in_memory"] = false // probably + } + if len(p.cfg.Logging) > 0 { + p.stats["log_level"] = p.cfg.Logging[0].Level + } else { + p.stats["log_level"] = "info" + } + + // database configuration + db, err := sqlutil.Open(&p.cfg.UserAPI.AccountDatabase) + if err != nil { + logrus.WithError(err).Error("unable to connecto to database") + return + } + defer db.Close() + + dbVersion := "unknown" + dbEngine := "unknown" + switch { + case p.cfg.UserAPI.AccountDatabase.ConnectionString.IsSQLite(): + dbEngine = "SQLite" + row := db.QueryRow("select sqlite_version();") + if err := row.Scan(&dbVersion); err != nil { + logrus.WithError(err).Error("unable to query version") + return + } + case p.cfg.UserAPI.AccountDatabase.ConnectionString.IsPostgres(): + dbEngine = "Postgres" + row := db.QueryRow("SHOW server_version;") + if err := row.Scan(&dbVersion); err != nil { + logrus.WithError(err).Error("unable to query version") + return + } + } + p.stats["database_engine"] = dbEngine + p.stats["database_server_version"] = dbVersion + + // message and room stats + // TODO: Find a solution to actually set these values + p.stats["total_room_count"] = 0 + p.stats["daily_messages"] = 0 + p.stats["daily_sent_messages"] = 0 + p.stats["daily_e2ee_messages"] = 0 + p.stats["daily_sent_e2ee_messages"] = 0 + + count, err := p.db.AllUsers(ctx) + if err != nil { + logrus.WithError(err).Error("unable to query AllUsers") + return + } + p.stats["total_users"] = count + + count, err = p.db.NonBridgedUsers(ctx) + if err != nil { + logrus.WithError(err).Error("unable to query NonBridgedUsers") + return + } + p.stats["total_nonbridged_users"] = count + + count, err = p.db.DailyUsers(ctx) + if err != nil { + logrus.WithError(err).Error("unable to query DailyUsers") + return + } + p.stats["daily_active_users"] = count + + count, err = p.db.MonthlyUsers(ctx) + if err != nil { + logrus.WithError(err).Error("unable to query MonthlyUsers") + return + } + p.stats["monthly_active_users"] = count + + res, err := p.db.RegisteredUserByType(ctx) + if err != nil { + logrus.WithError(err).Error("unable to query RegisteredUserByType") + return + } + for t, c := range res { + p.stats["daily_user_type_"+t] = c + } + + res, err = p.db.R30Users(ctx) + if err != nil { + logrus.WithError(err).Error("unable to query R30Users") + return + } + for t, c := range res { + p.stats["r30_users_"+t] = c + } + res, err = p.db.R30UsersV2(ctx) + if err != nil { + logrus.WithError(err).Error("unable to query R30UsersV2") + return + } + for t, c := range res { + p.stats["r30v2_users_"+t] = c + } + + output := bytes.Buffer{} + if err = json.NewEncoder(&output).Encode(p.stats); err != nil { + logrus.WithError(err).Error("unable to encode stats") + return + } + + logrus.Infof("Reporting stats to %s: %s", p.cfg.Global.ReportStatsEndpoint, output.String()) + + request, err := http.NewRequest("POST", p.cfg.Global.ReportStatsEndpoint, &output) + if err != nil { + logrus.WithError(err).Error("unable to create phone home stats request") + return + } + request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString()) + + _, err = p.client.Do(request) + if err != nil { + logrus.WithError(err).Warn("unable to send phone home stats") + return + } +} \ No newline at end of file diff --git a/userapi/stats.go b/userapi/util/stats.go similarity index 96% rename from userapi/stats.go rename to userapi/util/stats.go index 64cb5476f..b6475cf7d 100644 --- a/userapi/stats.go +++ b/userapi/util/stats.go @@ -15,7 +15,7 @@ //go:build !wasm // +build !wasm -package userapi +package util import ( "math" @@ -38,7 +38,6 @@ func getMemoryStats(p *phoneHomeStats) error { usedCPUTime := (newUsage.Utime.Sec + newUsage.Stime.Sec) - (oldUsage.usage.Utime.Sec + oldUsage.usage.Stime.Sec) if usedCPUTime == 0 || newData.timestamp == oldUsage.timestamp { - logrus.Info("setting cpu average to 0") p.stats["cpu_average"] = 0 } else { p.stats["cpu_average"] = math.Floor(float64(usedCPUTime / (newData.timestamp - oldUsage.timestamp) * 100)) diff --git a/userapi/stats_wasm.go b/userapi/util/stats_wasm.go similarity index 97% rename from userapi/stats_wasm.go rename to userapi/util/stats_wasm.go index 2dfed7c04..a182e4e6e 100644 --- a/userapi/stats_wasm.go +++ b/userapi/util/stats_wasm.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package userapi +package util // stub, since WASM doesn't support syscall.Getrusage func getMemoryStats(p *phoneHomeStats) error {