From a01ebba7877e06716560d947d0c10ba49ec9623d Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Wed, 2 Mar 2022 16:12:08 +0100 Subject: [PATCH] userapi stats tables & queries --- userapi/storage/interface.go | 8 +++ userapi/storage/postgres/stats_table.go | 95 ++++++++++++++++++++----- userapi/storage/shared/storage.go | 22 ++++++ userapi/storage/sqlite3/storage.go | 5 ++ userapi/storage/tables/interface.go | 10 ++- 5 files changed, 123 insertions(+), 17 deletions(-) diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go index 57270fdd5..41d9d0c47 100644 --- a/userapi/storage/interface.go +++ b/userapi/storage/interface.go @@ -89,6 +89,14 @@ type Database interface { // GetLoginTokenDataByToken returns the data associated with the given token. // May return sql.ErrNoRows. GetLoginTokenDataByToken(ctx context.Context, token string) (*api.LoginTokenData, error) + + AllUsers(ctx context.Context) (result int64, err error) + NonBridgedUsers(ctx context.Context) (result int64, err error) + RegisteredUserByType(ctx context.Context) (map[string]int64, error) + DailyUsers(ctx context.Context) (result int64, err error) + MonthlyUsers(ctx context.Context) (result int64, err error) + R30Users(ctx context.Context) (map[string]int64, error) + R30UsersV2(ctx context.Context) (map[string]int64, error) } // Err3PIDInUse is the error returned when trying to save an association involving diff --git a/userapi/storage/postgres/stats_table.go b/userapi/storage/postgres/stats_table.go index 5bcd93663..752b22ace 100644 --- a/userapi/storage/postgres/stats_table.go +++ b/userapi/storage/postgres/stats_table.go @@ -19,6 +19,8 @@ import ( "database/sql" "time" + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib" @@ -101,8 +103,25 @@ FROM GROUP BY client_type ` +const countUserByAccountTypeSQL = ` +SELECT COUNT(*) FROM account_accounts WHERE account_type IN $1 +` + +const countRegisteredUserByTypeStmt = ` +SELECT user_type, COUNT(*) AS count FROM ( + SELECT + CASE + WHEN account_type<>2 AND appservice_id IS NULL THEN 'native' + WHEN account_type=2 AND appservice_id IS NULL THEN 'guest' + WHEN account_type<>2 AND appservice_id IS NOT NULL THEN 'bridged' + END AS user_type + FROM account_accounts + WHERE created_ts > $1 +) AS t GROUP BY user_type +` + // account_type 1 = users; 3 = admins -const updateUserDailyVists = ` +const updateUserDailyVisitsSQL = ` INSERT INTO user_daily_visits(localpart, device_id, timestamp, user_agent) SELECT u.localpart, u.device_id, $1, MAX(u.user_agent) FROM device_devices AS u @@ -121,12 +140,14 @@ ON CONFLICT (localpart, device_id, timestamp) DO NOTHING ` type statsStatements struct { - serverName gomatrixserverlib.ServerName - lastUpdate time.Time - countUsersLastSeenAfterStmt *sql.Stmt - countR30UsersStmt *sql.Stmt - countR30UsersV2Stmt *sql.Stmt - updateUserDailyVisits *sql.Stmt + serverName gomatrixserverlib.ServerName + lastUpdate time.Time + countUsersLastSeenAfterStmt *sql.Stmt + countR30UsersStmt *sql.Stmt + countR30UsersV2Stmt *sql.Stmt + updateUserDailyVisitsStmt *sql.Stmt + countUserByAccountTypeStmt *sql.Stmt + countRegisteredUserByTypeStmt *sql.Stmt } func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) { @@ -139,11 +160,14 @@ func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) if err != nil { return nil, err } + go s.startTimers() return s, sqlutil.StatementList{ {&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL}, {&s.countR30UsersStmt, countR30UsersSQL}, {&s.countR30UsersV2Stmt, countR30UsersV2SQL}, - {&s.updateUserDailyVisits, updateUserDailyVists}, + {&s.updateUserDailyVisitsStmt, updateUserDailyVisitsSQL}, + {&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL}, + {&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeStmt}, }.Prepare(db) } @@ -151,7 +175,7 @@ func (s *statsStatements) startTimers() { // initial run time.AfterFunc(time.Minute*5, func() { logrus.Infof("Executing UpdateUserDailyVisits") - if err := s.UpdateUserDailyVisits(context.Background(), nil); err != nil { + if err := s.updateUserDailyVisits(context.Background(), nil); err != nil { logrus.WithError(err).Error("failed to update daily user visits") } }) @@ -161,13 +185,53 @@ func (s *statsStatements) startTimers() { select { case <-ticker.C: logrus.Infof("Executing UpdateUserDailyVisits") - if err := s.UpdateUserDailyVisits(context.Background(), nil); err != nil { + if err := s.updateUserDailyVisits(context.Background(), nil); err != nil { logrus.WithError(err).Error("failed to update daily user visits") } } } } +func (s *statsStatements) AllUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) { + stmt := sqlutil.TxStmt(txn, s.countUserByAccountTypeStmt) + err = stmt.QueryRowContext(ctx, + pq.Int64Array{1, 2, 3, 4}, + ).Scan(&result) + return +} + +func (s *statsStatements) NonBridgedUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) { + stmt := sqlutil.TxStmt(txn, s.countUserByAccountTypeStmt) + err = stmt.QueryRowContext(ctx, + pq.Int64Array{1, 2, 3}, + ).Scan(&result) + return +} + +func (s *statsStatements) RegisteredUserByType(ctx context.Context, txn *sql.Tx) (map[string]int64, error) { + stmt := sqlutil.TxStmt(txn, s.countRegisteredUserByTypeStmt) + registeredAfter := time.Now().AddDate(0, 0, -1) + + rows, err := stmt.QueryContext(ctx, + gomatrixserverlib.AsTimestamp(registeredAfter), + ) + if err != nil { + return nil, err + } + + var userType string + var count int64 + var result = make(map[string]int64) + for rows.Next() { + if err = rows.Scan(&userType, &count); err != nil { + return nil, err + } + result[userType] = count + } + + return result, rows.Err() +} + func (s *statsStatements) DailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) { stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt) lastSeenAfter := time.Now().AddDate(0, 0, -1) @@ -208,7 +272,7 @@ func (s *statsStatements) R30Users(ctx context.Context, txn *sql.Tx) (map[string var count int64 var result = make(map[string]int64) for rows.Next() { - if err := rows.Scan(&platform, &count); err != nil { + if err = rows.Scan(&platform, &count); err != nil { return nil, err } result["all"] += count @@ -250,7 +314,7 @@ func (s *statsStatements) R30UsersV2(ctx context.Context, txn *sql.Tx) (map[stri "all": 0, } for rows.Next() { - if err := rows.Scan(&platform, &count); err != nil { + if err = rows.Scan(&platform, &count); err != nil { return nil, err } result["all"] += count @@ -263,11 +327,10 @@ func (s *statsStatements) R30UsersV2(ctx context.Context, txn *sql.Tx) (map[stri return result, rows.Err() } -func (s *statsStatements) UpdateUserDailyVisits(ctx context.Context, txn *sql.Tx) error { - stmt := sqlutil.TxStmt(txn, s.updateUserDailyVisits) +func (s *statsStatements) updateUserDailyVisits(ctx context.Context, txn *sql.Tx) error { + stmt := sqlutil.TxStmt(txn, s.updateUserDailyVisitsStmt) _ = stmt todayStart := time.Now().Truncate(time.Hour * 24) - lastUpdate := s.lastUpdate // edge case if todayStart.After(s.lastUpdate) { @@ -275,7 +338,7 @@ func (s *statsStatements) UpdateUserDailyVisits(ctx context.Context, txn *sql.Tx } _, err := stmt.ExecContext(ctx, gomatrixserverlib.AsTimestamp(todayStart), - gomatrixserverlib.AsTimestamp(lastUpdate), + gomatrixserverlib.AsTimestamp(s.lastUpdate), gomatrixserverlib.AsTimestamp(time.Now()), ) if err == nil { diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 858009469..d483d5bd9 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -671,3 +671,25 @@ func (d *Database) RemoveLoginToken(ctx context.Context, token string) error { func (d *Database) GetLoginTokenDataByToken(ctx context.Context, token string) (*api.LoginTokenData, error) { return d.LoginTokens.SelectLoginToken(ctx, token) } + +func (d *Database) AllUsers(ctx context.Context) (result int64, err error) { + return d.Stats.AllUsers(ctx, nil) +} +func (d *Database) NonBridgedUsers(ctx context.Context) (result int64, err error) { + return d.Stats.NonBridgedUsers(ctx, nil) +} +func (d *Database) RegisteredUserByType(ctx context.Context) (map[string]int64, error) { + return d.Stats.RegisteredUserByType(ctx, nil) +} +func (d *Database) DailyUsers(ctx context.Context) (result int64, err error) { + return d.Stats.DailyUsers(ctx, nil) +} +func (d *Database) MonthlyUsers(ctx context.Context) (result int64, err error) { + return d.Stats.MonthlyUsers(ctx, nil) +} +func (d *Database) R30Users(ctx context.Context) (map[string]int64, error) { + return d.Stats.R30Users(ctx, nil) +} +func (d *Database) R30UsersV2(ctx context.Context) (map[string]int64, error) { + return d.Stats.R30UsersV2(ctx, nil) +} \ No newline at end of file diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go index 98c244977..0d77e45ed 100644 --- a/userapi/storage/sqlite3/storage.go +++ b/userapi/storage/sqlite3/storage.go @@ -86,6 +86,10 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err != nil { return nil, fmt.Errorf("NewSQLiteThreePIDTable: %w", err) } + statsTable, err := NewSQLiteStatsTable(db, serverName) + if err != nil { + return nil, fmt.Errorf("NewSQLiteStatsTable: %w", err) + } return &shared.Database{ AccountDatas: accountDataTable, Accounts: accountsTable, @@ -96,6 +100,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver OpenIDTokens: openIDTable, Profiles: profilesTable, ThreePIDs: threePIDTable, + Stats: statsTable, ServerName: serverName, DB: db, Writer: sqlutil.NewExclusiveWriter(), diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go index 6311711f4..7fdb08bc5 100644 --- a/userapi/storage/tables/interface.go +++ b/userapi/storage/tables/interface.go @@ -94,4 +94,12 @@ type ThreePIDTable interface { DeleteThreePID(ctx context.Context, txn *sql.Tx, threepid string, medium string) (err error) } -type StatsTable interface{} \ No newline at end of file +type StatsTable interface { + AllUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) + NonBridgedUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) + RegisteredUserByType(ctx context.Context, txn *sql.Tx) (map[string]int64, error) + DailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) + MonthlyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) + R30Users(ctx context.Context, txn *sql.Tx) (map[string]int64, error) + R30UsersV2(ctx context.Context, txn *sql.Tx) (map[string]int64, error) +} \ No newline at end of file