From 4ed2c1d458deadd463f2763548cc3ebc3eef82a7 Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Mon, 11 Apr 2022 13:50:11 +0200 Subject: [PATCH] Use new testing structure Fix issues with getting values when using SQLite Fix wrong AddDate value Export UpdateUserDailyVisits --- userapi/storage/postgres/stats_table.go | 20 +- userapi/storage/sqlite3/stats_table.go | 45 +-- userapi/storage/sqlite3/stats_table_test.go | 283 ----------------- userapi/storage/tables/interface.go | 2 + userapi/storage/tables/stats_table_test.go | 319 ++++++++++++++++++++ 5 files changed, 355 insertions(+), 314 deletions(-) delete mode 100644 userapi/storage/sqlite3/stats_table_test.go create mode 100644 userapi/storage/tables/stats_table_test.go diff --git a/userapi/storage/postgres/stats_table.go b/userapi/storage/postgres/stats_table.go index a75ca2ea7..ca31407e6 100644 --- a/userapi/storage/postgres/stats_table.go +++ b/userapi/storage/postgres/stats_table.go @@ -194,7 +194,7 @@ func (s *statsStatements) startTimers() { var updateStatsFunc func() updateStatsFunc = func() { logrus.Infof("Executing UpdateUserDailyVisits") - if err := s.updateUserDailyVisits(context.Background(), nil); err != nil { + if err := s.UpdateUserDailyVisits(context.Background(), nil, time.Now(), s.lastUpdate); err != nil { logrus.WithError(err).Error("failed to update daily user visits") } time.AfterFunc(time.Hour*3, updateStatsFunc) @@ -229,7 +229,7 @@ func (s *statsStatements) nonBridgedUsers(ctx context.Context, txn *sql.Tx) (res func (s *statsStatements) registeredUserByType(ctx context.Context, txn *sql.Tx) (map[string]int64, error) { stmt := sqlutil.TxStmt(txn, s.countRegisteredUserByTypeStmt) - registeredAfter := time.Now().AddDate(0, 0, -1) + registeredAfter := time.Now().AddDate(0, 0, -30) rows, err := stmt.QueryContext(ctx, pq.Int64Array{ @@ -410,18 +410,20 @@ func (s *statsStatements) UserStatistics(ctx context.Context, txn *sql.Tx) (*typ return stats, dbEngine, err } -func (s *statsStatements) updateUserDailyVisits(ctx context.Context, txn *sql.Tx) error { +func (s *statsStatements) UpdateUserDailyVisits( + ctx context.Context, txn *sql.Tx, + startTime, lastUpdate time.Time, +) error { stmt := sqlutil.TxStmt(txn, s.updateUserDailyVisitsStmt) - _ = stmt - todayStart := time.Now().Truncate(time.Hour * 24) + startTime = startTime.Truncate(time.Hour * 24) // edge case - if todayStart.After(s.lastUpdate) { - todayStart = todayStart.AddDate(0, 0, -1) + if startTime.After(s.lastUpdate) { + startTime = startTime.AddDate(0, 0, -1) } _, err := stmt.ExecContext(ctx, - gomatrixserverlib.AsTimestamp(todayStart), - gomatrixserverlib.AsTimestamp(s.lastUpdate), + gomatrixserverlib.AsTimestamp(startTime), + gomatrixserverlib.AsTimestamp(lastUpdate), gomatrixserverlib.AsTimestamp(time.Now()), ) if err == nil { diff --git a/userapi/storage/sqlite3/stats_table.go b/userapi/storage/sqlite3/stats_table.go index 4ec847198..9298d1150 100644 --- a/userapi/storage/sqlite3/stats_table.go +++ b/userapi/storage/sqlite3/stats_table.go @@ -122,16 +122,16 @@ SELECT COUNT(*) FROM account_accounts WHERE account_type IN ($1) // $1 = Guest AccountType // $3 & $4 = All non guest AccountType IDs -const countRegisteredUserByTypeStmt = ` +const countRegisteredUserByTypeSQL = ` SELECT user_type, COUNT(*) AS count FROM ( SELECT CASE - WHEN account_type IN ($3) AND appservice_id IS NULL THEN 'native' - WHEN account_type = $1 AND appservice_id IS NULL THEN 'guest' - WHEN account_type IN ($4) AND appservice_id IS NOT NULL THEN 'bridged' + WHEN account_type IN ($1) AND appservice_id IS NULL THEN 'native' + WHEN account_type = $4 AND appservice_id IS NULL THEN 'guest' + WHEN account_type IN ($5) AND appservice_id IS NOT NULL THEN 'bridged' END AS user_type FROM account_accounts - WHERE created_ts > $2 + WHERE created_ts > $8 ) AS t GROUP BY user_type ` @@ -187,7 +187,7 @@ func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (t {&s.countR30UsersV2Stmt, countR30UsersV2SQL}, {&s.updateUserDailyVisitsStmt, updateUserDailyVisitsSQL}, {&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL}, - {&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeStmt}, + {&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeSQL}, {&s.dbEngineVersionStmt, queryDBEngineVersion}, }.Prepare(db) } @@ -196,7 +196,7 @@ func (s *statsStatements) startTimers() { var updateStatsFunc func() updateStatsFunc = func() { logrus.Infof("Executing UpdateUserDailyVisits") - if err := s.updateUserDailyVisits(context.Background(), nil); err != nil { + if err := s.UpdateUserDailyVisits(context.Background(), nil, time.Now(), s.lastUpdate); err != nil { logrus.WithError(err).Error("failed to update daily user visits") } time.AfterFunc(time.Hour*3, updateStatsFunc) @@ -234,24 +234,23 @@ func (s *statsStatements) registeredUserByType(ctx context.Context, txn *sql.Tx) // $1 = Guest AccountType; $2 = timestamp // $3 & $4 = All non guest AccountType IDs nonGuests := []api.AccountType{api.AccountTypeUser, api.AccountTypeAdmin, api.AccountTypeAppService} - countSQL := strings.Replace(countRegisteredUserByTypeStmt, "($3)", sqlutil.QueryVariadicOffset(len(nonGuests), 2), 1) - countSQL = strings.Replace(countSQL, "($4)", sqlutil.QueryVariadicOffset(len(nonGuests), 2+len(nonGuests)), 1) - + countSQL := strings.Replace(countRegisteredUserByTypeSQL, "($1)", sqlutil.QueryVariadicOffset(len(nonGuests), 0), 1) + countSQL = strings.Replace(countSQL, "($5)", sqlutil.QueryVariadicOffset(len(nonGuests), 1+len(nonGuests)), 1) queryStmt, err := s.db.Prepare(countSQL) if err != nil { return nil, err } stmt := sqlutil.TxStmt(txn, queryStmt) - registeredAfter := time.Now().AddDate(0, 0, -1) + registeredAfter := time.Now().AddDate(0, 0, -30) params := make([]interface{}, len(nonGuests)*2+2) - params[0] = api.AccountTypeGuest // $1 - params[1] = gomatrixserverlib.AsTimestamp(registeredAfter) // $2 // nonGuests is used twice for i, v := range nonGuests { - params[i+2] = v // i: 2 3 4 => ($3, $4, $5) - params[i+2+len(nonGuests)] = v // i: 5 6 7 => ($6, $7, $8) + params[i] = v // i: 0 1 2 => ($1, $2, $3) + params[i+2] = v // i: 3 4 5 => ($5, $6, $7) } + params[3] = api.AccountTypeGuest // $4 + params[7] = gomatrixserverlib.AsTimestamp(registeredAfter) // $8 rows, err := stmt.QueryContext(ctx, params...) if err != nil { @@ -422,18 +421,20 @@ func (s *statsStatements) UserStatistics(ctx context.Context, txn *sql.Tx) (*typ return stats, dbEngine, err } -func (s *statsStatements) updateUserDailyVisits(ctx context.Context, txn *sql.Tx) error { +func (s *statsStatements) UpdateUserDailyVisits( + ctx context.Context, txn *sql.Tx, + startTime, lastUpdate time.Time, +) error { stmt := sqlutil.TxStmt(txn, s.updateUserDailyVisitsStmt) - _ = stmt - todayStart := time.Now().Truncate(time.Hour * 24) + startTime = startTime.Truncate(time.Hour * 24) // edge case - if todayStart.After(s.lastUpdate) { - todayStart = todayStart.AddDate(0, 0, -1) + if startTime.After(s.lastUpdate) { + startTime = startTime.AddDate(0, 0, -1) } _, err := stmt.ExecContext(ctx, - gomatrixserverlib.AsTimestamp(todayStart), - gomatrixserverlib.AsTimestamp(s.lastUpdate), + gomatrixserverlib.AsTimestamp(startTime), + gomatrixserverlib.AsTimestamp(lastUpdate), gomatrixserverlib.AsTimestamp(time.Now()), ) if err == nil { diff --git a/userapi/storage/sqlite3/stats_table_test.go b/userapi/storage/sqlite3/stats_table_test.go deleted file mode 100644 index ac4fdb949..000000000 --- a/userapi/storage/sqlite3/stats_table_test.go +++ /dev/null @@ -1,283 +0,0 @@ -package sqlite3 - -import ( - "context" - "database/sql" - "reflect" - "testing" - "time" - - "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/dendrite/userapi/storage/tables" - "github.com/matrix-org/dendrite/userapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -func mustMakeDBs(t *testing.T) (*sql.DB, tables.AccountsTable, tables.DevicesTable, tables.StatsTable) { - t.Helper() - - db, err := sql.Open("sqlite3", ":memory:") - if err != nil { - t.Fatalf("unable to open in-memory database: %v", err) - } - - accDB, err := NewSQLiteAccountsTable(db, "localhost") - if err != nil { - t.Fatalf("unable to create acc db: %v", err) - } - devDB, err := NewSQLiteDevicesTable(db, "localhost") - if err != nil { - t.Fatalf("unable to open device db: %v", err) - } - statsDB, err := NewSQLiteStatsTable(db, "localhost") - if err != nil { - t.Fatalf("unable to open stats db: %v", err) - } - return db, accDB, devDB, statsDB -} - -func mustMakeAccountAndDevice( - t *testing.T, - ctx context.Context, - accDB tables.AccountsTable, - devDB tables.DevicesTable, - localpart string, - accType api.AccountType, - userAgent string, -) { - t.Helper() - - appServiceID := "" - if accType == api.AccountTypeAppService { - appServiceID = util.RandomString(16) - } - - _, err := accDB.InsertAccount(ctx, nil, localpart, "", appServiceID, accType) - if err != nil { - t.Fatalf("unable to create account: %v", err) - } - _, err = devDB.InsertDevice(ctx, nil, "deviceID", localpart, util.RandomString(16), nil, "", userAgent) - if err != nil { - t.Fatalf("unable to create device: %v", err) - } -} - -func mustUpdateDeviceLastSeen( - t *testing.T, - ctx context.Context, - db *sql.DB, - localpart string, - timestamp time.Time, -) { - _, err := db.ExecContext(ctx, "UPDATE device_devices SET last_seen_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart) - if err != nil { - t.Fatalf("unable to update device last seen") - } -} - -func mustUserUpdateRegistered( - t *testing.T, - ctx context.Context, - db *sql.DB, - localpart string, - timestamp time.Time, -) { - _, err := db.ExecContext(ctx, "UPDATE account_accounts SET created_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart) - if err != nil { - t.Fatalf("unable to update device last seen") - } -} - -func mustUpdateUserDailyVisits( - t *testing.T, - ctx context.Context, - db *sql.DB, - startTime time.Time, -) { - _, err := db.ExecContext(ctx, updateUserDailyVisitsSQL, - gomatrixserverlib.AsTimestamp(startTime.Truncate(time.Hour*24)), - gomatrixserverlib.AsTimestamp(startTime.Truncate(time.Hour*24).Add(time.Hour)), - gomatrixserverlib.AsTimestamp(time.Now()), - ) - if err != nil { - t.Fatalf("unable to update device last seen") - } -} - -// These tests must run sequentially, as they build up on each other -func Test_statsStatements_UserStatistics(t *testing.T) { - - ctx := context.Background() - db, accDB, devDB, statsDB := mustMakeDBs(t) - - t.Run("want SQLite engine", func(t *testing.T) { - _, gotDB, err := statsDB.UserStatistics(ctx, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if "SQLite" != gotDB.Engine { // can't use DeepEqual, as the Version might differ - t.Errorf("UserStatistics() gotDB = %+v, want SQLite", gotDB.Engine) - } - }) - - t.Run("Want Users", func(t *testing.T) { - mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user1", api.AccountTypeUser, "Element Android") - mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user2", api.AccountTypeUser, "Element iOS") - mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user3", api.AccountTypeUser, "Element web") - mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user4", api.AccountTypeGuest, "Element electron") - mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user5", api.AccountTypeAdmin, "gecko") - mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user6", api.AccountTypeAppService, "gecko") - gotStats, _, err := statsDB.UserStatistics(ctx, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wantStats := &types.UserStatistics{ - RegisteredUsersByType: map[string]int64{ - "native": 4, - "guest": 1, - "bridged": 1, - }, - R30Users: map[string]int64{}, - R30UsersV2: map[string]int64{ - "ios": 0, - "android": 0, - "web": 0, - "electron": 0, - "all": 0, - }, - AllUsers: 6, - NonBridgedUsers: 5, - DailyUsers: 6, - MonthlyUsers: 6, - } - if !reflect.DeepEqual(gotStats, wantStats) { - t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats) - } - }) - - t.Run("Users not active for one/two month", func(t *testing.T) { - mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now().AddDate(0, -2, 0)) - mustUpdateDeviceLastSeen(t, ctx, db, "user2", time.Now().AddDate(0, -1, 0)) - gotStats, _, err := statsDB.UserStatistics(ctx, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wantStats := &types.UserStatistics{ - RegisteredUsersByType: map[string]int64{ - "native": 4, - "guest": 1, - "bridged": 1, - }, - R30Users: map[string]int64{}, - R30UsersV2: map[string]int64{ - "ios": 0, - "android": 0, - "web": 0, - "electron": 0, - "all": 0, - }, - AllUsers: 6, - NonBridgedUsers: 5, - DailyUsers: 4, - MonthlyUsers: 4, - } - if !reflect.DeepEqual(gotStats, wantStats) { - t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats) - } - }) - - /* R30Users counts the number of 30 day retained users, defined as: - - Users who have created their accounts more than 30 days ago - - Where last seen at most 30 days ago - - Where account creation and last_seen are > 30 days apart - */ - t.Run("R30Users tests", func(t *testing.T) { - mustUserUpdateRegistered(t, ctx, db, "user1", time.Now().AddDate(0, -2, 0)) - mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now()) - mustUserUpdateRegistered(t, ctx, db, "user4", time.Now().AddDate(0, -2, 0)) - mustUpdateDeviceLastSeen(t, ctx, db, "user4", time.Now()) - mustUpdateUserDailyVisits(t, ctx, db, time.Now().AddDate(0, -1, 0)) - gotStats, _, err := statsDB.UserStatistics(ctx, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wantStats := &types.UserStatistics{ - RegisteredUsersByType: map[string]int64{ - "native": 4, - "guest": 1, - "bridged": 1, - }, - R30Users: map[string]int64{ - "all": 2, - "android": 1, - "electron": 1, - }, - R30UsersV2: map[string]int64{ - "ios": 0, - "android": 0, - "web": 0, - "electron": 0, - "all": 0, - }, - AllUsers: 6, - NonBridgedUsers: 5, - DailyUsers: 5, - MonthlyUsers: 5, - } - if !reflect.DeepEqual(gotStats, wantStats) { - t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats) - } - }) - - /* - R30UsersV2 counts the number of 30 day retained users, defined as users that: - - Appear more than once in the past 60 days - - Have more than 30 days between the most and least recent appearances that occurred in the past 60 days. - most recent -> neueste - least recent -> älteste - - */ - t.Run("R30UsersV2 tests", func(t *testing.T) { - // generate some data - for i := 100; i > 0; i-- { - mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now().AddDate(0, 0, -i)) - mustUpdateDeviceLastSeen(t, ctx, db, "user5", time.Now().AddDate(0, 0, -i)) - mustUpdateUserDailyVisits(t, ctx, db, time.Now().AddDate(0, 0, -i)) - } - gotStats, _, err := statsDB.UserStatistics(ctx, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wantStats := &types.UserStatistics{ - RegisteredUsersByType: map[string]int64{ - "native": 4, - "guest": 1, - "bridged": 1, - }, - R30Users: map[string]int64{ - "all": 2, - "android": 1, - "electron": 1, - }, - R30UsersV2: map[string]int64{ - "ios": 0, - "android": 1, - "web": 1, - "electron": 0, - "all": 2, - }, - AllUsers: 6, - NonBridgedUsers: 5, - DailyUsers: 3, - MonthlyUsers: 5, - } - if !reflect.DeepEqual(gotStats, wantStats) { - t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats) - } - }) -} diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go index 92db4ebad..ecd85bd7d 100644 --- a/userapi/storage/tables/interface.go +++ b/userapi/storage/tables/interface.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "encoding/json" + "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/userapi/api" @@ -115,6 +116,7 @@ type NotificationTable interface { type StatsTable interface { UserStatistics(ctx context.Context, txn *sql.Tx) (*types.UserStatistics, *types.DatabaseEngine, error) + UpdateUserDailyVisits(ctx context.Context, txn *sql.Tx, startTime, lastUpdate time.Time) error } type NotificationFilter uint32 diff --git a/userapi/storage/tables/stats_table_test.go b/userapi/storage/tables/stats_table_test.go new file mode 100644 index 000000000..2b875835b --- /dev/null +++ b/userapi/storage/tables/stats_table_test.go @@ -0,0 +1,319 @@ +package tables_test + +import ( + "context" + "database/sql" + "fmt" + "reflect" + "testing" + "time" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/postgres" + "github.com/matrix-org/dendrite/userapi/storage/sqlite3" + "github.com/matrix-org/dendrite/userapi/storage/tables" + "github.com/matrix-org/dendrite/userapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +func mustMakeDBs(t *testing.T, dbType test.DBType) ( + *sql.DB, tables.AccountsTable, tables.DevicesTable, tables.StatsTable, func(), +) { + t.Helper() + + var ( + accTable tables.AccountsTable + devTable tables.DevicesTable + statsTable tables.StatsTable + err error + ) + + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }) + if err != nil { + t.Fatalf("failed to open db: %s", err) + } + + switch dbType { + case test.DBTypeSQLite: + accTable, err = sqlite3.NewSQLiteAccountsTable(db, "localhost") + if err != nil { + t.Fatalf("unable to create acc db: %v", err) + } + devTable, err = sqlite3.NewSQLiteDevicesTable(db, "localhost") + if err != nil { + t.Fatalf("unable to open device db: %v", err) + } + statsTable, err = sqlite3.NewSQLiteStatsTable(db, "localhost") + if err != nil { + t.Fatalf("unable to open stats db: %v", err) + } + case test.DBTypePostgres: + accTable, err = postgres.NewPostgresAccountsTable(db, "localhost") + if err != nil { + t.Fatalf("unable to create acc db: %v", err) + } + devTable, err = postgres.NewPostgresDevicesTable(db, "localhost") + if err != nil { + t.Fatalf("unable to open device db: %v", err) + } + statsTable, err = postgres.NewPostgresStatsTable(db, "localhost") + if err != nil { + t.Fatalf("unable to open stats db: %v", err) + } + } + + return db, accTable, devTable, statsTable, close +} + +func mustMakeAccountAndDevice( + t *testing.T, + ctx context.Context, + accDB tables.AccountsTable, + devDB tables.DevicesTable, + localpart string, + accType api.AccountType, + userAgent string, +) { + t.Helper() + + appServiceID := "" + if accType == api.AccountTypeAppService { + appServiceID = util.RandomString(16) + } + + _, err := accDB.InsertAccount(ctx, nil, localpart, "", appServiceID, accType) + if err != nil { + t.Fatalf("unable to create account: %v", err) + } + _, err = devDB.InsertDevice(ctx, nil, "deviceID", localpart, util.RandomString(16), nil, "", userAgent) + if err != nil { + t.Fatalf("unable to create device: %v", err) + } +} + +func mustUpdateDeviceLastSeen( + t *testing.T, + ctx context.Context, + db *sql.DB, + localpart string, + timestamp time.Time, +) { + t.Helper() + _, err := db.ExecContext(ctx, "UPDATE device_devices SET last_seen_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart) + if err != nil { + t.Fatalf("unable to update device last seen") + } +} + +func mustUserUpdateRegistered( + t *testing.T, + ctx context.Context, + db *sql.DB, + localpart string, + timestamp time.Time, +) { + _, err := db.ExecContext(ctx, "UPDATE account_accounts SET created_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart) + if err != nil { + t.Fatalf("unable to update device last seen") + } +} + +// These tests must run sequentially, as they build up on each other +func Test_UserStatistics(t *testing.T) { + + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, accDB, devDB, statsDB, close := mustMakeDBs(t, dbType) + defer close() + wantType := "SQLite" + if dbType == test.DBTypePostgres { + wantType = "Postgres" + } + + t.Run(fmt.Sprintf("want %s database engine", wantType), func(t *testing.T) { + _, gotDB, err := statsDB.UserStatistics(ctx, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if wantType != gotDB.Engine { // can't use DeepEqual, as the Version might differ + t.Errorf("UserStatistics() gotDB = %+v, want SQLite", gotDB.Engine) + } + }) + + t.Run("Want Users", func(t *testing.T) { + mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user1", api.AccountTypeUser, "Element Android") + mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user2", api.AccountTypeUser, "Element iOS") + mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user3", api.AccountTypeUser, "Element web") + mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user4", api.AccountTypeGuest, "Element Electron") + mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user5", api.AccountTypeAdmin, "gecko") + mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user6", api.AccountTypeAppService, "gecko") + gotStats, _, err := statsDB.UserStatistics(ctx, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wantStats := &types.UserStatistics{ + RegisteredUsersByType: map[string]int64{ + "native": 4, + "guest": 1, + "bridged": 1, + }, + R30Users: map[string]int64{}, + R30UsersV2: map[string]int64{ + "ios": 0, + "android": 0, + "web": 0, + "electron": 0, + "all": 0, + }, + AllUsers: 6, + NonBridgedUsers: 5, + DailyUsers: 6, + MonthlyUsers: 6, + } + if !reflect.DeepEqual(gotStats, wantStats) { + t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats) + } + }) + + t.Run("Users not active for one/two month", func(t *testing.T) { + mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now().AddDate(0, -2, 0)) + mustUpdateDeviceLastSeen(t, ctx, db, "user2", time.Now().AddDate(0, -1, 0)) + gotStats, _, err := statsDB.UserStatistics(ctx, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wantStats := &types.UserStatistics{ + RegisteredUsersByType: map[string]int64{ + "native": 4, + "guest": 1, + "bridged": 1, + }, + R30Users: map[string]int64{}, + R30UsersV2: map[string]int64{ + "ios": 0, + "android": 0, + "web": 0, + "electron": 0, + "all": 0, + }, + AllUsers: 6, + NonBridgedUsers: 5, + DailyUsers: 4, + MonthlyUsers: 4, + } + if !reflect.DeepEqual(gotStats, wantStats) { + t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats) + } + }) + + /* R30Users counts the number of 30 day retained users, defined as: + - Users who have created their accounts more than 30 days ago + - Where last seen at most 30 days ago + - Where account creation and last_seen are > 30 days apart + */ + t.Run("R30Users tests", func(t *testing.T) { + mustUserUpdateRegistered(t, ctx, db, "user1", time.Now().AddDate(0, -2, 0)) + mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now()) + mustUserUpdateRegistered(t, ctx, db, "user4", time.Now().AddDate(0, -2, 0)) + mustUpdateDeviceLastSeen(t, ctx, db, "user4", time.Now()) + startTime := time.Now().AddDate(0, 0, -2) + err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24).Add(time.Hour)) + if err != nil { + t.Fatalf("unable to update daily visits stats: %v", err) + } + + gotStats, _, err := statsDB.UserStatistics(ctx, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wantStats := &types.UserStatistics{ + RegisteredUsersByType: map[string]int64{ + "native": 3, + "bridged": 1, + }, + R30Users: map[string]int64{ + "all": 2, + "android": 1, + "electron": 1, + }, + R30UsersV2: map[string]int64{ + "ios": 0, + "android": 0, + "web": 0, + "electron": 0, + "all": 0, + }, + AllUsers: 6, + NonBridgedUsers: 5, + DailyUsers: 5, + MonthlyUsers: 5, + } + if !reflect.DeepEqual(gotStats, wantStats) { + t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats) + } + }) + + /* + R30UsersV2 counts the number of 30 day retained users, defined as users that: + - Appear more than once in the past 60 days + - Have more than 30 days between the most and least recent appearances that occurred in the past 60 days. + most recent -> neueste + least recent -> älteste + + */ + t.Run("R30UsersV2 tests", func(t *testing.T) { + // generate some data + for i := 100; i > 0; i-- { + mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now().AddDate(0, 0, -i)) + mustUpdateDeviceLastSeen(t, ctx, db, "user5", time.Now().AddDate(0, 0, -i)) + startTime := time.Now().AddDate(0, 0, -i) + err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24).Add(time.Hour)) + if err != nil { + t.Fatalf("unable to update daily visits stats: %v", err) + } + } + gotStats, _, err := statsDB.UserStatistics(ctx, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wantStats := &types.UserStatistics{ + RegisteredUsersByType: map[string]int64{ + "native": 3, + "bridged": 1, + }, + R30Users: map[string]int64{ + "all": 2, + "android": 1, + "electron": 1, + }, + R30UsersV2: map[string]int64{ + "ios": 0, + "android": 1, + "web": 1, + "electron": 0, + "all": 2, + }, + AllUsers: 6, + NonBridgedUsers: 5, + DailyUsers: 3, + MonthlyUsers: 5, + } + if !reflect.DeepEqual(gotStats, wantStats) { + t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats) + } + }) + }) + +}