diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index ca35951a0..58b2e5155 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -102,6 +102,7 @@ func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) UserID: device.UserID, DeviceID: device.ID, RemoteAddr: remoteAddr, + UserAgent: req.UserAgent(), } lsres := &userapi.PerformLastSeenUpdateResponse{} go rp.userAPI.PerformLastSeenUpdate(req.Context(), lsreq, lsres) // nolint:errcheck diff --git a/userapi/api/api.go b/userapi/api/api.go index 2be662e55..cd872d3c9 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -274,6 +274,7 @@ type PerformLastSeenUpdateRequest struct { UserID string DeviceID string RemoteAddr string + UserAgent string } // PerformLastSeenUpdateResponse is the response for PerformLastSeenUpdate. diff --git a/userapi/internal/api.go b/userapi/internal/api.go index f54cc6137..d818c3fb6 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -194,7 +194,7 @@ func (a *UserInternalAPI) PerformLastSeenUpdate( if err != nil { return fmt.Errorf("gomatrixserverlib.SplitID: %w", err) } - if err := a.DB.UpdateDeviceLastSeen(ctx, localpart, req.DeviceID, req.RemoteAddr); err != nil { + if err := a.DB.UpdateDeviceLastSeen(ctx, localpart, req.DeviceID, req.RemoteAddr, req.UserAgent); err != nil { return fmt.Errorf("a.DeviceDB.UpdateDeviceLastSeen: %w", err) } return nil diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go index a131dac47..57270fdd5 100644 --- a/userapi/storage/interface.go +++ b/userapi/storage/interface.go @@ -73,7 +73,7 @@ type Database interface { // Returns the device on success. CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string) (dev *api.Device, returnErr error) UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error - UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error + UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr, userAgent string) error RemoveDevice(ctx context.Context, deviceID, localpart string) error RemoveDevices(ctx context.Context, localpart string, devices []string) error // RemoveAllDevices deleted all devices for this user. Returns the devices deleted. diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go index 7bc5dc69b..9a2406977 100644 --- a/userapi/storage/postgres/devices_table.go +++ b/userapi/storage/postgres/devices_table.go @@ -96,7 +96,7 @@ const selectDevicesByIDSQL = "" + "SELECT device_id, localpart, display_name FROM device_devices WHERE device_id = ANY($1)" const updateDeviceLastSeen = "" + - "UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4" + "UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5" type devicesStatements struct { insertDeviceStmt *sql.Stmt @@ -293,9 +293,9 @@ func (s *devicesStatements) SelectDevicesByLocalpart( return devices, rows.Err() } -func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error { +func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error { lastSeenTs := time.Now().UnixNano() / 1000000 stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt) - _, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID) + _, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, userAgent, localpart, deviceID) return err } diff --git a/userapi/storage/postgres/stats.go b/userapi/storage/postgres/stats.go deleted file mode 100644 index d9f20d60b..000000000 --- a/userapi/storage/postgres/stats.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2022 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - "time" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/syncapi/storage/tables" - "github.com/matrix-org/gomatrixserverlib" -) - -const countUsersLastSeenAfterSQL = ""+ - "SELECT COUNT(*) FROM ("+ - " SELECT user_id FROM device_devices WHERE last_seen > $1 "+ - " GROUP BY user_id"+ - " )" - -const countActiveRoomsSQL = ""+ - "SELECT COUNT(DISTINCT room_id) FROM syncapi_output_room_events"+ - " WHERE type = $1 AND id > $2" - -type statsStatements struct { - serverName string - countUsersLastSeenAfterStmt *sql.Stmt - countActiveRoomsStmt *sql.Stmt -} - -func PrepareStats(db *sql.DB, serverName string) (tables.Stats, error) { - s := &statsStatements{ - serverName: serverName, - } - return s, sqlutil.StatementList{ - {&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL}, - {&s.countActiveRoomsStmt, countActiveRoomsSQL}, - }.Prepare(db) -} - -func (s *statsStatements) DailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) { - stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt) - lastSeenAfter := time.Now().AddDate(0, 0, 1) - err = stmt.QueryRowContext(ctx, - gomatrixserverlib.AsTimestamp(lastSeenAfter), - ).Scan(&result) - return -} - -func (s *statsStatements) MonthlyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) { - stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt) - lastSeenAfter := time.Now().AddDate(0, 0, 30) - err = stmt.QueryRowContext(ctx, - gomatrixserverlib.AsTimestamp(lastSeenAfter), - ).Scan(&result) - return -} \ No newline at end of file diff --git a/userapi/storage/postgres/stats_table.go b/userapi/storage/postgres/stats_table.go new file mode 100644 index 000000000..2ca365c62 --- /dev/null +++ b/userapi/storage/postgres/stats_table.go @@ -0,0 +1,199 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "time" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/storage/tables" + "github.com/matrix-org/gomatrixserverlib" +) + +const userDailyVisitsSchema = ` +CREATE TABLE IF NOT EXISTS user_daily_visits ( + localpart TEXT NOT NULL, + device_id TEXT NOT NULL, + timestamp BIGINT NOT NULL, + user_agent TEXT +); + +-- Device IDs and timestamp must be unique for a given user per day +CREATE UNIQUE INDEX IF NOT EXISTS localpart_device_timestamp_idx ON user_daily_visits(localpart, device_id, timestamp); +CREATE INDEX IF NOT EXISTS timestamp_idx ON user_daily_visits(timestamp); +CREATE INDEX IF NOT EXISTS localpart_timestamp_idx ON user_daily_visits(localpart, timestamp); +` + +const countUsersLastSeenAfterSQL = "" + + "SELECT COUNT(*) FROM (" + + " SELECT user_id FROM device_devices WHERE last_seen > $1 " + + " GROUP BY user_id" + + " )" + +const countR30UsersSQL = ` +SELECT platform, COUNT(*) FROM ( + SELECT users.localpart, platform, users.created_ts, MAX(uip.last_seen_ts) + FROM account_accounts users + INNER JOIN + (SELECT + localpart, last_seen_ts, + CASE + WHEN user_agent LIKE '%%Android%%' OR display_name LIKE '%%android%%' THEN 'android' + WHEN user_agent LIKE '%%iOS%%' THEN 'ios' + WHEN user_agent LIKE '%%Electron%%' THEN 'electron' + WHEN user_agent LIKE '%%Mozilla%%' THEN 'web' + WHEN user_agent LIKE '%%Gecko%%' THEN 'web' + ELSE 'unknown' + END + AS platform + FROM device_devices + ) uip + ON users.localpart = uip.localpart + AND users.account_type <> 4 + AND users.created_ts < $1 + AND uip.last_seen_ts > $1, + AND (uip.last_seen_ts) - users.created_ts > $2 + GROUP BY users.localpart, platform, users.created_ts + ) u GROUP BY PLATFORM +` + +const countR30UsersV2SQL = ` + +` + +const insertUserDailyVists = ` + INSERT INTO user_daily_visits(localpart, device_id, timestamp, user_agent) VALUES ($1, $2, $3, $4) + ON CONFLICT ON CONSTRAINT localpart_device_timestamp_idx DO NOTHING +` + +type statsStatements struct { + serverName gomatrixserverlib.ServerName + countUsersLastSeenAfterStmt *sql.Stmt + countR30UsersStmt *sql.Stmt + countR30UsersV2Stmt *sql.Stmt + insertUserDailyVisits *sql.Stmt +} + +func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) { + s := &statsStatements{ + serverName: serverName, + } + _, err := db.Exec(userDailyVisitsSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL}, + {&s.countR30UsersStmt, countR30UsersSQL}, + {&s.countR30UsersV2Stmt, countR30UsersV2SQL}, + {&s.insertUserDailyVisits, insertUserDailyVists}, + }.Prepare(db) +} + +func (s *statsStatements) DailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) { + stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt) + lastSeenAfter := time.Now().AddDate(0, 0, -1) + err = stmt.QueryRowContext(ctx, + gomatrixserverlib.AsTimestamp(lastSeenAfter), + ).Scan(&result) + return +} + +func (s *statsStatements) MonthlyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) { + stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt) + lastSeenAfter := time.Now().AddDate(0, 0, -30) + err = stmt.QueryRowContext(ctx, + gomatrixserverlib.AsTimestamp(lastSeenAfter), + ).Scan(&result) + return +} + +/* R30Users counts the number of 30 day retained users, defined as: +- Users who have created their accounts more than 30 days ago +- Where last seen at most 30 days ago +- Where account creation and last_seen are > 30 days apart +*/ +func (s *statsStatements) R30Users(ctx context.Context, txn *sql.Tx) (map[string]int64, error) { + stmt := sqlutil.TxStmt(txn, s.countR30UsersStmt) + lastSeenAfter := time.Now().AddDate(0, 0, -30) + diff := time.Hour * 24 * 30 + + rows, err := stmt.QueryContext(ctx, + gomatrixserverlib.AsTimestamp(lastSeenAfter), + diff.Milliseconds(), + ) + if err != nil { + return nil, err + } + + var platform string + var count int64 + var result = make(map[string]int64) + for rows.Next() { + if err := rows.Scan(&platform, &count); err != nil { + return nil, err + } + result["all"] += count + if platform == "unknown" { + continue + } + result[platform] = count + } + + return map[string]int64{}, rows.Err() +} + +/* R30UsersV2 counts the number of 30 day retained users, defined as users that: +- Appear more than once in the past 60 days +- Have more than 30 days between the most and least recent appearances that occurred in the past 60 days. +*/ +func (s *statsStatements) R30UsersV2(ctx context.Context, txn *sql.Tx) (map[string]int64, error) { + stmt := sqlutil.TxStmt(txn, s.countR30UsersV2Stmt) + lastSeenAfter := time.Now().AddDate(0, 0, -30) + diff := time.Hour * 24 * 30 + diff.Milliseconds() + + rows, err := stmt.QueryContext(ctx, + gomatrixserverlib.AsTimestamp(lastSeenAfter), + diff.Milliseconds(), + ) + if err != nil { + return nil, err + } + + var platform string + var count int64 + var result = make(map[string]int64) + for rows.Next() { + if err := rows.Scan(&platform, &count); err != nil { + return nil, err + } + result["all"] += count + if platform == "unknown" { + continue + } + result[platform] = count + } + + return map[string]int64{}, rows.Err() +} + +func (s *statsStatements) InsertUserDailyVisits(ctx context.Context, txn *sql.Tx, localpart, deviceID string, timestamp int64, userAgent string) error { + stmt := sqlutil.TxStmt(txn, s.insertUserDailyVisits) + _, err := stmt.ExecContext(ctx, localpart, deviceID, timestamp, userAgent) + return err +} diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go index ac5c59b81..cdb312468 100644 --- a/userapi/storage/postgres/storage.go +++ b/userapi/storage/postgres/storage.go @@ -85,6 +85,10 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err != nil { return nil, fmt.Errorf("NewPostgresThreePIDTable: %w", err) } + statsTable, err := NewPostgresStatsTable(db, serverName) + if err != nil { + return nil, fmt.Errorf("NewPostgresStatsTable: %w", err) + } return &shared.Database{ AccountDatas: accountDataTable, Accounts: accountsTable, @@ -95,6 +99,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver OpenIDTokens: openIDTable, Profiles: profilesTable, ThreePIDs: threePIDTable, + Stats: statsTable, ServerName: serverName, DB: db, Writer: sqlutil.NewDummyWriter(), diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 5f1f95005..d28373764 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -47,6 +47,7 @@ type Database struct { KeyBackupVersions tables.KeyBackupVersionTable Devices tables.DevicesTable LoginTokens tables.LoginTokenTable + Stats tables.StatsTable LoginTokenLifetime time.Duration ServerName gomatrixserverlib.ServerName BcryptCost int @@ -620,10 +621,18 @@ func (d *Database) RemoveAllDevices( return } -// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address -func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error { +// UpdateDeviceLastSeen updates a last seen timestamp and the ip address. +func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr, userAgent string) error { + err := d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.Devices.UpdateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr, userAgent) + }) + if err != nil { + return err + } + // calculate start of the day + timestamp := time.Now().UTC().Truncate(time.Hour*24).UnixNano() / 1000000 return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.Devices.UpdateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr) + return d.Stats.InsertUserDailyVisits(ctx, txn, localpart, deviceID, timestamp, userAgent) }) } diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go index 423640e90..f46dbace4 100644 --- a/userapi/storage/sqlite3/devices_table.go +++ b/userapi/storage/sqlite3/devices_table.go @@ -81,7 +81,7 @@ const selectDevicesByIDSQL = "" + "SELECT device_id, localpart, display_name FROM device_devices WHERE device_id IN ($1)" const updateDeviceLastSeen = "" + - "UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4" + "UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5" type devicesStatements struct { db *sql.DB @@ -295,9 +295,9 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s return devices, rows.Err() } -func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error { +func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error { lastSeenTs := time.Now().UnixNano() / 1000000 stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt) - _, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID) + _, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, userAgent, localpart, deviceID) return err } diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go index 186b388ef..eb4fbf969 100644 --- a/userapi/storage/tables/interface.go +++ b/userapi/storage/tables/interface.go @@ -48,7 +48,7 @@ type DevicesTable interface { SelectDeviceByID(ctx context.Context, localpart, deviceID string) (*api.Device, error) SelectDevicesByLocalpart(ctx context.Context, txn *sql.Tx, localpart, exceptDeviceID string) ([]api.Device, error) SelectDevicesByID(ctx context.Context, deviceIDs []string) ([]api.Device, error) - UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error + UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error } type KeyBackupTable interface { @@ -94,4 +94,6 @@ type ThreePIDTable interface { DeleteThreePID(ctx context.Context, txn *sql.Tx, threepid string, medium string) (err error) } -type Stats interface{} \ No newline at end of file +type StatsTable interface { + InsertUserDailyVisits(ctx context.Context, txn *sql.Tx, localpart, deviceID string, timestamp int64, userAgent string, ) error +} \ No newline at end of file