Add userAgent to UpdateDeviceLastSeen

Add new Table for tracking daily user vists

syncapi/sync/requestpool.go

@@ -102,6 +102,7 @@ func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device)
 		UserID:     device.UserID,
 		DeviceID:   device.ID,
 		RemoteAddr: remoteAddr,
+		UserAgent:  req.UserAgent(),
 	}
 	lsres := &userapi.PerformLastSeenUpdateResponse{}
 	go rp.userAPI.PerformLastSeenUpdate(req.Context(), lsreq, lsres) // nolint:errcheck

userapi/api/api.go

@@ -274,6 +274,7 @@ type PerformLastSeenUpdateRequest struct {
 	UserID     string
 	DeviceID   string
 	RemoteAddr string
+	UserAgent  string
 }
 
 // PerformLastSeenUpdateResponse is the response for PerformLastSeenUpdate.

userapi/internal/api.go

@@ -194,7 +194,7 @@ func (a *UserInternalAPI) PerformLastSeenUpdate(
 	if err != nil {
 		return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
 	}
-	if err := a.DB.UpdateDeviceLastSeen(ctx, localpart, req.DeviceID, req.RemoteAddr); err != nil {
+	if err := a.DB.UpdateDeviceLastSeen(ctx, localpart, req.DeviceID, req.RemoteAddr, req.UserAgent); err != nil {
 		return fmt.Errorf("a.DeviceDB.UpdateDeviceLastSeen: %w", err)
 	}
 	return nil

userapi/storage/interface.go

@@ -73,7 +73,7 @@ type Database interface {
 	// Returns the device on success.
 	CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string) (dev *api.Device, returnErr error)
 	UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error
-	UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error
+	UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr, userAgent string) error
 	RemoveDevice(ctx context.Context, deviceID, localpart string) error
 	RemoveDevices(ctx context.Context, localpart string, devices []string) error
 	// RemoveAllDevices deleted all devices for this user. Returns the devices deleted.

userapi/storage/postgres/devices_table.go

@@ -96,7 +96,7 @@ const selectDevicesByIDSQL = "" +
 	"SELECT device_id, localpart, display_name FROM device_devices WHERE device_id = ANY($1)"
 
 const updateDeviceLastSeen = "" +
-	"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
+	"UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5"
 
 type devicesStatements struct {
 	insertDeviceStmt             *sql.Stmt
@@ -293,9 +293,9 @@ func (s *devicesStatements) SelectDevicesByLocalpart(
 	return devices, rows.Err()
 }
 
-func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error {
+func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error {
 	lastSeenTs := time.Now().UnixNano() / 1000000
 	stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt)
-	_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID)
+	_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, userAgent, localpart, deviceID)
 	return err
 }

userapi/storage/postgres/stats.go

@@ -1,69 +0,0 @@
-// Copyright 2022 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"context"
-	"database/sql"
-	"time"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/syncapi/storage/tables"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-const countUsersLastSeenAfterSQL = ""+
-	"SELECT COUNT(*) FROM ("+
-	" SELECT user_id FROM device_devices WHERE last_seen > $1 "+
-	" GROUP BY user_id"+
-	" )"
-
-const countActiveRoomsSQL = ""+
-	"SELECT COUNT(DISTINCT room_id) FROM syncapi_output_room_events"+
-	" WHERE type = $1 AND id > $2"
-
-type statsStatements struct {
-	serverName string
-	countUsersLastSeenAfterStmt *sql.Stmt
-	countActiveRoomsStmt *sql.Stmt
-}
-
-func PrepareStats(db *sql.DB, serverName string) (tables.Stats, error) {
-	s := &statsStatements{
-		serverName: serverName,
-	}
-	return s, sqlutil.StatementList{
-		{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
-		{&s.countActiveRoomsStmt, countActiveRoomsSQL},
-	}.Prepare(db)
-}
-
-func (s *statsStatements) DailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
-	stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
-	lastSeenAfter := time.Now().AddDate(0, 0, 1)
-	err = stmt.QueryRowContext(ctx,
-		gomatrixserverlib.AsTimestamp(lastSeenAfter),
-	).Scan(&result)
-	return
-}
-
-func (s *statsStatements) MonthlyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
-	stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
-	lastSeenAfter := time.Now().AddDate(0, 0, 30)
-	err = stmt.QueryRowContext(ctx,
-		gomatrixserverlib.AsTimestamp(lastSeenAfter),
-	).Scan(&result)
-	return
-}

userapi/storage/postgres/stats_table.go

@@ -0,0 +1,199 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/userapi/storage/tables"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const userDailyVisitsSchema = `
+CREATE TABLE IF NOT EXISTS user_daily_visits (
+    localpart TEXT NOT NULL,
+	device_id TEXT NOT NULL,
+	timestamp BIGINT NOT NULL,
+	user_agent TEXT
+);
+
+-- Device IDs and timestamp must be unique for a given user per day
+CREATE UNIQUE INDEX IF NOT EXISTS localpart_device_timestamp_idx ON user_daily_visits(localpart, device_id, timestamp);
+CREATE INDEX IF NOT EXISTS timestamp_idx ON user_daily_visits(timestamp);
+CREATE INDEX IF NOT EXISTS localpart_timestamp_idx ON user_daily_visits(localpart, timestamp);
+`
+
+const countUsersLastSeenAfterSQL = "" +
+	"SELECT COUNT(*) FROM (" +
+	" SELECT user_id FROM device_devices WHERE last_seen > $1 " +
+	" GROUP BY user_id" +
+	" )"
+
+const countR30UsersSQL = `
+SELECT platform, COUNT(*) FROM (
+	SELECT users.localpart, platform, users.created_ts, MAX(uip.last_seen_ts)
+	FROM account_accounts users
+	INNER JOIN
+	(SELECT 
+		localpart, last_seen_ts,
+		CASE
+	    	WHEN user_agent LIKE '%%Android%%' OR display_name LIKE '%%android%%' THEN 'android'
+    	    WHEN user_agent LIKE '%%iOS%%' THEN 'ios'
+        	WHEN user_agent LIKE '%%Electron%%' THEN 'electron'
+        	WHEN user_agent LIKE '%%Mozilla%%' THEN 'web'
+        	WHEN user_agent LIKE '%%Gecko%%' THEN 'web'
+        	ELSE 'unknown'
+		END
+    	AS platform
+		FROM device_devices
+	) uip
+	ON users.localpart = uip.localpart
+	AND users.account_type <> 4
+	AND users.created_ts < $1
+	AND uip.last_seen_ts > $1,
+	AND (uip.last_seen_ts) - users.created_ts > $2
+	GROUP BY users.localpart, platform, users.created_ts
+	) u GROUP BY PLATFORM
+`
+
+const countR30UsersV2SQL = `
+
+`
+
+const insertUserDailyVists = `
+	INSERT INTO user_daily_visits(localpart, device_id, timestamp, user_agent) VALUES ($1, $2, $3, $4)
+	ON CONFLICT ON CONSTRAINT localpart_device_timestamp_idx DO NOTHING
+`
+
+type statsStatements struct {
+	serverName                  gomatrixserverlib.ServerName
+	countUsersLastSeenAfterStmt *sql.Stmt
+	countR30UsersStmt           *sql.Stmt
+	countR30UsersV2Stmt         *sql.Stmt
+	insertUserDailyVisits       *sql.Stmt
+}
+
+func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) {
+	s := &statsStatements{
+		serverName: serverName,
+	}
+	_, err := db.Exec(userDailyVisitsSchema)
+	if err != nil {
+		return nil, err
+	}
+	return s, sqlutil.StatementList{
+		{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
+		{&s.countR30UsersStmt, countR30UsersSQL},
+		{&s.countR30UsersV2Stmt, countR30UsersV2SQL},
+		{&s.insertUserDailyVisits, insertUserDailyVists},
+	}.Prepare(db)
+}
+
+func (s *statsStatements) DailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -1)
+	err = stmt.QueryRowContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+	).Scan(&result)
+	return
+}
+
+func (s *statsStatements) MonthlyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -30)
+	err = stmt.QueryRowContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+	).Scan(&result)
+	return
+}
+
+/* R30Users counts the number of 30 day retained users, defined as:
+- Users who have created their accounts more than 30 days ago
+- Where last seen at most 30 days ago
+- Where account creation and last_seen are > 30 days apart
+*/
+func (s *statsStatements) R30Users(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.countR30UsersStmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -30)
+	diff := time.Hour * 24 * 30
+
+	rows, err := stmt.QueryContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+		diff.Milliseconds(),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	var platform string
+	var count int64
+	var result = make(map[string]int64)
+	for rows.Next() {
+		if err := rows.Scan(&platform, &count); err != nil {
+			return nil, err
+		}
+		result["all"] += count
+		if platform == "unknown" {
+			continue
+		}
+		result[platform] = count
+	}
+
+	return map[string]int64{}, rows.Err()
+}
+
+/* R30UsersV2 counts the number of 30 day retained users, defined as users that:
+- Appear more than once in the past 60 days
+- Have more than 30 days between the most and least recent appearances that occurred in the past 60 days.
+*/
+func (s *statsStatements) R30UsersV2(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.countR30UsersV2Stmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -30)
+	diff := time.Hour * 24 * 30
+	diff.Milliseconds()
+
+	rows, err := stmt.QueryContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+		diff.Milliseconds(),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	var platform string
+	var count int64
+	var result = make(map[string]int64)
+	for rows.Next() {
+		if err := rows.Scan(&platform, &count); err != nil {
+			return nil, err
+		}
+		result["all"] += count
+		if platform == "unknown" {
+			continue
+		}
+		result[platform] = count
+	}
+
+	return map[string]int64{}, rows.Err()
+}
+
+func (s *statsStatements) InsertUserDailyVisits(ctx context.Context, txn *sql.Tx, localpart, deviceID string, timestamp int64, userAgent string) error {
+	stmt := sqlutil.TxStmt(txn, s.insertUserDailyVisits)
+	_, err := stmt.ExecContext(ctx, localpart, deviceID, timestamp, userAgent)
+	return err
+}

userapi/storage/postgres/storage.go

@@ -85,6 +85,10 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 	if err != nil {
 		return nil, fmt.Errorf("NewPostgresThreePIDTable: %w", err)
 	}
+	statsTable, err := NewPostgresStatsTable(db, serverName)
+	if err != nil {
+		return nil, fmt.Errorf("NewPostgresStatsTable: %w", err)
+	}
 	return &shared.Database{
 		AccountDatas:          accountDataTable,
 		Accounts:              accountsTable,
@@ -95,6 +99,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 		OpenIDTokens:          openIDTable,
 		Profiles:              profilesTable,
 		ThreePIDs:             threePIDTable,
+		Stats:                 statsTable,
 		ServerName:            serverName,
 		DB:                    db,
 		Writer:                sqlutil.NewDummyWriter(),

userapi/storage/shared/storage.go

@@ -47,6 +47,7 @@ type Database struct {
 	KeyBackupVersions     tables.KeyBackupVersionTable
 	Devices               tables.DevicesTable
 	LoginTokens           tables.LoginTokenTable
+	Stats                 tables.StatsTable
 	LoginTokenLifetime    time.Duration
 	ServerName            gomatrixserverlib.ServerName
 	BcryptCost            int
@@ -620,10 +621,18 @@ func (d *Database) RemoveAllDevices(
 	return
 }
 
-// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address
+// UpdateDeviceLastSeen updates a last seen timestamp and the ip address.
-func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error {
+func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr, userAgent string) error {
+	err := d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.Devices.UpdateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr, userAgent)
+	})
+	if err != nil {
+		return err
+	}
+	// calculate start of the day
+	timestamp := time.Now().UTC().Truncate(time.Hour*24).UnixNano() / 1000000
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.Devices.UpdateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr)
+		return d.Stats.InsertUserDailyVisits(ctx, txn, localpart, deviceID, timestamp, userAgent)
 	})
 }
 

userapi/storage/sqlite3/devices_table.go

@@ -81,7 +81,7 @@ const selectDevicesByIDSQL = "" +
 	"SELECT device_id, localpart, display_name FROM device_devices WHERE device_id IN ($1)"
 
 const updateDeviceLastSeen = "" +
-	"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
+	"UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5"
 
 type devicesStatements struct {
 	db                           *sql.DB
@@ -295,9 +295,9 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s
 	return devices, rows.Err()
 }
 
-func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error {
+func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error {
 	lastSeenTs := time.Now().UnixNano() / 1000000
 	stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt)
-	_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID)
+	_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, userAgent, localpart, deviceID)
 	return err
 }

userapi/storage/tables/interface.go

@@ -48,7 +48,7 @@ type DevicesTable interface {
 	SelectDeviceByID(ctx context.Context, localpart, deviceID string) (*api.Device, error)
 	SelectDevicesByLocalpart(ctx context.Context, txn *sql.Tx, localpart, exceptDeviceID string) ([]api.Device, error)
 	SelectDevicesByID(ctx context.Context, deviceIDs []string) ([]api.Device, error)
-	UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error
+	UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error
 }
 
 type KeyBackupTable interface {
@@ -94,4 +94,6 @@ type ThreePIDTable interface {
 	DeleteThreePID(ctx context.Context, txn *sql.Tx, threepid string, medium string) (err error)
 }
 
-type Stats interface{}
+type StatsTable interface {
+	InsertUserDailyVisits(ctx context.Context, txn *sql.Tx, localpart, deviceID string, timestamp int64, userAgent string, ) error
+}