mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-01 03:03:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add userAgent to UpdateDeviceLastSeen

Browse source 
Add new Table for tracking daily user vists
This commit is contained in:
Till Faelligen 2022-03-02 09:53:49 +01:00
parent 9e14e6afe0
commit 3f0ed455b0
11 changed files with 230 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
syncapi/sync/requestpool.go
View file

@ -102,6 +102,7 @@ func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device)
UserID: device.UserID,
DeviceID: device.ID,
RemoteAddr: remoteAddr,
UserAgent: req.UserAgent(),
}
lsres := &userapi.PerformLastSeenUpdateResponse{}
go rp.userAPI.PerformLastSeenUpdate(req.Context(), lsreq, lsres) // nolint:errcheck

1
userapi/api/api.go
View file

@ -274,6 +274,7 @@ type PerformLastSeenUpdateRequest struct {
UserID string
DeviceID string
RemoteAddr string
UserAgent string
}
// PerformLastSeenUpdateResponse is the response for PerformLastSeenUpdate.

2
userapi/internal/api.go
View file

@ -194,7 +194,7 @@ func (a *UserInternalAPI) PerformLastSeenUpdate(
if err != nil {
return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
}
if err := a.DB.UpdateDeviceLastSeen(ctx, localpart, req.DeviceID, req.RemoteAddr); err != nil {
if err := a.DB.UpdateDeviceLastSeen(ctx, localpart, req.DeviceID, req.RemoteAddr, req.UserAgent); err != nil {
return fmt.Errorf("a.DeviceDB.UpdateDeviceLastSeen: %w", err)
}
return nil

2
userapi/storage/interface.go
View file

@ -73,7 +73,7 @@ type Database interface {
// Returns the device on success.
CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string) (dev *api.Device, returnErr error)
UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error
UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error
UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr, userAgent string) error
RemoveDevice(ctx context.Context, deviceID, localpart string) error
RemoveDevices(ctx context.Context, localpart string, devices []string) error
// RemoveAllDevices deleted all devices for this user. Returns the devices deleted.

6
userapi/storage/postgres/devices_table.go
View file

@ -96,7 +96,7 @@ const selectDevicesByIDSQL = "" +
"SELECT device_id, localpart, display_name FROM device_devices WHERE device_id = ANY($1)"
const updateDeviceLastSeen = "" +
"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
"UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5"
type devicesStatements struct {
insertDeviceStmt *sql.Stmt
@ -293,9 +293,9 @@ func (s *devicesStatements) SelectDevicesByLocalpart(
return devices, rows.Err()
}
func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error {
func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error {
lastSeenTs := time.Now().UnixNano() / 1000000
stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt)
_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID)
_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, userAgent, localpart, deviceID)
return err
}

69
userapi/storage/postgres/stats.go
View file

@ -1,69 +0,0 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
"time"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/syncapi/storage/tables"
"github.com/matrix-org/gomatrixserverlib"
)
const countUsersLastSeenAfterSQL = ""+
"SELECT COUNT(*) FROM ("+
" SELECT user_id FROM device_devices WHERE last_seen > $1 "+
" GROUP BY user_id"+
" )"
const countActiveRoomsSQL = ""+
"SELECT COUNT(DISTINCT room_id) FROM syncapi_output_room_events"+
" WHERE type = $1 AND id > $2"
type statsStatements struct {
serverName string
countUsersLastSeenAfterStmt *sql.Stmt
countActiveRoomsStmt *sql.Stmt
}
func PrepareStats(db *sql.DB, serverName string) (tables.Stats, error) {
s := &statsStatements{
serverName: serverName,
}
return s, sqlutil.StatementList{
{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
{&s.countActiveRoomsStmt, countActiveRoomsSQL},
}.Prepare(db)
}
func (s *statsStatements) DailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
lastSeenAfter := time.Now().AddDate(0, 0, 1)
err = stmt.QueryRowContext(ctx,
gomatrixserverlib.AsTimestamp(lastSeenAfter),
).Scan(&result)
return
}
func (s *statsStatements) MonthlyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
lastSeenAfter := time.Now().AddDate(0, 0, 30)
err = stmt.QueryRowContext(ctx,
gomatrixserverlib.AsTimestamp(lastSeenAfter),
).Scan(&result)
return
}

199
userapi/storage/postgres/stats_table.go Normal file
View file

@ -0,0 +1,199 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
"time"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/userapi/storage/tables"
"github.com/matrix-org/gomatrixserverlib"
)
const userDailyVisitsSchema = `
CREATE TABLE IF NOT EXISTS user_daily_visits (
localpart TEXT NOT NULL,
device_id TEXT NOT NULL,
timestamp BIGINT NOT NULL,
user_agent TEXT
);
-- Device IDs and timestamp must be unique for a given user per day
CREATE UNIQUE INDEX IF NOT EXISTS localpart_device_timestamp_idx ON user_daily_visits(localpart, device_id, timestamp);
CREATE INDEX IF NOT EXISTS timestamp_idx ON user_daily_visits(timestamp);
CREATE INDEX IF NOT EXISTS localpart_timestamp_idx ON user_daily_visits(localpart, timestamp);
`
const countUsersLastSeenAfterSQL = "" +
"SELECT COUNT(*) FROM (" +
" SELECT user_id FROM device_devices WHERE last_seen > $1 " +
" GROUP BY user_id" +
" )"
const countR30UsersSQL = `
SELECT platform, COUNT(*) FROM (
SELECT users.localpart, platform, users.created_ts, MAX(uip.last_seen_ts)
FROM account_accounts users
INNER JOIN
(SELECT
localpart, last_seen_ts,
CASE
WHEN user_agent LIKE '%%Android%%' OR display_name LIKE '%%android%%' THEN 'android'
WHEN user_agent LIKE '%%iOS%%' THEN 'ios'
WHEN user_agent LIKE '%%Electron%%' THEN 'electron'
WHEN user_agent LIKE '%%Mozilla%%' THEN 'web'
WHEN user_agent LIKE '%%Gecko%%' THEN 'web'
ELSE 'unknown'
END
AS platform
FROM device_devices
) uip
ON users.localpart = uip.localpart
AND users.account_type <> 4
AND users.created_ts < $1
AND uip.last_seen_ts > $1,
AND (uip.last_seen_ts) - users.created_ts > $2
GROUP BY users.localpart, platform, users.created_ts
) u GROUP BY PLATFORM
`
const countR30UsersV2SQL = `
`
const insertUserDailyVists = `
INSERT INTO user_daily_visits(localpart, device_id, timestamp, user_agent) VALUES ($1, $2, $3, $4)
ON CONFLICT ON CONSTRAINT localpart_device_timestamp_idx DO NOTHING
`
type statsStatements struct {
serverName gomatrixserverlib.ServerName
countUsersLastSeenAfterStmt *sql.Stmt
countR30UsersStmt *sql.Stmt
countR30UsersV2Stmt *sql.Stmt
insertUserDailyVisits *sql.Stmt
}
func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) {
s := &statsStatements{
serverName: serverName,
}
_, err := db.Exec(userDailyVisitsSchema)
if err != nil {
return nil, err
}
return s, sqlutil.StatementList{
{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
{&s.countR30UsersStmt, countR30UsersSQL},
{&s.countR30UsersV2Stmt, countR30UsersV2SQL},
{&s.insertUserDailyVisits, insertUserDailyVists},
}.Prepare(db)
}
func (s *statsStatements) DailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
lastSeenAfter := time.Now().AddDate(0, 0, -1)
err = stmt.QueryRowContext(ctx,
gomatrixserverlib.AsTimestamp(lastSeenAfter),
).Scan(&result)
return
}
func (s *statsStatements) MonthlyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
lastSeenAfter := time.Now().AddDate(0, 0, -30)
err = stmt.QueryRowContext(ctx,
gomatrixserverlib.AsTimestamp(lastSeenAfter),
).Scan(&result)
return
}
/* R30Users counts the number of 30 day retained users, defined as:
- Users who have created their accounts more than 30 days ago
- Where last seen at most 30 days ago
- Where account creation and last_seen are > 30 days apart
*/
func (s *statsStatements) R30Users(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
stmt := sqlutil.TxStmt(txn, s.countR30UsersStmt)
lastSeenAfter := time.Now().AddDate(0, 0, -30)
diff := time.Hour * 24 * 30
rows, err := stmt.QueryContext(ctx,
gomatrixserverlib.AsTimestamp(lastSeenAfter),
diff.Milliseconds(),
)
if err != nil {
return nil, err
}
var platform string
var count int64
var result = make(map[string]int64)
for rows.Next() {
if err := rows.Scan(&platform, &count); err != nil {
return nil, err
}
result["all"] += count
if platform == "unknown" {
continue
}
result[platform] = count
}
return map[string]int64{}, rows.Err()
}
/* R30UsersV2 counts the number of 30 day retained users, defined as users that:
- Appear more than once in the past 60 days
- Have more than 30 days between the most and least recent appearances that occurred in the past 60 days.
*/
func (s *statsStatements) R30UsersV2(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
stmt := sqlutil.TxStmt(txn, s.countR30UsersV2Stmt)
lastSeenAfter := time.Now().AddDate(0, 0, -30)
diff := time.Hour * 24 * 30
diff.Milliseconds()
rows, err := stmt.QueryContext(ctx,
gomatrixserverlib.AsTimestamp(lastSeenAfter),
diff.Milliseconds(),
)
if err != nil {
return nil, err
}
var platform string
var count int64
var result = make(map[string]int64)
for rows.Next() {
if err := rows.Scan(&platform, &count); err != nil {
return nil, err
}
result["all"] += count
if platform == "unknown" {
continue
}
result[platform] = count
}
return map[string]int64{}, rows.Err()
}
func (s *statsStatements) InsertUserDailyVisits(ctx context.Context, txn *sql.Tx, localpart, deviceID string, timestamp int64, userAgent string) error {
stmt := sqlutil.TxStmt(txn, s.insertUserDailyVisits)
_, err := stmt.ExecContext(ctx, localpart, deviceID, timestamp, userAgent)
return err
}

5
userapi/storage/postgres/storage.go
View file

@ -85,6 +85,10 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
if err != nil {
return nil, fmt.Errorf("NewPostgresThreePIDTable: %w", err)
}
statsTable, err := NewPostgresStatsTable(db, serverName)
if err != nil {
return nil, fmt.Errorf("NewPostgresStatsTable: %w", err)
}
return &shared.Database{
AccountDatas: accountDataTable,
Accounts: accountsTable,
@ -95,6 +99,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
OpenIDTokens: openIDTable,
Profiles: profilesTable,
ThreePIDs: threePIDTable,
Stats: statsTable,
ServerName: serverName,
DB: db,
Writer: sqlutil.NewDummyWriter(),

15
userapi/storage/shared/storage.go
View file

@ -47,6 +47,7 @@ type Database struct {
KeyBackupVersions tables.KeyBackupVersionTable
Devices tables.DevicesTable
LoginTokens tables.LoginTokenTable
Stats tables.StatsTable
LoginTokenLifetime time.Duration
ServerName gomatrixserverlib.ServerName
BcryptCost int
@ -620,10 +621,18 @@ func (d *Database) RemoveAllDevices(
return
}
// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address
func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error {
// UpdateDeviceLastSeen updates a last seen timestamp and the ip address.
func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr, userAgent string) error {
err := d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
return d.Devices.UpdateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr, userAgent)
})
if err != nil {
return err
}
// calculate start of the day
timestamp := time.Now().UTC().Truncate(time.Hour*24).UnixNano() / 1000000
return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
return d.Devices.UpdateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr)
return d.Stats.InsertUserDailyVisits(ctx, txn, localpart, deviceID, timestamp, userAgent)
})
}

6
userapi/storage/sqlite3/devices_table.go
View file

@ -81,7 +81,7 @@ const selectDevicesByIDSQL = "" +
"SELECT device_id, localpart, display_name FROM device_devices WHERE device_id IN ($1)"
const updateDeviceLastSeen = "" +
"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
"UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5"
type devicesStatements struct {
db *sql.DB
@ -295,9 +295,9 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s
return devices, rows.Err()
}
func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error {
func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error {
lastSeenTs := time.Now().UnixNano() / 1000000
stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt)
_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID)
_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, userAgent, localpart, deviceID)
return err
}

6
userapi/storage/tables/interface.go
View file

@ -48,7 +48,7 @@ type DevicesTable interface {
SelectDeviceByID(ctx context.Context, localpart, deviceID string) (*api.Device, error)
SelectDevicesByLocalpart(ctx context.Context, txn *sql.Tx, localpart, exceptDeviceID string) ([]api.Device, error)
SelectDevicesByID(ctx context.Context, deviceIDs []string) ([]api.Device, error)
UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error
UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error
}
type KeyBackupTable interface {
@ -94,4 +94,6 @@ type ThreePIDTable interface {
DeleteThreePID(ctx context.Context, txn *sql.Tx, threepid string, medium string) (err error)
}
type Stats interface{}
type StatsTable interface {
InsertUserDailyVisits(ctx context.Context, txn *sql.Tx, localpart, deviceID string, timestamp int64, userAgent string, ) error
}