mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-16 18:43:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

SQLite

Browse source
This commit is contained in:
Neil Alexander 2022-10-17 15:00:20 +01:00
parent a679a62c9f
commit 56c4f3c42f
No known key found for this signature in database
GPG key ID: A02A2019A2BB0944
6 changed files with 121 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
userapi/storage/postgres/deltas/2022101711000000_rename_tables.go
View file

@ -34,12 +34,12 @@ var renameIndicesMappings = map[string]string{
"account_threepid_localpart": "userapi_threepid_idx",
}
func UpRenameTables(ctx context.Context, tx *sql.Tx) error {
// I know what you're thinking: you're wondering "why doesn't this use $1
// and pass variadic parameters to ExecContext?" — the answer is because
// PostgreSQL doesn't expect the table name to be specified as a substituted
// argument in that way so it results in a syntax error in the query.
// I know what you're thinking: you're wondering "why doesn't this use $1
// and pass variadic parameters to ExecContext?" — the answer is because
// PostgreSQL doesn't expect the table name to be specified as a substituted
// argument in that way so it results in a syntax error in the query.
func UpRenameTables(ctx context.Context, tx *sql.Tx) error {
for old, new := range renameTableMappings {
q := fmt.Sprintf(
"ALTER TABLE IF EXISTS %s RENAME TO %s;",

20
userapi/storage/sqlite3/deltas/20200929203058_is_active.go
View file

@ -8,8 +8,8 @@ import (
func UpIsActive(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, `
ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
CREATE TABLE account_accounts (
ALTER TABLE userapi_accounts RENAME TO userapi_accounts_tmp;
CREATE TABLE userapi_accounts (
localpart TEXT NOT NULL PRIMARY KEY,
created_ts BIGINT NOT NULL,
password_hash TEXT,
@ -17,13 +17,13 @@ CREATE TABLE account_accounts (
is_deactivated BOOLEAN DEFAULT 0
);
INSERT
INTO account_accounts (
INTO userapi_accounts (
localpart, created_ts, password_hash, appservice_id
) SELECT
localpart, created_ts, password_hash, appservice_id
FROM account_accounts_tmp
FROM userapi_accounts_tmp
;
DROP TABLE account_accounts_tmp;`)
DROP TABLE userapi_accounts_tmp;`)
if err != nil {
return fmt.Errorf("failed to execute upgrade: %w", err)
}
@ -32,21 +32,21 @@ DROP TABLE account_accounts_tmp;`)
func DownIsActive(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, `
ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
CREATE TABLE account_accounts (
ALTER TABLE userapi_accounts RENAME TO userapi_accounts_tmp;
CREATE TABLE userapi_accounts (
localpart TEXT NOT NULL PRIMARY KEY,
created_ts BIGINT NOT NULL,
password_hash TEXT,
appservice_id TEXT
);
INSERT
INTO account_accounts (
INTO userapi_accounts (
localpart, created_ts, password_hash, appservice_id
) SELECT
localpart, created_ts, password_hash, appservice_id
FROM account_accounts_tmp
FROM userapi_accounts_tmp
;
DROP TABLE account_accounts_tmp;`)
DROP TABLE userapi_accounts_tmp;`)
if err != nil {
return fmt.Errorf("failed to execute downgrade: %w", err)
}

20
userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go
View file

@ -8,8 +8,8 @@ import (
func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, `
ALTER TABLE device_devices RENAME TO device_devices_tmp;
CREATE TABLE device_devices (
ALTER TABLE userapi_devices RENAME TO userapi_devices_tmp;
CREATE TABLE userapi_devices (
access_token TEXT PRIMARY KEY,
session_id INTEGER,
device_id TEXT ,
@ -22,12 +22,12 @@ func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
UNIQUE (localpart, device_id)
);
INSERT
INTO device_devices (
INTO userapi_devices (
access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent
) SELECT
access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', ''
FROM device_devices_tmp;
DROP TABLE device_devices_tmp;`)
FROM userapi_devices_tmp;
DROP TABLE userapi_devices_tmp;`)
if err != nil {
return fmt.Errorf("failed to execute upgrade: %w", err)
}
@ -36,8 +36,8 @@ func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
func DownLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, `
ALTER TABLE device_devices RENAME TO device_devices_tmp;
CREATE TABLE IF NOT EXISTS device_devices (
ALTER TABLE userapi_devices RENAME TO userapi_devices_tmp;
CREATE TABLE IF NOT EXISTS userapi_devices (
access_token TEXT PRIMARY KEY,
session_id INTEGER,
device_id TEXT ,
@ -47,12 +47,12 @@ CREATE TABLE IF NOT EXISTS device_devices (
UNIQUE (localpart, device_id)
);
INSERT
INTO device_devices (
INTO userapi_devices (
access_token, session_id, device_id, localpart, created_ts, display_name
) SELECT
access_token, session_id, device_id, localpart, created_ts, display_name
FROM device_devices_tmp;
DROP TABLE device_devices_tmp;`)
FROM userapi_devices_tmp;
DROP TABLE userapi_devices_tmp;`)
if err != nil {
return fmt.Errorf("failed to execute downgrade: %w", err)
}

16
userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go
View file

@ -9,8 +9,8 @@ import (
func UpAddAccountType(ctx context.Context, tx *sql.Tx) error {
// initially set every account to useraccount, change appservice and guest accounts afterwards
// (user = 1, guest = 2, admin = 3, appservice = 4)
_, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
CREATE TABLE account_accounts (
_, err := tx.ExecContext(ctx, `ALTER TABLE userapi_accounts RENAME TO userapi_accounts_tmp;
CREATE TABLE userapi_accounts (
localpart TEXT NOT NULL PRIMARY KEY,
created_ts BIGINT NOT NULL,
password_hash TEXT,
@ -19,15 +19,15 @@ CREATE TABLE account_accounts (
account_type INTEGER NOT NULL
);
INSERT
INTO account_accounts (
INTO userapi_accounts (
localpart, created_ts, password_hash, appservice_id, account_type
) SELECT
localpart, created_ts, password_hash, appservice_id, 1
FROM account_accounts_tmp
FROM userapi_accounts_tmp
;
UPDATE account_accounts SET account_type = 4 WHERE appservice_id <> '';
UPDATE account_accounts SET account_type = 2 WHERE localpart GLOB '[0-9]*';
DROP TABLE account_accounts_tmp;`)
UPDATE userapi_accounts SET account_type = 4 WHERE appservice_id <> '';
UPDATE userapi_accounts SET account_type = 2 WHERE localpart GLOB '[0-9]*';
DROP TABLE userapi_accounts_tmp;`)
if err != nil {
return fmt.Errorf("failed to add column: %w", err)
}
@ -35,7 +35,7 @@ DROP TABLE account_accounts_tmp;`)
}
func DownAddAccountType(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts DROP COLUMN account_type;`)
_, err := tx.ExecContext(ctx, `ALTER TABLE userapi_accounts DROP COLUMN account_type;`)
if err != nil {
return fmt.Errorf("failed to execute downgrade: %w", err)
}

84
userapi/storage/sqlite3/deltas/2022101711000000_rename_tables.go
View file

@ -4,6 +4,9 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"github.com/lib/pq"
)
var renameTableMappings = map[string]string{
@ -18,10 +21,51 @@ var renameTableMappings = map[string]string{
"account_threepid": "userapi_threepids",
}
var renameIndicesMappings = map[string]string{
"device_localpart_id_idx": "userapi_device_localpart_id_idx",
"e2e_room_keys_idx": "userapi_key_backups_idx",
"e2e_room_keys_versions_idx": "userapi_key_backups_versions_idx",
"account_e2e_room_keys_versions_idx": "userapi_key_backup_versions_idx",
"login_tokens_expiration_idx": "userapi_login_tokens_expiration_idx",
"account_threepid_localpart": "userapi_threepid_idx",
}
func UpRenameTables(ctx context.Context, tx *sql.Tx) error {
for old, new := range renameTableMappings {
if _, err := tx.ExecContext(ctx, "ALTER TABLE $1 RENAME TO $2;", old, new); err != nil {
return fmt.Errorf("rename %q to %q error: %w", old, new, err)
// SQLite has no "IF EXISTS" so check if the table exists.
var name string
if err := tx.QueryRowContext(
ctx, "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = $1;", old,
).Scan(&name); err != nil {
if err == sql.ErrNoRows {
continue
}
return err
}
q := fmt.Sprintf(
"ALTER TABLE %s RENAME TO %s;",
pq.QuoteIdentifier(old), pq.QuoteIdentifier(new),
)
if _, err := tx.ExecContext(ctx, q); err != nil {
return fmt.Errorf("rename table %q to %q error: %w", old, new, err)
}
}
for old, new := range renameIndicesMappings {
var query string
if err := tx.QueryRowContext(
ctx, "SELECT sql FROM sqlite_schema WHERE type = 'index' AND name = $1;", old,
).Scan(&query); err != nil {
if err == sql.ErrNoRows {
continue
}
return err
}
query = strings.Replace(query, old, new, 1)
if _, err := tx.ExecContext(ctx, "DROP INDEX %s;", pq.QuoteIdentifier(old)); err != nil {
return fmt.Errorf("drop index %q to %q error: %w", old, new, err)
}
if _, err := tx.ExecContext(ctx, query); err != nil {
return fmt.Errorf("recreate index %q to %q error: %w", old, new, err)
}
}
return nil
@ -29,8 +73,40 @@ func UpRenameTables(ctx context.Context, tx *sql.Tx) error {
func DownRenameTables(ctx context.Context, tx *sql.Tx) error {
for old, new := range renameTableMappings {
if _, err := tx.ExecContext(ctx, "ALTER TABLE $1 RENAME TO $2;", new, old); err != nil {
return fmt.Errorf("rename %q to %q error: %w", new, old, err)
// SQLite has no "IF EXISTS" so check if the table exists.
var name string
if err := tx.QueryRowContext(
ctx, "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = $1;", new,
).Scan(&name); err != nil {
if err == sql.ErrNoRows {
continue
}
return err
}
q := fmt.Sprintf(
"ALTER TABLE %s RENAME TO %s;",
pq.QuoteIdentifier(new), pq.QuoteIdentifier(old),
)
if _, err := tx.ExecContext(ctx, q); err != nil {
return fmt.Errorf("rename table %q to %q error: %w", new, old, err)
}
}
for old, new := range renameIndicesMappings {
var query string
if err := tx.QueryRowContext(
ctx, "SELECT sql FROM sqlite_schema WHERE type = 'index' AND name = $1;", new,
).Scan(&query); err != nil {
if err == sql.ErrNoRows {
continue
}
return err
}
query = strings.Replace(query, new, old, 1)
if _, err := tx.ExecContext(ctx, "DROP INDEX %s;", pq.QuoteIdentifier(new)); err != nil {
return fmt.Errorf("drop index %q to %q error: %w", new, old, err)
}
if _, err := tx.ExecContext(ctx, query); err != nil {
return fmt.Errorf("recreate index %q to %q error: %w", new, old, err)
}
}
return nil

16
userapi/storage/sqlite3/stats_table.go
View file

@ -46,7 +46,7 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON usera
const countUsersLastSeenAfterSQL = "" +
"SELECT COUNT(*) FROM (" +
" SELECT localpart FROM device_devices WHERE last_seen_ts > $1 " +
" SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " +
" GROUP BY localpart" +
" ) u"
@ -63,7 +63,7 @@ R30Users counts the number of 30 day retained users, defined as:
const countR30UsersSQL = `
SELECT platform, COUNT(*) FROM (
SELECT users.localpart, platform, users.created_ts, MAX(uip.last_seen_ts)
FROM account_accounts users
FROM userapi_accounts users
INNER JOIN
(SELECT
localpart, last_seen_ts,
@ -76,7 +76,7 @@ SELECT platform, COUNT(*) FROM (
ELSE 'unknown'
END
AS platform
FROM device_devices
FROM userapi_devices
) uip
ON users.localpart = uip.localpart
AND users.account_type <> 4
@ -126,7 +126,7 @@ GROUP BY client_type
`
const countUserByAccountTypeSQL = `
SELECT COUNT(*) FROM account_accounts WHERE account_type IN ($1)
SELECT COUNT(*) FROM userapi_accounts WHERE account_type IN ($1)
`
// $1 = Guest AccountType
@ -139,7 +139,7 @@ SELECT user_type, COUNT(*) AS count FROM (
WHEN account_type = $4 AND appservice_id IS NULL THEN 'guest'
WHEN account_type IN ($5) AND appservice_id IS NOT NULL THEN 'bridged'
END AS user_type
FROM account_accounts
FROM userapi_accounts
WHERE created_ts > $8
) AS t GROUP BY user_type
`
@ -148,14 +148,14 @@ SELECT user_type, COUNT(*) AS count FROM (
const updateUserDailyVisitsSQL = `
INSERT INTO userapi_daily_visits(localpart, device_id, timestamp, user_agent)
SELECT u.localpart, u.device_id, $1, MAX(u.user_agent)
FROM device_devices AS u
FROM userapi_devices AS u
LEFT JOIN (
SELECT localpart, device_id, timestamp FROM userapi_daily_visits
WHERE timestamp = $1
) udv
ON u.localpart = udv.localpart AND u.device_id = udv.device_id
INNER JOIN device_devices d ON d.localpart = u.localpart
INNER JOIN account_accounts a ON a.localpart = u.localpart
INNER JOIN userapi_devices d ON d.localpart = u.localpart
INNER JOIN userapi_accounts a ON a.localpart = u.localpart
WHERE $2 <= d.last_seen_ts AND d.last_seen_ts < $3
AND a.account_type in (1, 3)
GROUP BY u.localpart, u.device_id