diff --git a/userapi/storage/postgres/deltas/2022101711000000_rename_tables.go b/userapi/storage/postgres/deltas/2022101711000000_rename_tables.go index 01f697c38..1d73d0af4 100644 --- a/userapi/storage/postgres/deltas/2022101711000000_rename_tables.go +++ b/userapi/storage/postgres/deltas/2022101711000000_rename_tables.go @@ -34,12 +34,12 @@ var renameIndicesMappings = map[string]string{ "account_threepid_localpart": "userapi_threepid_idx", } -func UpRenameTables(ctx context.Context, tx *sql.Tx) error { - // I know what you're thinking: you're wondering "why doesn't this use $1 - // and pass variadic parameters to ExecContext?" — the answer is because - // PostgreSQL doesn't expect the table name to be specified as a substituted - // argument in that way so it results in a syntax error in the query. +// I know what you're thinking: you're wondering "why doesn't this use $1 +// and pass variadic parameters to ExecContext?" — the answer is because +// PostgreSQL doesn't expect the table name to be specified as a substituted +// argument in that way so it results in a syntax error in the query. +func UpRenameTables(ctx context.Context, tx *sql.Tx) error { for old, new := range renameTableMappings { q := fmt.Sprintf( "ALTER TABLE IF EXISTS %s RENAME TO %s;", diff --git a/userapi/storage/sqlite3/deltas/20200929203058_is_active.go b/userapi/storage/sqlite3/deltas/20200929203058_is_active.go index e25efc695..9158cb365 100644 --- a/userapi/storage/sqlite3/deltas/20200929203058_is_active.go +++ b/userapi/storage/sqlite3/deltas/20200929203058_is_active.go @@ -8,8 +8,8 @@ import ( func UpIsActive(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` - ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( + ALTER TABLE userapi_accounts RENAME TO userapi_accounts_tmp; +CREATE TABLE userapi_accounts ( localpart TEXT NOT NULL PRIMARY KEY, created_ts BIGINT NOT NULL, password_hash TEXT, @@ -17,13 +17,13 @@ CREATE TABLE account_accounts ( is_deactivated BOOLEAN DEFAULT 0 ); INSERT - INTO account_accounts ( + INTO userapi_accounts ( localpart, created_ts, password_hash, appservice_id ) SELECT localpart, created_ts, password_hash, appservice_id - FROM account_accounts_tmp + FROM userapi_accounts_tmp ; -DROP TABLE account_accounts_tmp;`) +DROP TABLE userapi_accounts_tmp;`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } @@ -32,21 +32,21 @@ DROP TABLE account_accounts_tmp;`) func DownIsActive(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` - ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( + ALTER TABLE userapi_accounts RENAME TO userapi_accounts_tmp; +CREATE TABLE userapi_accounts ( localpart TEXT NOT NULL PRIMARY KEY, created_ts BIGINT NOT NULL, password_hash TEXT, appservice_id TEXT ); INSERT - INTO account_accounts ( + INTO userapi_accounts ( localpart, created_ts, password_hash, appservice_id ) SELECT localpart, created_ts, password_hash, appservice_id - FROM account_accounts_tmp + FROM userapi_accounts_tmp ; -DROP TABLE account_accounts_tmp;`) +DROP TABLE userapi_accounts_tmp;`) if err != nil { return fmt.Errorf("failed to execute downgrade: %w", err) } diff --git a/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go index 7f7e95d2d..a9224db6b 100644 --- a/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go +++ b/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go @@ -8,8 +8,8 @@ import ( func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` - ALTER TABLE device_devices RENAME TO device_devices_tmp; - CREATE TABLE device_devices ( + ALTER TABLE userapi_devices RENAME TO userapi_devices_tmp; + CREATE TABLE userapi_devices ( access_token TEXT PRIMARY KEY, session_id INTEGER, device_id TEXT , @@ -22,12 +22,12 @@ func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { UNIQUE (localpart, device_id) ); INSERT - INTO device_devices ( + INTO userapi_devices ( access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent ) SELECT access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', '' - FROM device_devices_tmp; - DROP TABLE device_devices_tmp;`) + FROM userapi_devices_tmp; + DROP TABLE userapi_devices_tmp;`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } @@ -36,8 +36,8 @@ func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { func DownLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` -ALTER TABLE device_devices RENAME TO device_devices_tmp; -CREATE TABLE IF NOT EXISTS device_devices ( +ALTER TABLE userapi_devices RENAME TO userapi_devices_tmp; +CREATE TABLE IF NOT EXISTS userapi_devices ( access_token TEXT PRIMARY KEY, session_id INTEGER, device_id TEXT , @@ -47,12 +47,12 @@ CREATE TABLE IF NOT EXISTS device_devices ( UNIQUE (localpart, device_id) ); INSERT -INTO device_devices ( +INTO userapi_devices ( access_token, session_id, device_id, localpart, created_ts, display_name ) SELECT access_token, session_id, device_id, localpart, created_ts, display_name -FROM device_devices_tmp; -DROP TABLE device_devices_tmp;`) +FROM userapi_devices_tmp; +DROP TABLE userapi_devices_tmp;`) if err != nil { return fmt.Errorf("failed to execute downgrade: %w", err) } diff --git a/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go b/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go index 46532698c..230bc1433 100644 --- a/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go +++ b/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go @@ -9,8 +9,8 @@ import ( func UpAddAccountType(ctx context.Context, tx *sql.Tx) error { // initially set every account to useraccount, change appservice and guest accounts afterwards // (user = 1, guest = 2, admin = 3, appservice = 4) - _, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( + _, err := tx.ExecContext(ctx, `ALTER TABLE userapi_accounts RENAME TO userapi_accounts_tmp; +CREATE TABLE userapi_accounts ( localpart TEXT NOT NULL PRIMARY KEY, created_ts BIGINT NOT NULL, password_hash TEXT, @@ -19,15 +19,15 @@ CREATE TABLE account_accounts ( account_type INTEGER NOT NULL ); INSERT - INTO account_accounts ( + INTO userapi_accounts ( localpart, created_ts, password_hash, appservice_id, account_type ) SELECT localpart, created_ts, password_hash, appservice_id, 1 - FROM account_accounts_tmp + FROM userapi_accounts_tmp ; -UPDATE account_accounts SET account_type = 4 WHERE appservice_id <> ''; -UPDATE account_accounts SET account_type = 2 WHERE localpart GLOB '[0-9]*'; -DROP TABLE account_accounts_tmp;`) +UPDATE userapi_accounts SET account_type = 4 WHERE appservice_id <> ''; +UPDATE userapi_accounts SET account_type = 2 WHERE localpart GLOB '[0-9]*'; +DROP TABLE userapi_accounts_tmp;`) if err != nil { return fmt.Errorf("failed to add column: %w", err) } @@ -35,7 +35,7 @@ DROP TABLE account_accounts_tmp;`) } func DownAddAccountType(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts DROP COLUMN account_type;`) + _, err := tx.ExecContext(ctx, `ALTER TABLE userapi_accounts DROP COLUMN account_type;`) if err != nil { return fmt.Errorf("failed to execute downgrade: %w", err) } diff --git a/userapi/storage/sqlite3/deltas/2022101711000000_rename_tables.go b/userapi/storage/sqlite3/deltas/2022101711000000_rename_tables.go index 949008cda..074ef2cdc 100644 --- a/userapi/storage/sqlite3/deltas/2022101711000000_rename_tables.go +++ b/userapi/storage/sqlite3/deltas/2022101711000000_rename_tables.go @@ -4,6 +4,9 @@ import ( "context" "database/sql" "fmt" + "strings" + + "github.com/lib/pq" ) var renameTableMappings = map[string]string{ @@ -18,10 +21,51 @@ var renameTableMappings = map[string]string{ "account_threepid": "userapi_threepids", } +var renameIndicesMappings = map[string]string{ + "device_localpart_id_idx": "userapi_device_localpart_id_idx", + "e2e_room_keys_idx": "userapi_key_backups_idx", + "e2e_room_keys_versions_idx": "userapi_key_backups_versions_idx", + "account_e2e_room_keys_versions_idx": "userapi_key_backup_versions_idx", + "login_tokens_expiration_idx": "userapi_login_tokens_expiration_idx", + "account_threepid_localpart": "userapi_threepid_idx", +} + func UpRenameTables(ctx context.Context, tx *sql.Tx) error { for old, new := range renameTableMappings { - if _, err := tx.ExecContext(ctx, "ALTER TABLE $1 RENAME TO $2;", old, new); err != nil { - return fmt.Errorf("rename %q to %q error: %w", old, new, err) + // SQLite has no "IF EXISTS" so check if the table exists. + var name string + if err := tx.QueryRowContext( + ctx, "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = $1;", old, + ).Scan(&name); err != nil { + if err == sql.ErrNoRows { + continue + } + return err + } + q := fmt.Sprintf( + "ALTER TABLE %s RENAME TO %s;", + pq.QuoteIdentifier(old), pq.QuoteIdentifier(new), + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", old, new, err) + } + } + for old, new := range renameIndicesMappings { + var query string + if err := tx.QueryRowContext( + ctx, "SELECT sql FROM sqlite_schema WHERE type = 'index' AND name = $1;", old, + ).Scan(&query); err != nil { + if err == sql.ErrNoRows { + continue + } + return err + } + query = strings.Replace(query, old, new, 1) + if _, err := tx.ExecContext(ctx, "DROP INDEX %s;", pq.QuoteIdentifier(old)); err != nil { + return fmt.Errorf("drop index %q to %q error: %w", old, new, err) + } + if _, err := tx.ExecContext(ctx, query); err != nil { + return fmt.Errorf("recreate index %q to %q error: %w", old, new, err) } } return nil @@ -29,8 +73,40 @@ func UpRenameTables(ctx context.Context, tx *sql.Tx) error { func DownRenameTables(ctx context.Context, tx *sql.Tx) error { for old, new := range renameTableMappings { - if _, err := tx.ExecContext(ctx, "ALTER TABLE $1 RENAME TO $2;", new, old); err != nil { - return fmt.Errorf("rename %q to %q error: %w", new, old, err) + // SQLite has no "IF EXISTS" so check if the table exists. + var name string + if err := tx.QueryRowContext( + ctx, "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = $1;", new, + ).Scan(&name); err != nil { + if err == sql.ErrNoRows { + continue + } + return err + } + q := fmt.Sprintf( + "ALTER TABLE %s RENAME TO %s;", + pq.QuoteIdentifier(new), pq.QuoteIdentifier(old), + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", new, old, err) + } + } + for old, new := range renameIndicesMappings { + var query string + if err := tx.QueryRowContext( + ctx, "SELECT sql FROM sqlite_schema WHERE type = 'index' AND name = $1;", new, + ).Scan(&query); err != nil { + if err == sql.ErrNoRows { + continue + } + return err + } + query = strings.Replace(query, new, old, 1) + if _, err := tx.ExecContext(ctx, "DROP INDEX %s;", pq.QuoteIdentifier(new)); err != nil { + return fmt.Errorf("drop index %q to %q error: %w", new, old, err) + } + if _, err := tx.ExecContext(ctx, query); err != nil { + return fmt.Errorf("recreate index %q to %q error: %w", new, old, err) } } return nil diff --git a/userapi/storage/sqlite3/stats_table.go b/userapi/storage/sqlite3/stats_table.go index 8aa1746c5..35e3c653e 100644 --- a/userapi/storage/sqlite3/stats_table.go +++ b/userapi/storage/sqlite3/stats_table.go @@ -46,7 +46,7 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON usera const countUsersLastSeenAfterSQL = "" + "SELECT COUNT(*) FROM (" + - " SELECT localpart FROM device_devices WHERE last_seen_ts > $1 " + + " SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " + " GROUP BY localpart" + " ) u" @@ -63,7 +63,7 @@ R30Users counts the number of 30 day retained users, defined as: const countR30UsersSQL = ` SELECT platform, COUNT(*) FROM ( SELECT users.localpart, platform, users.created_ts, MAX(uip.last_seen_ts) - FROM account_accounts users + FROM userapi_accounts users INNER JOIN (SELECT localpart, last_seen_ts, @@ -76,7 +76,7 @@ SELECT platform, COUNT(*) FROM ( ELSE 'unknown' END AS platform - FROM device_devices + FROM userapi_devices ) uip ON users.localpart = uip.localpart AND users.account_type <> 4 @@ -126,7 +126,7 @@ GROUP BY client_type ` const countUserByAccountTypeSQL = ` -SELECT COUNT(*) FROM account_accounts WHERE account_type IN ($1) +SELECT COUNT(*) FROM userapi_accounts WHERE account_type IN ($1) ` // $1 = Guest AccountType @@ -139,7 +139,7 @@ SELECT user_type, COUNT(*) AS count FROM ( WHEN account_type = $4 AND appservice_id IS NULL THEN 'guest' WHEN account_type IN ($5) AND appservice_id IS NOT NULL THEN 'bridged' END AS user_type - FROM account_accounts + FROM userapi_accounts WHERE created_ts > $8 ) AS t GROUP BY user_type ` @@ -148,14 +148,14 @@ SELECT user_type, COUNT(*) AS count FROM ( const updateUserDailyVisitsSQL = ` INSERT INTO userapi_daily_visits(localpart, device_id, timestamp, user_agent) SELECT u.localpart, u.device_id, $1, MAX(u.user_agent) - FROM device_devices AS u + FROM userapi_devices AS u LEFT JOIN ( SELECT localpart, device_id, timestamp FROM userapi_daily_visits WHERE timestamp = $1 ) udv ON u.localpart = udv.localpart AND u.device_id = udv.device_id - INNER JOIN device_devices d ON d.localpart = u.localpart - INNER JOIN account_accounts a ON a.localpart = u.localpart + INNER JOIN userapi_devices d ON d.localpart = u.localpart + INNER JOIN userapi_accounts a ON a.localpart = u.localpart WHERE $2 <= d.last_seen_ts AND d.last_seen_ts < $3 AND a.account_type in (1, 3) GROUP BY u.localpart, u.device_id