From db192382c60ed81346bf2ad5a971fb9b7ed408f7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 27 Jan 2020 14:33:06 +0000 Subject: [PATCH] Some keydb, accountdb, devicedb, common partition fixes, some more syncapi tweaking --- .../storage/accounts/sqlite3/accounts_table.go | 4 +--- .../auth/storage/accounts/sqlite3/filter_table.go | 8 +++++--- .../storage/accounts/sqlite3/membership_table.go | 9 ++++----- .../auth/storage/accounts/sqlite3/storage.go | 4 ++-- .../auth/storage/devices/sqlite3/devices_table.go | 15 +++++++-------- clientapi/auth/storage/devices/sqlite3/storage.go | 4 +++- common/keydb/sqlite3/server_key_table.go | 2 +- common/partition_offset_table.go | 2 +- syncapi/storage/sqlite3/account_data_table.go | 14 +++++++------- .../storage/sqlite3/current_room_state_table.go | 14 +++++++------- syncapi/storage/sqlite3/invites_table.go | 2 +- .../storage/sqlite3/output_room_events_table.go | 2 +- 12 files changed, 40 insertions(+), 40 deletions(-) diff --git a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go index cec3f18db..b029951f1 100644 --- a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go @@ -40,8 +40,6 @@ CREATE TABLE IF NOT EXISTS account_accounts ( -- TODO: -- is_guest, is_admin, upgraded_ts, devices, any email reset stuff? ); --- Create sequence for autogenerated numeric usernames -CREATE SEQUENCE IF NOT EXISTS numeric_username_seq START 1; ` const insertAccountSQL = "" + @@ -54,7 +52,7 @@ const selectPasswordHashSQL = "" + "SELECT password_hash FROM account_accounts WHERE localpart = $1" const selectNewNumericLocalpartSQL = "" + - "SELECT nextval('numeric_username_seq')" + "SELECT COUNT(localpart) FROM account_accounts" // TODO: Update password diff --git a/clientapi/auth/storage/accounts/sqlite3/filter_table.go b/clientapi/auth/storage/accounts/sqlite3/filter_table.go index 67d1f2ba2..a6f4f7bb7 100644 --- a/clientapi/auth/storage/accounts/sqlite3/filter_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/filter_table.go @@ -28,11 +28,13 @@ CREATE TABLE IF NOT EXISTS account_filter ( -- The filter filter TEXT NOT NULL, -- The ID - id SERIAL UNIQUE, + id SERIAL, -- The localpart of the Matrix user ID associated to this filter localpart TEXT NOT NULL, - PRIMARY KEY(id, localpart) + PRIMARY KEY(id, localpart), + + UNIQUE (id) ); CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart); @@ -45,7 +47,7 @@ const selectFilterIDByContentSQL = "" + "SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2" const insertFilterSQL = "" + - "INSERT INTO account_filter (filter, id, localpart) VALUES ($1, DEFAULT, $2) RETURNING id" + "INSERT INTO account_filter (filter, localpart) VALUES ($1, $2)" type filterStatements struct { selectFilterStmt *sql.Stmt diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index 203e228eb..8e5e69bad 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -33,11 +33,10 @@ CREATE TABLE IF NOT EXISTS account_memberships ( event_id TEXT NOT NULL, -- A user can only be member of a room once - PRIMARY KEY (localpart, room_id) -); + PRIMARY KEY (localpart, room_id), --- Use index to process deletion by ID more efficiently -CREATE UNIQUE INDEX IF NOT EXISTS account_membership_event_id ON account_memberships(event_id); + UNIQUE (event_id) +); ` const insertMembershipSQL = ` @@ -52,7 +51,7 @@ const selectMembershipInRoomByLocalpartSQL = "" + "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" const deleteMembershipsByEventIDsSQL = "" + - "DELETE FROM account_memberships WHERE event_id = ANY($1)" + "DELETE FROM account_memberships WHERE event_id IN ($1)" type membershipStatements struct { deleteMembershipsByEventIDsStmt *sql.Stmt diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index fde1269a5..656c7d39a 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -25,7 +25,7 @@ import ( "golang.org/x/crypto/bcrypt" // Import the postgres database driver. - _ "github.com/lib/pq" + _ "github.com/mattn/go-sqlite3" ) // Database represents an account database @@ -45,7 +45,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { + if db, err = sql.Open("sqlite3", dataSourceName); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/clientapi/auth/storage/devices/sqlite3/devices_table.go index 7d080edd4..b12a0b869 100644 --- a/clientapi/auth/storage/devices/sqlite3/devices_table.go +++ b/clientapi/auth/storage/devices/sqlite3/devices_table.go @@ -28,7 +28,7 @@ import ( const devicesSchema = ` -- This sequence is used for automatic allocation of session_id. -CREATE SEQUENCE IF NOT EXISTS device_session_id_seq START 1; +-- CREATE SEQUENCE IF NOT EXISTS device_session_id_seq START 1; -- Stores data about devices. CREATE TABLE IF NOT EXISTS device_devices ( @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS device_devices ( -- This can be used as a secure substitution of the access token in situations -- where data is associated with access tokens (e.g. transaction storage), -- so we don't have to store users' access tokens everywhere. - session_id BIGINT NOT NULL DEFAULT nextval('device_session_id_seq'), + session_id BIGINT, -- The device identifier. This only needs to uniquely identify a device for a given user, not globally. -- access_tokens will be clobbered based on the device ID for a user. device_id TEXT NOT NULL, @@ -50,17 +50,16 @@ CREATE TABLE IF NOT EXISTS device_devices ( -- When this devices was first recognised on the network, as a unix timestamp (ms resolution). created_ts BIGINT NOT NULL, -- The display name, human friendlier than device_id and updatable - display_name TEXT + display_name TEXT, -- TODO: device keys, device display names, last used ts and IP address?, token restrictions (if 3rd-party OAuth app) -); --- Device IDs must be unique for a given user. -CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(localpart, device_id); + UNIQUE (localpart, device_id) +); ` const insertDeviceSQL = "" + - "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" + - " RETURNING session_id" + "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5);" + + "SELECT last_insert_rowid() AS session_id" const selectDeviceByTokenSQL = "" + "SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1" diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index acd2c355a..3141617c0 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -23,6 +23,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" + + _ "github.com/mattn/go-sqlite3" ) // The length of generated device IDs @@ -38,7 +40,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { + if db, err = sql.Open("sqlite3", dataSourceName); err != nil { return nil, err } d := devicesStatements{} diff --git a/common/keydb/sqlite3/server_key_table.go b/common/keydb/sqlite3/server_key_table.go index afca92cd5..6c33f30a0 100644 --- a/common/keydb/sqlite3/server_key_table.go +++ b/common/keydb/sqlite3/server_key_table.go @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS keydb_server_keys ( expired_ts BIGINT NOT NULL, -- The base64-encoded public key. server_key TEXT NOT NULL, - CONSTRAINT UNIQUE (server_name, server_key_id) + UNIQUE (server_name, server_key_id) ); CREATE INDEX IF NOT EXISTS keydb_server_name_and_key_id ON keydb_server_keys (server_name_and_key_id); diff --git a/common/partition_offset_table.go b/common/partition_offset_table.go index fa2259ade..d60971239 100644 --- a/common/partition_offset_table.go +++ b/common/partition_offset_table.go @@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS ${prefix}_partition_offsets ( partition INTEGER NOT NULL, -- The 64-bit offset. partition_offset BIGINT NOT NULL, - CONSTRAINT UNIQUE (topic, partition) + UNIQUE (topic, partition) ); ` diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index b6fcaec86..7d9f882bb 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -27,11 +27,11 @@ import ( const accountDataSchema = ` CREATE TABLE IF NOT EXISTS syncapi_account_data_type ( - id BIGINT PRIMARY KEY AUTOINCREMENT, -- DEFAULT nextval('syncapi_stream_id'), + id INTEGER PRIMARY KEY AUTOINCREMENT, user_id TEXT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, - CONSTRAINT syncapi_account_data_unique UNIQUE (user_id, room_id, type) + UNIQUE (user_id, room_id, type) ); -- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_account_data_id_idx ON syncapi_account_data_type(id, type); @@ -39,15 +39,15 @@ CREATE TABLE IF NOT EXISTS syncapi_account_data_type ( const insertAccountDataSQL = "" + "INSERT INTO syncapi_account_data_type (user_id, room_id, type) VALUES ($1, $2, $3)" + - " ON CONFLICT DO UPDATE" + - " SET id = EXCLUDED.id" + - " RETURNING id" + " ON CONFLICT (user_id, room_id, type) DO UPDATE" + + " SET id = EXCLUDED.id;" + + "SELECT id FROM syncapi_account_data_type WHERE rowid = last_insert_rowid()" const selectAccountDataInRangeSQL = "" + "SELECT room_id, type FROM syncapi_account_data_type" + " WHERE user_id = $1 AND id > $2 AND id <= $3" + - " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + - " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + + // " AND ( $4 IS NULL OR type LIKE ANY($4) )" + + // " AND ( $5 IS NULL OR NOT(type LIKE ANY($5)) )" + " ORDER BY id ASC LIMIT $6" const selectMaxAccountDataIDSQL = "" + diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 80056528f..26f227738 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( -- part of the current state of the room. added_at BIGINT, -- Clobber based on 3-uple of room_id, type and state_key - CONSTRAINT syncapi_room_state_unique UNIQUE (room_id, type, state_key) + UNIQUE (room_id, type, state_key) ); -- for event deletion -- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender, contains_url); @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( const upsertRoomStateSQL = "" + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + - " ON CONFLICT" + // ON CONSTRAINT syncapi_room_state_unique" + + " ON CONFLICT (event_id, room_id, type, sender, contains_url)" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + @@ -73,11 +73,11 @@ const selectRoomIDsWithMembershipSQL = "" + const selectCurrentStateSQL = "" + "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + - " AND ( $2::text[] IS NULL OR sender = ANY($2) )" + - " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + - " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + - " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + - " AND ( $6::bool IS NULL OR contains_url = $6 )" + + " AND ( $2 IS NULL OR sender = ANY($2) )" + + " AND ( $3 IS NULL OR NOT(sender = ANY($3)) )" + + " AND ( $4 IS NULL OR type LIKE ANY($4) )" + + " AND ( $5 IS NULL OR NOT(type LIKE ANY($5)) )" + + " AND ( $6 IS NULL OR contains_url = $6 )" + " LIMIT $7" const selectJoinedUsersSQL = "" + diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 86ff0a98a..ee95aef01 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -26,7 +26,7 @@ import ( const inviteEventsSchema = ` CREATE TABLE IF NOT EXISTS syncapi_invite_events ( - id BIGINT PRIMARY KEY DEFAULT AUTOINCREMENT, -- nextval('syncapi_stream_id'), + id INTEGER PRIMARY KEY DEFAULT AUTOINCREMENT, -- nextval('syncapi_stream_id'), event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 3a3ac8fe9..f30e176e0 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( -- An incrementing ID which denotes the position in the log that this event resides at. -- NB: 'serial' makes no guarantees to increment by 1 every time, only that it increments. -- This isn't a problem for us since we just want to order by this field. - id BIGINT PRIMARY KEY AUTOINCREMENT, -- DEFAULT nextval('syncapi_stream_id'), + id INTEGER PRIMARY KEY AUTOINCREMENT, -- DEFAULT nextval('syncapi_stream_id'), -- The event ID for the event event_id TEXT NOT NULL UNIQUE, -- CONSTRAINT syncapi_event_id_idx UNIQUE, -- The 'room_id' key for the event.