Some keydb, accountdb, devicedb, common partition fixes, some more syncapi tweaking

clientapi/auth/storage/accounts/sqlite3/accounts_table.go

@@ -40,8 +40,6 @@ CREATE TABLE IF NOT EXISTS account_accounts (
     -- TODO:
     -- is_guest, is_admin, upgraded_ts, devices, any email reset stuff?
 );
--- Create sequence for autogenerated numeric usernames
-CREATE SEQUENCE IF NOT EXISTS numeric_username_seq START 1;
 `
 
 const insertAccountSQL = "" +
@@ -54,7 +52,7 @@ const selectPasswordHashSQL = "" +
 	"SELECT password_hash FROM account_accounts WHERE localpart = $1"
 
 const selectNewNumericLocalpartSQL = "" +
-	"SELECT nextval('numeric_username_seq')"
+	"SELECT COUNT(localpart) FROM account_accounts"
 
 // TODO: Update password
 

clientapi/auth/storage/accounts/sqlite3/filter_table.go

@@ -28,11 +28,13 @@ CREATE TABLE IF NOT EXISTS account_filter (
 	-- The filter
 	filter TEXT NOT NULL,
 	-- The ID
-	id SERIAL UNIQUE,
+	id SERIAL,
 	-- The localpart of the Matrix user ID associated to this filter
 	localpart TEXT NOT NULL,
 
-	PRIMARY KEY(id, localpart)
+	PRIMARY KEY(id, localpart),
+
+	UNIQUE (id)
 );
 
 CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart);
@@ -45,7 +47,7 @@ const selectFilterIDByContentSQL = "" +
 	"SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2"
 
 const insertFilterSQL = "" +
-	"INSERT INTO account_filter (filter, id, localpart) VALUES ($1, DEFAULT, $2) RETURNING id"
+	"INSERT INTO account_filter (filter, localpart) VALUES ($1, $2)"
 
 type filterStatements struct {
 	selectFilterStmt            *sql.Stmt

clientapi/auth/storage/accounts/sqlite3/membership_table.go

@@ -33,11 +33,10 @@ CREATE TABLE IF NOT EXISTS account_memberships (
     event_id TEXT NOT NULL,
 
     -- A user can only be member of a room once
-    PRIMARY KEY (localpart, room_id)
+    PRIMARY KEY (localpart, room_id),
-);
 
--- Use index to process deletion by ID more efficiently
+		UNIQUE (event_id)
-CREATE UNIQUE INDEX IF NOT EXISTS account_membership_event_id ON account_memberships(event_id);
+);
 `
 
 const insertMembershipSQL = `
@@ -52,7 +51,7 @@ const selectMembershipInRoomByLocalpartSQL = "" +
 	"SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2"
 
 const deleteMembershipsByEventIDsSQL = "" +
-	"DELETE FROM account_memberships WHERE event_id = ANY($1)"
+	"DELETE FROM account_memberships WHERE event_id IN ($1)"
 
 type membershipStatements struct {
 	deleteMembershipsByEventIDsStmt       *sql.Stmt

clientapi/auth/storage/accounts/sqlite3/storage.go

@@ -25,7 +25,7 @@ import (
 	"golang.org/x/crypto/bcrypt"
 
 	// Import the postgres database driver.
-	_ "github.com/lib/pq"
+	_ "github.com/mattn/go-sqlite3"
 )
 
 // Database represents an account database
@@ -45,7 +45,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if db, err = sql.Open("sqlite3", dataSourceName); err != nil {
 		return nil, err
 	}
 	partitions := common.PartitionOffsetStatements{}

clientapi/auth/storage/devices/sqlite3/devices_table.go

@@ -28,7 +28,7 @@ import (
 
 const devicesSchema = `
 -- This sequence is used for automatic allocation of session_id.
-CREATE SEQUENCE IF NOT EXISTS device_session_id_seq START 1;
+-- CREATE SEQUENCE IF NOT EXISTS device_session_id_seq START 1;
 
 -- Stores data about devices.
 CREATE TABLE IF NOT EXISTS device_devices (
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS device_devices (
     -- This can be used as a secure substitution of the access token in situations
     -- where data is associated with access tokens (e.g. transaction storage),
     -- so we don't have to store users' access tokens everywhere.
-    session_id BIGINT NOT NULL DEFAULT nextval('device_session_id_seq'),
+    session_id BIGINT,
     -- The device identifier. This only needs to uniquely identify a device for a given user, not globally.
     -- access_tokens will be clobbered based on the device ID for a user.
     device_id TEXT NOT NULL,
@@ -50,17 +50,16 @@ CREATE TABLE IF NOT EXISTS device_devices (
     -- When this devices was first recognised on the network, as a unix timestamp (ms resolution).
     created_ts BIGINT NOT NULL,
     -- The display name, human friendlier than device_id and updatable
-    display_name TEXT
+    display_name TEXT,
     -- TODO: device keys, device display names, last used ts and IP address?, token restrictions (if 3rd-party OAuth app)
-);
 
--- Device IDs must be unique for a given user.
+		UNIQUE (localpart, device_id)
-CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(localpart, device_id);
+);
 `
 
 const insertDeviceSQL = "" +
-	"INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" +
+	"INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5);" +
-	" RETURNING session_id"
+	"SELECT last_insert_rowid() AS session_id"
 
 const selectDeviceByTokenSQL = "" +
 	"SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1"

clientapi/auth/storage/devices/sqlite3/storage.go

@@ -23,6 +23,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 // The length of generated device IDs
@@ -38,7 +40,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if db, err = sql.Open("sqlite3", dataSourceName); err != nil {
 		return nil, err
 	}
 	d := devicesStatements{}

common/keydb/sqlite3/server_key_table.go

@@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS keydb_server_keys (
 	expired_ts BIGINT NOT NULL,
 	-- The base64-encoded public key.
 	server_key TEXT NOT NULL,
-	CONSTRAINT UNIQUE (server_name, server_key_id)
+	UNIQUE (server_name, server_key_id)
 );
 
 CREATE INDEX IF NOT EXISTS keydb_server_name_and_key_id ON keydb_server_keys (server_name_and_key_id);

common/partition_offset_table.go

@@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS ${prefix}_partition_offsets (
     partition INTEGER NOT NULL,
     -- The 64-bit offset.
     partition_offset BIGINT NOT NULL,
-    CONSTRAINT UNIQUE (topic, partition)
+    UNIQUE (topic, partition)
 );
 `
 

syncapi/storage/sqlite3/account_data_table.go

@@ -27,11 +27,11 @@ import (
 
 const accountDataSchema = `
 CREATE TABLE IF NOT EXISTS syncapi_account_data_type (
-    id BIGINT PRIMARY KEY AUTOINCREMENT, -- DEFAULT nextval('syncapi_stream_id'),
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
     user_id TEXT NOT NULL,
     room_id TEXT NOT NULL,
     type TEXT NOT NULL,
-    CONSTRAINT syncapi_account_data_unique UNIQUE (user_id, room_id, type)
+    UNIQUE (user_id, room_id, type)
 );
 
 -- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_account_data_id_idx ON syncapi_account_data_type(id, type);
@@ -39,15 +39,15 @@ CREATE TABLE IF NOT EXISTS syncapi_account_data_type (
 
 const insertAccountDataSQL = "" +
 	"INSERT INTO syncapi_account_data_type (user_id, room_id, type) VALUES ($1, $2, $3)" +
-	" ON CONFLICT DO UPDATE" +
+	" ON CONFLICT (user_id, room_id, type) DO UPDATE" +
-	" SET id = EXCLUDED.id" +
+	" SET id = EXCLUDED.id;" +
-	" RETURNING id"
+	"SELECT id FROM syncapi_account_data_type WHERE rowid = last_insert_rowid()"
 
 const selectAccountDataInRangeSQL = "" +
 	"SELECT room_id, type FROM syncapi_account_data_type" +
 	" WHERE user_id = $1 AND id > $2 AND id <= $3" +
-	" AND ( $4::text[] IS NULL OR     type LIKE ANY($4)  )" +
+	//	" AND ( $4 IS NULL OR     type LIKE ANY($4)  )" +
-	" AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" +
+	//	" AND ( $5 IS NULL OR NOT(type LIKE ANY($5)) )" +
 	" ORDER BY id ASC LIMIT $6"
 
 const selectMaxAccountDataIDSQL = "" +

syncapi/storage/sqlite3/current_room_state_table.go

@@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state (
     -- part of the current state of the room.
     added_at BIGINT,
     -- Clobber based on 3-uple of room_id, type and state_key
-    CONSTRAINT syncapi_room_state_unique UNIQUE (room_id, type, state_key)
+    UNIQUE (room_id, type, state_key)
 );
 -- for event deletion
 -- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender, contains_url);
@@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state (
 const upsertRoomStateSQL = "" +
 	"INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" +
 	" VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" +
-	" ON CONFLICT" + // ON CONSTRAINT syncapi_room_state_unique" +
+	" ON CONFLICT (event_id, room_id, type, sender, contains_url)" +
 	" DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9"
 
 const deleteRoomStateByEventIDSQL = "" +
@@ -73,11 +73,11 @@ const selectRoomIDsWithMembershipSQL = "" +
 
 const selectCurrentStateSQL = "" +
 	"SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" +
-	" AND ( $2::text[] IS NULL OR     sender  = ANY($2)  )" +
+	" AND ( $2 IS NULL OR     sender  = ANY($2)  )" +
-	" AND ( $3::text[] IS NULL OR NOT(sender  = ANY($3)) )" +
+	" AND ( $3 IS NULL OR NOT(sender  = ANY($3)) )" +
-	" AND ( $4::text[] IS NULL OR     type LIKE ANY($4)  )" +
+	" AND ( $4 IS NULL OR     type LIKE ANY($4)  )" +
-	" AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" +
+	" AND ( $5 IS NULL OR NOT(type LIKE ANY($5)) )" +
-	" AND ( $6::bool IS NULL   OR     contains_url = $6  )" +
+	" AND ( $6 IS NULL OR     contains_url = $6  )" +
 	" LIMIT $7"
 
 const selectJoinedUsersSQL = "" +

syncapi/storage/sqlite3/invites_table.go

@@ -26,7 +26,7 @@ import (
 
 const inviteEventsSchema = `
 CREATE TABLE IF NOT EXISTS syncapi_invite_events (
-	id BIGINT PRIMARY KEY DEFAULT AUTOINCREMENT, -- nextval('syncapi_stream_id'),
+	id INTEGER PRIMARY KEY DEFAULT AUTOINCREMENT, -- nextval('syncapi_stream_id'),
 	event_id TEXT NOT NULL,
 	room_id TEXT NOT NULL,
 	target_user_id TEXT NOT NULL,

syncapi/storage/sqlite3/output_room_events_table.go

@@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
   -- An incrementing ID which denotes the position in the log that this event resides at.
   -- NB: 'serial' makes no guarantees to increment by 1 every time, only that it increments.
   --     This isn't a problem for us since we just want to order by this field.
-  id BIGINT PRIMARY KEY AUTOINCREMENT, -- DEFAULT nextval('syncapi_stream_id'),
+  id INTEGER PRIMARY KEY AUTOINCREMENT, -- DEFAULT nextval('syncapi_stream_id'),
   -- The event ID for the event
   event_id TEXT NOT NULL UNIQUE, -- CONSTRAINT syncapi_event_id_idx UNIQUE,
   -- The 'room_id' key for the event.