From 3a3b252b31861ed82b4bf1e34589d1813e7dfbdf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 13 Oct 2022 13:07:37 +0100 Subject: [PATCH] Track child event types, more updates --- syncapi/storage/postgres/relations_table.go | 68 ++++++++------------ syncapi/storage/shared/storage_consumer.go | 2 +- syncapi/storage/shared/storage_sync.go | 5 +- syncapi/storage/sqlite3/relations_table.go | 70 ++++++++------------- syncapi/storage/tables/interface.go | 4 +- syncapi/storage/tables/relations_test.go | 7 ++- 6 files changed, 59 insertions(+), 97 deletions(-) diff --git a/syncapi/storage/postgres/relations_table.go b/syncapi/storage/postgres/relations_table.go index aead4bb46..5a76e9c33 100644 --- a/syncapi/storage/postgres/relations_table.go +++ b/syncapi/storage/postgres/relations_table.go @@ -32,6 +32,7 @@ CREATE TABLE IF NOT EXISTS syncapi_relations ( room_id TEXT NOT NULL, event_id TEXT NOT NULL, child_event_id TEXT NOT NULL, + child_event_type TEXT NOT NULL, rel_type TEXT NOT NULL, CONSTRAINT syncapi_relations_unique UNIQUE (room_id, event_id, child_event_id, rel_type) ); @@ -39,8 +40,8 @@ CREATE TABLE IF NOT EXISTS syncapi_relations ( const insertRelationSQL = "" + "INSERT INTO syncapi_relations (" + - " room_id, event_id, child_event_id, rel_type" + - ") VALUES ($1, $2, $3, $4) " + + " room_id, event_id, child_event_id, child_event_type, rel_type" + + ") VALUES ($1, $2, $3, $4, $5) " + " ON CONFLICT DO NOTHING" const deleteRelationSQL = "" + @@ -48,35 +49,29 @@ const deleteRelationSQL = "" + const selectRelationsInRangeAscSQL = "" + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + - " WHERE room_id = $1 AND event_id = $2 AND id > $3 AND id <= $4" + - " ORDER BY id ASC LIMIT $5" + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id > $5 AND id <= $6" + + " ORDER BY id ASC LIMIT $7" const selectRelationsInRangeDescSQL = "" + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + - " WHERE room_id = $1 AND event_id = $2 AND id >= $3 AND id < $4" + - " ORDER BY id DESC LIMIT $5" - -const selectRelationsByTypeInRangeAscSQL = "" + - "SELECT id, child_event_id, rel_type FROM syncapi_relations" + - " WHERE room_id = $1 AND event_id = $2 AND rel_type = $3 AND id > $4 AND id <= $5" + - " ORDER BY id ASC LIMIT $6" - -const selectRelationsByTypeInRangeDescSQL = "" + - "SELECT id, child_event_id, rel_type FROM syncapi_relations" + - " WHERE room_id = $1 AND event_id = $2 AND rel_type = $3 AND id >= $4 AND id < $5" + - " ORDER BY id DESC LIMIT $6" + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id >= $5 AND id < $6" + + " ORDER BY id DESC LIMIT $7" const selectMaxRelationIDSQL = "" + "SELECT COALESCE(MAX(id), 0) FROM syncapi_relations" type relationsStatements struct { - insertRelationStmt *sql.Stmt - selectRelationsInRangeAscStmt *sql.Stmt - selectRelationsInRangeDescStmt *sql.Stmt - selectRelationsByTypeInRangeAscStmt *sql.Stmt - selectRelationsByTypeInRangeDescStmt *sql.Stmt - deleteRelationStmt *sql.Stmt - selectMaxRelationIDStmt *sql.Stmt + insertRelationStmt *sql.Stmt + selectRelationsInRangeAscStmt *sql.Stmt + selectRelationsInRangeDescStmt *sql.Stmt + deleteRelationStmt *sql.Stmt + selectMaxRelationIDStmt *sql.Stmt } func NewPostgresRelationsTable(db *sql.DB) (tables.Relations, error) { @@ -89,18 +84,16 @@ func NewPostgresRelationsTable(db *sql.DB) (tables.Relations, error) { {&s.insertRelationStmt, insertRelationSQL}, {&s.selectRelationsInRangeAscStmt, selectRelationsInRangeAscSQL}, {&s.selectRelationsInRangeDescStmt, selectRelationsInRangeDescSQL}, - {&s.selectRelationsByTypeInRangeAscStmt, selectRelationsByTypeInRangeAscSQL}, - {&s.selectRelationsByTypeInRangeDescStmt, selectRelationsByTypeInRangeDescSQL}, {&s.deleteRelationStmt, deleteRelationSQL}, {&s.selectMaxRelationIDStmt, selectMaxRelationIDSQL}, }.Prepare(db) } func (s *relationsStatements) InsertRelation( - ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, relType string, + ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string, ) (err error) { _, err = sqlutil.TxStmt(txn, s.insertRelationStmt).ExecContext( - ctx, roomID, eventID, childEventID, relType, + ctx, roomID, eventID, childEventID, childEventType, relType, ) return } @@ -117,28 +110,17 @@ func (s *relationsStatements) DeleteRelation( // SelectRelationsInRange returns a map rel_type -> []child_event_id func (s *relationsStatements) SelectRelationsInRange( - ctx context.Context, txn *sql.Tx, roomID, eventID, relType string, + ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, r types.Range, limit int, ) (map[string][]types.RelationEntry, types.StreamPosition, error) { var lastPos types.StreamPosition var stmt *sql.Stmt - var rows *sql.Rows - var err error - if relType != "" { - if r.Backwards { - stmt = sqlutil.TxStmt(txn, s.selectRelationsByTypeInRangeDescStmt) - } else { - stmt = sqlutil.TxStmt(txn, s.selectRelationsByTypeInRangeAscStmt) - } - rows, err = stmt.QueryContext(ctx, roomID, eventID, relType, r.Low(), r.High(), limit) + if r.Backwards { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt) } else { - if r.Backwards { - stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt) - } else { - stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt) - } - rows, err = stmt.QueryContext(ctx, roomID, eventID, r.Low(), r.High(), limit) + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt) } + rows, err := stmt.QueryContext(ctx, roomID, eventID, relType, eventType, r.Low(), r.High(), limit) if err != nil { return nil, lastPos, err } diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index 0a83b18e0..bf12203db 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -606,7 +606,7 @@ func (d *Database) UpdateRelations(ctx context.Context, event *gomatrixserverlib return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { return d.Relations.InsertRelation( ctx, txn, event.RoomID(), content.Relations.EventID, - event.EventID(), content.Relations.RelationType, + event.EventID(), event.Type(), content.Relations.RelationType, ) }) } diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index dfd2edcd9..fdd363932 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -626,7 +626,7 @@ func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID, // First look up any relations from the database. We add one to the limit here // so that we can tell if we're overflowing, as we will only set the "next_batch" // in the response if we are. - relations, _, err := d.Relations.SelectRelationsInRange(ctx, d.txn, roomID, eventID, relType, r, limit+1) + relations, _, err := d.Relations.SelectRelationsInRange(ctx, d.txn, roomID, eventID, relType, eventType, r, limit+1) if err != nil { return nil, "", "", fmt.Errorf("d.Relations.SelectRelationsInRange: %w", err) } @@ -672,9 +672,6 @@ func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID, // type if it was specified. clientEvents = make([]gomatrixserverlib.ClientEvent, 0, len(events)) for _, event := range events { - if eventType != "" && event.Type() != eventType { - continue - } clientEvents = append( clientEvents, gomatrixserverlib.ToClientEvent(event.Event, gomatrixserverlib.FormatAll), diff --git a/syncapi/storage/sqlite3/relations_table.go b/syncapi/storage/sqlite3/relations_table.go index 6791d94da..7cbb5408f 100644 --- a/syncapi/storage/sqlite3/relations_table.go +++ b/syncapi/storage/sqlite3/relations_table.go @@ -30,6 +30,7 @@ CREATE TABLE IF NOT EXISTS syncapi_relations ( room_id TEXT NOT NULL, event_id TEXT NOT NULL, child_event_id TEXT NOT NULL, + child_event_type TEXT NOT NULL, rel_type TEXT NOT NULL, UNIQUE (room_id, event_id, child_event_id, rel_type) ); @@ -37,8 +38,8 @@ CREATE TABLE IF NOT EXISTS syncapi_relations ( const insertRelationSQL = "" + "INSERT INTO syncapi_relations (" + - " id, room_id, event_id, child_event_id, rel_type" + - ") VALUES ($1, $2, $3, $4, $5) " + + " id, room_id, event_id, child_event_id, child_event_type, rel_type" + + ") VALUES ($1, $2, $3, $4, $5, $6) " + " ON CONFLICT DO NOTHING" const deleteRelationSQL = "" + @@ -46,36 +47,30 @@ const deleteRelationSQL = "" + const selectRelationsInRangeAscSQL = "" + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + - " WHERE room_id = $1 AND event_id = $2 AND id > $3 AND id <= $4" + - " ORDER BY id ASC LIMIT $5" + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id > $5 AND id <= $6" + + " ORDER BY id ASC LIMIT $7" const selectRelationsInRangeDescSQL = "" + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + - " WHERE room_id = $1 AND event_id = $2 AND id >= $3 AND id < $4" + - " ORDER BY id DESC LIMIT $5" - -const selectRelationsByTypeInRangeAscSQL = "" + - "SELECT id, child_event_id, rel_type FROM syncapi_relations" + - " WHERE room_id = $1 AND event_id = $2 AND rel_type = $3 AND id > $4 AND id <= $5" + - " ORDER BY id ASC LIMIT $6" - -const selectRelationsByTypeInRangeDescSQL = "" + - "SELECT id, child_event_id, rel_type FROM syncapi_relations" + - " WHERE room_id = $1 AND event_id = $2 AND rel_type = $3 AND id >= $4 AND id < $5" + - " ORDER BY id DESC LIMIT $6" + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id >= $5 AND id < $6" + + " ORDER BY id DESC LIMIT $7" const selectMaxRelationIDSQL = "" + "SELECT COALESCE(MAX(id), 0) FROM syncapi_relations" type relationsStatements struct { - streamIDStatements *StreamIDStatements - insertRelationStmt *sql.Stmt - selectRelationsInRangeAscStmt *sql.Stmt - selectRelationsInRangeDescStmt *sql.Stmt - selectRelationsByTypeInRangeAscStmt *sql.Stmt - selectRelationsByTypeInRangeDescStmt *sql.Stmt - deleteRelationStmt *sql.Stmt - selectMaxRelationIDStmt *sql.Stmt + streamIDStatements *StreamIDStatements + insertRelationStmt *sql.Stmt + selectRelationsInRangeAscStmt *sql.Stmt + selectRelationsInRangeDescStmt *sql.Stmt + deleteRelationStmt *sql.Stmt + selectMaxRelationIDStmt *sql.Stmt } func NewSqliteRelationsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Relations, error) { @@ -90,22 +85,20 @@ func NewSqliteRelationsTable(db *sql.DB, streamID *StreamIDStatements) (tables.R {&s.insertRelationStmt, insertRelationSQL}, {&s.selectRelationsInRangeAscStmt, selectRelationsInRangeAscSQL}, {&s.selectRelationsInRangeDescStmt, selectRelationsInRangeDescSQL}, - {&s.selectRelationsByTypeInRangeAscStmt, selectRelationsByTypeInRangeAscSQL}, - {&s.selectRelationsByTypeInRangeDescStmt, selectRelationsByTypeInRangeDescSQL}, {&s.deleteRelationStmt, deleteRelationSQL}, {&s.selectMaxRelationIDStmt, selectMaxRelationIDSQL}, }.Prepare(db) } func (s *relationsStatements) InsertRelation( - ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, relType string, + ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string, ) (err error) { var streamPos types.StreamPosition if streamPos, err = s.streamIDStatements.nextRelationID(ctx, txn); err != nil { return } _, err = sqlutil.TxStmt(txn, s.insertRelationStmt).ExecContext( - ctx, streamPos, roomID, eventID, childEventID, relType, + ctx, streamPos, roomID, eventID, childEventID, childEventType, relType, ) return } @@ -122,28 +115,17 @@ func (s *relationsStatements) DeleteRelation( // SelectRelationsInRange returns a map rel_type -> []child_event_id func (s *relationsStatements) SelectRelationsInRange( - ctx context.Context, txn *sql.Tx, roomID, eventID, relType string, + ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, r types.Range, limit int, ) (map[string][]types.RelationEntry, types.StreamPosition, error) { var lastPos types.StreamPosition var stmt *sql.Stmt - var rows *sql.Rows - var err error - if relType != "" { - if r.Backwards { - stmt = sqlutil.TxStmt(txn, s.selectRelationsByTypeInRangeDescStmt) - } else { - stmt = sqlutil.TxStmt(txn, s.selectRelationsByTypeInRangeAscStmt) - } - rows, err = stmt.QueryContext(ctx, roomID, eventID, relType, r.Low(), r.High(), limit) + if r.Backwards { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt) } else { - if r.Backwards { - stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt) - } else { - stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt) - } - rows, err = stmt.QueryContext(ctx, roomID, eventID, r.Low(), r.High(), limit) + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt) } + rows, err := stmt.QueryContext(ctx, roomID, eventID, relType, eventType, r.Low(), r.High(), limit) if err != nil { return nil, lastPos, err } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 1dba56875..e48c050dd 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -210,7 +210,7 @@ type Presence interface { type Relations interface { // Inserts a relation which refers from the child event ID to the event ID in the given room. // If the relation already exists then this function will do nothing and return no error. - InsertRelation(ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, relType string) (err error) + InsertRelation(ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string) (err error) // Deletes a relation which already exists as the result of an event redaction. If the relation // does not exist then this function will do nothing and return no error. DeleteRelation(ctx context.Context, txn *sql.Tx, roomID, childEventID string) error @@ -219,7 +219,7 @@ type Relations interface { // contain relations of that type, otherwise if "" is specified then all relations in the range // will be returned, inclusive of the "to" position but excluding the "from" position. The stream // position returned is the maximum position of the returned results. - SelectRelationsInRange(ctx context.Context, txn *sql.Tx, roomID, eventID, relType string, r types.Range, limit int) (map[string][]types.RelationEntry, types.StreamPosition, error) + SelectRelationsInRange(ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, r types.Range, limit int) (map[string][]types.RelationEntry, types.StreamPosition, error) // SelectMaxRelationID returns the maximum ID of all relations, used to determine what the boundaries // should be if there are no boundaries supplied (i.e. we want to work backwards but don't have a // "from" or want to work forwards and don't have a "to"). diff --git a/syncapi/storage/tables/relations_test.go b/syncapi/storage/tables/relations_test.go index 12de799cb..46270e36d 100644 --- a/syncapi/storage/tables/relations_test.go +++ b/syncapi/storage/tables/relations_test.go @@ -43,7 +43,7 @@ func newRelationsTable(t *testing.T, dbType test.DBType) (tables.Relations, *sql func compareRelationsToExpected(t *testing.T, tab tables.Relations, r types.Range, expected []types.RelationEntry) { ctx := context.Background() - relations, _, err := tab.SelectRelationsInRange(ctx, nil, roomID, "a", "", r, 50) + relations, _, err := tab.SelectRelationsInRange(ctx, nil, roomID, "a", "", "", r, 50) if err != nil { t.Fatal(err) } @@ -60,6 +60,7 @@ func compareRelationsToExpected(t *testing.T, tab tables.Relations, r types.Rang } const roomID = "!roomid:server" +const childType = "m.room.something" const relType = "m.reaction" func TestRelationsTable(t *testing.T) { @@ -70,7 +71,7 @@ func TestRelationsTable(t *testing.T) { // Insert some relations for _, child := range []string{"b", "c", "d"} { - if err := tab.InsertRelation(ctx, nil, roomID, "a", child, relType); err != nil { + if err := tab.InsertRelation(ctx, nil, roomID, "a", child, childType, relType); err != nil { t.Fatal(err) } } @@ -138,7 +139,7 @@ func TestRelationsTable(t *testing.T) { // Insert some new relations for _, child := range []string{"e", "f", "g", "h"} { - if err := tab.InsertRelation(ctx, nil, roomID, "a", child, relType); err != nil { + if err := tab.InsertRelation(ctx, nil, roomID, "a", child, childType, relType); err != nil { t.Fatal(err) } }