From 57be20e7da3cb788a3ef7da13e1660d0fc9e67a6 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 15 Dec 2020 16:28:33 +0000 Subject: [PATCH] Hit the database far less to find room NIDs for event NIDs --- roomserver/storage/postgres/events_table.go | 30 +++++++++++----- roomserver/storage/shared/storage.go | 10 +++--- roomserver/storage/sqlite3/events_table.go | 39 ++++++++++++++++----- roomserver/storage/tables/interface.go | 7 ++-- 4 files changed, 61 insertions(+), 25 deletions(-) diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index c8eb8e2d2..25c5be62c 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -120,8 +120,8 @@ const bulkSelectEventNIDSQL = "" + const selectMaxEventDepthSQL = "" + "SELECT COALESCE(MAX(depth) + 1, 0) FROM roomserver_events WHERE event_nid = ANY($1)" -const selectRoomNIDForEventNIDSQL = "" + - "SELECT room_nid FROM roomserver_events WHERE event_nid = $1" +const selectRoomNIDsForEventNIDsSQL = "" + + "SELECT event_nid, room_nid FROM roomserver_events WHERE event_nid = ANY($1)" type eventStatements struct { insertEventStmt *sql.Stmt @@ -137,7 +137,7 @@ type eventStatements struct { bulkSelectEventIDStmt *sql.Stmt bulkSelectEventNIDStmt *sql.Stmt selectMaxEventDepthStmt *sql.Stmt - selectRoomNIDForEventNIDStmt *sql.Stmt + selectRoomNIDsForEventNIDsStmt *sql.Stmt } func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { @@ -161,7 +161,7 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { {&s.bulkSelectEventIDStmt, bulkSelectEventIDSQL}, {&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL}, {&s.selectMaxEventDepthStmt, selectMaxEventDepthSQL}, - {&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL}, + {&s.selectRoomNIDsForEventNIDsStmt, selectRoomNIDsForEventNIDsSQL}, }.Prepare(db) } @@ -432,11 +432,23 @@ func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, return result, nil } -func (s *eventStatements) SelectRoomNIDForEventNID( - ctx context.Context, eventNID types.EventNID, -) (roomNID types.RoomNID, err error) { - err = s.selectRoomNIDForEventNIDStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) - return +func (s *eventStatements) SelectRoomNIDsForEventNIDs( + ctx context.Context, eventNIDs []types.EventNID, +) (map[types.EventNID]types.RoomNID, error) { + rows, err := s.selectRoomNIDsForEventNIDsStmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs)) + if err != nil { + return nil, err + } + result := make(map[types.EventNID]types.RoomNID) + for rows.Next() { + var eventNID types.EventNID + var roomNID types.RoomNID + if err = rows.Scan(&eventNID, &roomNID); err != nil { + return nil, err + } + result[eventNID] = roomNID + } + return result, nil } func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 83982299a..8301f56d3 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -313,16 +313,18 @@ func (d *Database) Events( if err != nil { eventIDs = map[types.EventNID]string{} } + var roomNIDs map[types.EventNID]types.RoomNID + roomNIDs, err = d.EventsTable.SelectRoomNIDsForEventNIDs(ctx, eventNIDs) + if err != nil { + return nil, err + } results := make([]types.Event, len(eventJSONs)) for i, eventJSON := range eventJSONs { var roomNID types.RoomNID var roomVersion gomatrixserverlib.RoomVersion result := &results[i] result.EventNID = eventJSON.EventNID - roomNID, err = d.EventsTable.SelectRoomNIDForEventNID(ctx, eventJSON.EventNID) - if err != nil { - return nil, err - } + roomNID = roomNIDs[result.EventNID] if roomID, ok := d.Cache.GetRoomServerRoomID(roomNID); ok { roomVersion, _ = d.Cache.GetRoomVersion(roomID) } diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 773e9ade3..3c82c094e 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -95,8 +95,8 @@ const bulkSelectEventNIDSQL = "" + const selectMaxEventDepthSQL = "" + "SELECT COALESCE(MAX(depth) + 1, 0) FROM roomserver_events WHERE event_nid IN ($1)" -const selectRoomNIDForEventNIDSQL = "" + - "SELECT room_nid FROM roomserver_events WHERE event_nid = $1" +const selectRoomNIDsForEventNIDsSQL = "" + + "SELECT room_nid FROM roomserver_events WHERE event_nid IN ($1)" type eventStatements struct { db *sql.DB @@ -112,7 +112,7 @@ type eventStatements struct { bulkSelectEventReferenceStmt *sql.Stmt bulkSelectEventIDStmt *sql.Stmt bulkSelectEventNIDStmt *sql.Stmt - selectRoomNIDForEventNIDStmt *sql.Stmt + //selectRoomNIDsForEventNIDsStmt *sql.Stmt } func NewSqliteEventsTable(db *sql.DB) (tables.Events, error) { @@ -137,7 +137,7 @@ func NewSqliteEventsTable(db *sql.DB) (tables.Events, error) { {&s.bulkSelectEventReferenceStmt, bulkSelectEventReferenceSQL}, {&s.bulkSelectEventIDStmt, bulkSelectEventIDSQL}, {&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL}, - {&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL}, + //{&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL}, }.Prepare(db) } @@ -480,11 +480,32 @@ func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, return result, nil } -func (s *eventStatements) SelectRoomNIDForEventNID( - ctx context.Context, eventNID types.EventNID, -) (roomNID types.RoomNID, err error) { - err = s.selectRoomNIDForEventNIDStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) - return +func (s *eventStatements) SelectRoomNIDsForEventNIDs( + ctx context.Context, eventNIDs []types.EventNID, +) (map[types.EventNID]types.RoomNID, error) { + sqlStr := strings.Replace(selectRoomNIDsForEventNIDsSQL, "($1)", sqlutil.QueryVariadic(len(eventNIDs)), 1) + sqlPrep, err := s.db.Prepare(sqlStr) + if err != nil { + return nil, err + } + iEventNIDs := make([]interface{}, len(eventNIDs)) + for i, v := range eventNIDs { + iEventNIDs[i] = v + } + rows, err := sqlPrep.QueryContext(ctx, iEventNIDs...) + if err != nil { + return nil, err + } + result := make(map[types.EventNID]types.RoomNID) + for rows.Next() { + var eventNID types.EventNID + var roomNID types.RoomNID + if err = rows.Scan(&eventNID, &roomNID); err != nil { + return nil, err + } + result[eventNID] = roomNID + } + return result, nil } func eventNIDsAsArray(eventNIDs []types.EventNID) string { diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index d73445846..cfd034f31 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -10,8 +10,9 @@ import ( ) type EventJSONPair struct { - EventNID types.EventNID - EventJSON []byte + EventNID types.EventNID + RoomVersion gomatrixserverlib.RoomVersion + EventJSON []byte } type EventJSON interface { @@ -58,7 +59,7 @@ type Events interface { // If an event ID is not in the database then it is omitted from the map. BulkSelectEventNID(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) - SelectRoomNIDForEventNID(ctx context.Context, eventNID types.EventNID) (roomNID types.RoomNID, err error) + SelectRoomNIDsForEventNIDs(ctx context.Context, eventNIDs []types.EventNID) (roomNIDs map[types.EventNID]types.RoomNID, err error) } type Rooms interface {