diff --git a/src/github.com/matrix-org/dendrite/roomserver/storage/event_json_table.go b/src/github.com/matrix-org/dendrite/roomserver/storage/event_json_table.go new file mode 100644 index 000000000..6296d7061 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/roomserver/storage/event_json_table.go @@ -0,0 +1,93 @@ +package storage + +import ( + "database/sql" + "github.com/lib/pq" + "github.com/matrix-org/dendrite/roomserver/types" +) + +const eventJSONSchema = ` +-- Stores the JSON for each event. This kept separate from the main events +-- table to keep the rows in the main events table small. +CREATE TABLE IF NOT EXISTS event_json ( + -- Local numeric ID for the event. + event_nid BIGINT NOT NULL PRIMARY KEY, + -- The JSON for the event. + -- Stored as TEXT because this should be valid UTF-8. + -- Not stored as a JSONB because we always just pull the entire event + -- so there is no point in postgres parsing it. + -- Not stored as JSON because we already validate the JSON in the server + -- so there is no point in postgres validating it. + -- TODO: Should we be compressing the events with Snappy or DEFLATE? + event_json TEXT NOT NULL +); +` + +const insertEventJSONSQL = "" + + "INSERT INTO event_json (event_nid, event_json) VALUES ($1, $2)" + + " ON CONFLICT DO NOTHING" + +// Bulk event JSON lookup by numeric event ID. +// Sort by the numeric event ID. +// This means that we can use binary search to lookup by numeric event ID. +const bulkSelectEventJSONSQL = "" + + "SELECT event_nid, event_json FROM event_json" + + " WHERE event_nid = ANY($1)" + + " ORDER BY event_nid ASC" + +type eventJSONStatements struct { + insertEventJSONStmt *sql.Stmt + bulkSelectEventJSONStmt *sql.Stmt +} + +func (s *eventJSONStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(eventJSONSchema) + if err != nil { + return + } + if s.insertEventJSONStmt, err = db.Prepare(insertEventJSONSQL); err != nil { + return + } + if s.bulkSelectEventJSONStmt, err = db.Prepare(bulkSelectEventJSONSQL); err != nil { + return + } + return +} + +func (s *eventJSONStatements) insertEventJSON(eventNID types.EventNID, eventJSON []byte) error { + _, err := s.insertEventJSONStmt.Exec(int64(eventNID), eventJSON) + return err +} + +type eventJSONPair struct { + EventNID types.EventNID + EventJSON []byte +} + +func (s *eventJSONStatements) bulkSelectEventJSON(eventNIDs []types.EventNID) ([]eventJSONPair, error) { + nids := make([]int64, len(eventNIDs)) + for i := range eventNIDs { + nids[i] = int64(eventNIDs[i]) + } + rows, err := s.bulkSelectEventJSONStmt.Query(pq.Int64Array(nids)) + if err != nil { + return nil, err + } + defer rows.Close() + + // We know that we will only get as many results as event NIDs + // because of the unique constraint on event NIDs. + // So we can allocate an array of the correct size now. + // We might get fewer results than NIDs so we adjust the length of the slice before returning it. + results := make([]eventJSONPair, len(eventNIDs)) + i := 0 + for ; rows.Next(); i++ { + result := &results[i] + var eventNID int64 + if err := rows.Scan(&eventNID, &result.EventJSON); err != nil { + return nil, err + } + result.EventNID = types.EventNID(eventNID) + } + return results[:i], nil +} diff --git a/src/github.com/matrix-org/dendrite/roomserver/storage/sql.go b/src/github.com/matrix-org/dendrite/roomserver/storage/sql.go index b8a56ddcb..12706d31a 100644 --- a/src/github.com/matrix-org/dendrite/roomserver/storage/sql.go +++ b/src/github.com/matrix-org/dendrite/roomserver/storage/sql.go @@ -13,8 +13,7 @@ type statements struct { eventStateKeyStatements roomStatements eventStatements - insertEventJSONStmt *sql.Stmt - bulkSelectEventJSONStmt *sql.Stmt + eventJSONStatements insertStateStmt *sql.Stmt bulkSelectStateBlockNIDsStmt *sql.Stmt insertStateDataStmt *sql.Stmt @@ -45,94 +44,13 @@ func (s *statements) prepare(db *sql.DB) error { return err } - if err = s.prepareEventJSON(db); err != nil { + if err = s.eventJSONStatements.prepare(db); err != nil { return err } return nil } -func (s *statements) prepareEventJSON(db *sql.DB) (err error) { - _, err = db.Exec(eventJSONSchema) - if err != nil { - return - } - if s.insertEventJSONStmt, err = db.Prepare(insertEventJSONSQL); err != nil { - return - } - if s.bulkSelectEventJSONStmt, err = db.Prepare(bulkSelectEventJSONSQL); err != nil { - return - } - return -} - -const eventJSONSchema = ` --- Stores the JSON for each event. This kept separate from the main events --- table to keep the rows in the main events table small. -CREATE TABLE IF NOT EXISTS event_json ( - -- Local numeric ID for the event. - event_nid BIGINT NOT NULL PRIMARY KEY, - -- The JSON for the event. - -- Stored as TEXT because this should be valid UTF-8. - -- Not stored as a JSONB because we always just pull the entire event - -- so there is no point in postgres parsing it. - -- Not stored as JSON because we already validate the JSON in the server - -- so there is no point in postgres validating it. - -- TODO: Should we be compressing the events with Snappy or DEFLATE? - event_json TEXT NOT NULL -); -` - -const insertEventJSONSQL = "" + - "INSERT INTO event_json (event_nid, event_json) VALUES ($1, $2)" + - " ON CONFLICT DO NOTHING" - -// Bulk event JSON lookup by numeric event ID. -// Sort by the numeric event ID. -// This means that we can use binary search to lookup by numeric event ID. -const bulkSelectEventJSONSQL = "" + - "SELECT event_nid, event_json FROM event_json" + - " WHERE event_nid = ANY($1)" + - " ORDER BY event_nid ASC" - -func (s *statements) insertEventJSON(eventNID types.EventNID, eventJSON []byte) error { - _, err := s.insertEventJSONStmt.Exec(int64(eventNID), eventJSON) - return err -} - -type eventJSONPair struct { - EventNID types.EventNID - EventJSON []byte -} - -func (s *statements) bulkSelectEventJSON(eventNIDs []types.EventNID) ([]eventJSONPair, error) { - nids := make([]int64, len(eventNIDs)) - for i := range eventNIDs { - nids[i] = int64(eventNIDs[i]) - } - rows, err := s.bulkSelectEventJSONStmt.Query(pq.Int64Array(nids)) - if err != nil { - return nil, err - } - defer rows.Close() - - // We know that we will only get as many results as event NIDs - // because of the unique constraint on event NIDs. - // So we can allocate an array of the correct size now. - // We might get fewer results than NIDs so we adjust the length of the slice before returning it. - results := make([]eventJSONPair, len(eventNIDs)) - i := 0 - for ; rows.Next(); i++ { - result := &results[i] - var eventNID int64 - if err := rows.Scan(&eventNID, &result.EventJSON); err != nil { - return nil, err - } - result.EventNID = types.EventNID(eventNID) - } - return results[:i], nil -} - const stateSchema = ` -- The state of a room before an event. -- Stored as a list of state_block entries stored in a separate table.