From b53aac00843a856686ee66da7d2f17e939603331 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 30 Jun 2020 14:09:50 +0100 Subject: [PATCH] Use content_value instead of membership --- .../postgres/current_room_state_table.go | 23 ++++---- currentstateserver/storage/shared/storage.go | 22 +------- .../sqlite3/current_room_state_table.go | 25 +++------ .../storage/tables/interface.go | 56 ++++++++++++++----- go.mod | 2 +- go.sum | 2 + 6 files changed, 66 insertions(+), 64 deletions(-) diff --git a/currentstateserver/storage/postgres/current_room_state_table.go b/currentstateserver/storage/postgres/current_room_state_table.go index 95621913b..9a940f840 100644 --- a/currentstateserver/storage/postgres/current_room_state_table.go +++ b/currentstateserver/storage/postgres/current_room_state_table.go @@ -18,7 +18,6 @@ import ( "context" "database/sql" "encoding/json" - "strconv" "github.com/lib/pq" "github.com/matrix-org/dendrite/currentstateserver/storage/tables" @@ -27,8 +26,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -var leaveEnum = strconv.Itoa(tables.MembershipToEnum["leave"]) - var currentRoomStateSchema = ` -- Stores the current room state for every room. CREATE TABLE IF NOT EXISTS currentstate_current_room_state ( @@ -44,8 +41,8 @@ CREATE TABLE IF NOT EXISTS currentstate_current_room_state ( state_key TEXT NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. headered_event_json TEXT NOT NULL, - -- The 'content.membership' enum value if this event is an m.room.member event. - membership SMALLINT NOT NULL DEFAULT 0, + -- A piece of extracted content e.g membership for m.room.member events + content_value TEXT NOT NULL DEFAULT '', -- Clobber based on 3-uple of room_id, type and state_key CONSTRAINT currentstate_current_room_state_unique UNIQUE (room_id, type, state_key) ); @@ -53,20 +50,20 @@ CREATE TABLE IF NOT EXISTS currentstate_current_room_state ( CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender); -- for querying membership states of users CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership) -WHERE membership IS NOT NULL AND membership != ` + leaveEnum + `; +WHERE type='m.room.member' AND content_value IS NOT NULL AND content_value != 'leave'; ` const upsertRoomStateSQL = "" + - "INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, membership)" + + "INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" + " VALUES ($1, $2, $3, $4, $5, $6, $7)" + " ON CONFLICT ON CONSTRAINT currentstate_current_room_state_unique" + - " DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, membership = $7" + " DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM currentstate_current_room_state WHERE event_id = $1" const selectRoomIDsWithMembershipSQL = "" + - "SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" + "SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2" const selectStateEventSQL = "" + "SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" @@ -111,10 +108,10 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( ctx context.Context, txn *sql.Tx, userID string, - membershipEnum int, + contentVal string, ) ([]string, error) { stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) - rows, err := stmt.QueryContext(ctx, userID, membershipEnum) + rows, err := stmt.QueryContext(ctx, userID, contentVal) if err != nil { return nil, err } @@ -141,7 +138,7 @@ func (s *currentRoomStateStatements) DeleteRoomStateByEventID( func (s *currentRoomStateStatements) UpsertRoomState( ctx context.Context, txn *sql.Tx, - event gomatrixserverlib.HeaderedEvent, membershipEnum int, + event gomatrixserverlib.HeaderedEvent, contentVal string, ) error { headeredJSON, err := json.Marshal(event) if err != nil { @@ -158,7 +155,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( event.Sender(), *event.StateKey(), headeredJSON, - membershipEnum, + contentVal, ) return err } diff --git a/currentstateserver/storage/shared/storage.go b/currentstateserver/storage/shared/storage.go index d78b3e0ed..cc5e7cda6 100644 --- a/currentstateserver/storage/shared/storage.go +++ b/currentstateserver/storage/shared/storage.go @@ -17,7 +17,6 @@ package shared import ( "context" "database/sql" - "fmt" "github.com/matrix-org/dendrite/currentstateserver/storage/tables" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -48,20 +47,9 @@ func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatr // ignore non state events continue } - var membershipEnum int - if event.Type() == "m.room.member" { - membership, err := event.Membership() - if err != nil { - return err - } - enum, ok := tables.MembershipToEnum[membership] - if !ok { - return fmt.Errorf("unknown membership: %s", membership) - } - membershipEnum = enum - } + contentVal := tables.ExtractContentValue(&event) - if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membershipEnum); err != nil { + if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, contentVal); err != nil { return err } } @@ -70,9 +58,5 @@ func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatr } func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) { - enum, ok := tables.MembershipToEnum[membership] - if !ok { - return nil, fmt.Errorf("unknown membership: %s", membership) - } - return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, enum) + return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership) } diff --git a/currentstateserver/storage/sqlite3/current_room_state_table.go b/currentstateserver/storage/sqlite3/current_room_state_table.go index 2e2b0e423..7030fd099 100644 --- a/currentstateserver/storage/sqlite3/current_room_state_table.go +++ b/currentstateserver/storage/sqlite3/current_room_state_table.go @@ -35,37 +35,30 @@ CREATE TABLE IF NOT EXISTS currentstate_current_room_state ( sender TEXT NOT NULL, state_key TEXT NOT NULL, headered_event_json TEXT NOT NULL, - membership INTEGER NOT NULL DEFAULT 0, + content_value TEXT NOT NULL DEFAULT '', UNIQUE (room_id, type, state_key) ); -- for event deletion CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender); --- for querying membership states of users --- CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave'; ` const upsertRoomStateSQL = "" + - "INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, membership)" + + "INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" + " VALUES ($1, $2, $3, $4, $5, $6, $7)" + " ON CONFLICT (event_id, room_id, type, sender)" + - " DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, membership = $7" + " DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM currentstate_current_room_state WHERE event_id = $1" const selectRoomIDsWithMembershipSQL = "" + - "SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" + "SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2" const selectStateEventSQL = "" + "SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + - // TODO: The session_id and transaction_id blanks are here because otherwise - // the rowsToStreamEvents expects there to be exactly five columns. We need to - // figure out if these really need to be in the DB, and if so, we need a - // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + - " FROM currentstate_current_room_state WHERE event_id IN ($1)" + "SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id IN ($1)" type currentRoomStateStatements struct { upsertRoomStateStmt *sql.Stmt @@ -100,10 +93,10 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( ctx context.Context, txn *sql.Tx, userID string, - membershipEnum int, + membership string, ) ([]string, error) { stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) - rows, err := stmt.QueryContext(ctx, userID, membershipEnum) + rows, err := stmt.QueryContext(ctx, userID, membership) if err != nil { return nil, err } @@ -130,7 +123,7 @@ func (s *currentRoomStateStatements) DeleteRoomStateByEventID( func (s *currentRoomStateStatements) UpsertRoomState( ctx context.Context, txn *sql.Tx, - event gomatrixserverlib.HeaderedEvent, membershipEnum int, + event gomatrixserverlib.HeaderedEvent, contentVal string, ) error { headeredJSON, err := json.Marshal(event) if err != nil { @@ -147,7 +140,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( event.Sender(), *event.StateKey(), headeredJSON, - membershipEnum, + contentVal, ) return err } diff --git a/currentstateserver/storage/tables/interface.go b/currentstateserver/storage/tables/interface.go index f2c8b14ed..42dc58ccc 100644 --- a/currentstateserver/storage/tables/interface.go +++ b/currentstateserver/storage/tables/interface.go @@ -19,26 +19,52 @@ import ( "database/sql" "github.com/matrix-org/gomatrixserverlib" + "github.com/tidwall/gjson" ) -var MembershipToEnum = map[string]int{ - gomatrixserverlib.Invite: 1, - gomatrixserverlib.Join: 2, - gomatrixserverlib.Leave: 3, - gomatrixserverlib.Ban: 4, -} -var EnumToMembership = map[int]string{ - 1: gomatrixserverlib.Invite, - 2: gomatrixserverlib.Join, - 3: gomatrixserverlib.Leave, - 4: gomatrixserverlib.Ban, -} - type CurrentRoomState interface { SelectStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) SelectEventsWithEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) - UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, membershipEnum int) error + // UpsertRoomState stores the given event in the database, along with an extracted piece of content. + // The piece of content will vary depending on the event type, and table implementations may use this information to optimise + // lookups e.g membership lookups. The mapped value of `contentVal` is outlined in ExtractContentValue. An empty `contentVal` + // means there is nothing to store for this field. + UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, contentVal string) error DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error // SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. - SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membershipEnum int) ([]string, error) + SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error) +} + +// ExtractContentValue from the given state event. For example, given an m.room.name event with: +// content: { name: "Foo" } +// this returns "Foo". +func ExtractContentValue(ev *gomatrixserverlib.HeaderedEvent) string { + content := ev.Content() + key := "" + switch ev.Type() { + case gomatrixserverlib.MRoomCreate: + key = "creator" + case gomatrixserverlib.MRoomCanonicalAlias: + key = "alias" + case gomatrixserverlib.MRoomHistoryVisibility: + key = "history_visibility" + case gomatrixserverlib.MRoomJoinRules: + key = "join_rule" + case gomatrixserverlib.MRoomMember: + key = "membership" + case gomatrixserverlib.MRoomName: + key = "name" + case "m.room.avatar": + key = "avatar" + case "m.room.topic": + key = "topic" + case "m.room.guest_access": + key = "guest_access" + } + result := gjson.GetBytes(content, key) + if !result.Exists() { + return "" + } + // this returns the empty string if this is not a string type + return result.Str } diff --git a/go.mod b/go.mod index 5f5a74a1b..4add4c3f5 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 24c8d74a0..52a556977 100644 --- a/go.sum +++ b/go.sum @@ -379,6 +379,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d h1:v1 github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328 h1:rz6aiTpUyNPRcWZBWUGDkQjI7lfeLdhzy+x/Pw2jha8= github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe h1:rCjG+azihYsO+EIdm//Zx5gQ7hzeJVraeSukLsW1Mic= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=