diff --git a/src/github.com/matrix-org/dendrite/syncapi/storage/current_room_state_table.go b/src/github.com/matrix-org/dendrite/syncapi/storage/current_room_state_table.go index c5554f8cf..0990f429b 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/storage/current_room_state_table.go +++ b/src/github.com/matrix-org/dendrite/syncapi/storage/current_room_state_table.go @@ -17,6 +17,7 @@ package storage import ( "context" "database/sql" + "encoding/json" "github.com/lib/pq" "github.com/matrix-org/dendrite/common" @@ -35,6 +36,8 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( type TEXT NOT NULL, -- The 'sender' property for the event. sender TEXT NOT NULL, + -- true if the event content contains a url key + contains_url BOOL NOT NULL, -- The state_key value for this state event e.g '' state_key TEXT NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. @@ -49,17 +52,17 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( CONSTRAINT syncapi_room_state_unique UNIQUE (room_id, type, state_key) ); -- for event deletion -CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender); +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender, contains_url); -- for querying membership states of users CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave'; ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, state_key, event_json, membership, added_at)" + - " VALUES ($1, $2, $3, $4, $5, $6, $7, 8)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + " ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" + - " DO UPDATE SET event_id = $2, sender=$4, event_json = $6, membership = $7, added_at = $8" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" @@ -73,7 +76,8 @@ const selectCurrentStateSQL = "" + " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + - " LIMIT $6" + " AND ( $6::bool IS NULL OR contains_url = $6 )" + + " LIMIT $7" const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" @@ -190,6 +194,7 @@ func (s *currentRoomStateStatements) selectCurrentState( pq.StringArray(filter.NotSenders), pq.StringArray(filterConvertWildcardToSQL(filter.Types)), pq.StringArray(filterConvertWildcardToSQL(filter.NotTypes)), + filter.ContainsURL, stateFilter.Limit, ) if err != nil { @@ -212,6 +217,10 @@ func (s *currentRoomStateStatements) upsertRoomState( ctx context.Context, txn *sql.Tx, event gomatrixserverlib.Event, membership *string, addedAt int64, ) error { + var content map[string]interface{} + json.Unmarshal(event.Content(), content) + _, containsURL := content["url"] + stmt := common.TxStmt(txn, s.upsertRoomStateStmt) _, err := stmt.ExecContext( ctx, @@ -219,6 +228,7 @@ func (s *currentRoomStateStatements) upsertRoomState( event.EventID(), event.Type(), event.Sender(), + containsURL, *event.StateKey(), event.JSON(), membership, diff --git a/src/github.com/matrix-org/dendrite/syncapi/storage/invites_table.go b/src/github.com/matrix-org/dendrite/syncapi/storage/invites_table.go index 4f80adf25..37a361837 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/storage/invites_table.go +++ b/src/github.com/matrix-org/dendrite/syncapi/storage/invites_table.go @@ -3,6 +3,7 @@ package storage import ( "context" "database/sql" + "encoding/json" "github.com/lib/pq" "github.com/matrix-org/dendrite/common" @@ -17,13 +18,14 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events ( room_id TEXT NOT NULL, type TEXT NOT NULL, sender TEXT NOT NULL, + contains_url BOOL NOT NULL, target_user_id TEXT NOT NULL, event_json TEXT NOT NULL ); -- For looking up the invites for a given user. CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx - ON syncapi_invite_events (target_user_id, id, room_id, type, sender); + ON syncapi_invite_events (target_user_id, id, room_id, type, sender, contains_url); -- For deleting old invites CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx @@ -32,8 +34,8 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events (" + - " room_id, event_id, type, sender, target_user_id, event_json" + - ") VALUES ($1, $2, $3, $4, $5, $6) RETURNING id" + " room_id, event_id, type, sender, contains_url, target_user_id, event_json" + + ") VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id" const deleteInviteEventSQL = "" + "DELETE FROM syncapi_invite_events WHERE event_id = $1" @@ -47,6 +49,7 @@ const selectInviteEventsInRangeSQL = "" + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + " AND ( $8::text[] IS NULL OR room_id = ANY($8) )" + " AND ( $9::text[] IS NULL OR NOT(room_id = ANY($9)) )" + + " AND ( $10::bool IS NULL OR contains_url = $10 )" + " ORDER BY id DESC" const selectMaxInviteIDSQL = "" + @@ -82,12 +85,17 @@ func (s *inviteEventsStatements) prepare(db *sql.DB) (err error) { func (s *inviteEventsStatements) insertInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.Event, ) (streamPos int64, err error) { + var content map[string]interface{} + json.Unmarshal(inviteEvent.Content(), content) + _, containsURL := content["url"] + err = s.insertInviteEventStmt.QueryRowContext( ctx, inviteEvent.RoomID(), inviteEvent.EventID(), inviteEvent.Type(), inviteEvent.Sender(), + containsURL, *inviteEvent.StateKey(), inviteEvent.JSON(), ).Scan(&streamPos) @@ -115,6 +123,7 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( pq.StringArray(filter.Room.State.NotSenders), pq.StringArray(append(filter.Room.Rooms, filter.Room.State.Rooms...)), pq.StringArray(append(filter.Room.NotRooms, filter.Room.State.NotRooms...)), + filter.Room.State.ContainsURL, ) if err != nil { return nil, err diff --git a/src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go b/src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go index a45c90331..0d4525b20 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go +++ b/src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go @@ -17,6 +17,7 @@ package storage import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" @@ -46,6 +47,8 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( type TEXT NOT NULL, -- The 'sender' property for the event. sender TEXT NOT NULL, + -- true if the event content contains a url key + contains_url BOOL NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. event_json TEXT NOT NULL, -- A list of event IDs which represent a delta of added/removed room state. This can be NULL @@ -60,13 +63,14 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_output_room_ev event_id, room_id, type, - sender); + sender, + contains_url); ` const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - " room_id, event_id, type, sender, event_json, add_state_ids, remove_state_ids, device_id, transaction_id" + - ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING id" + " room_id, event_id, type, sender, contains_url, event_json, add_state_ids, remove_state_ids, device_id, transaction_id" + + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id" const selectEventsSQL = "" + "SELECT id, event_json FROM syncapi_output_room_events WHERE event_id = ANY($1)" @@ -79,7 +83,8 @@ const selectRoomRecentEventsSQL = "" + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + - " ORDER BY id DESC LIMIT $8" + " AND ( $8::bool IS NULL OR contains_url = $8 )" + + " ORDER BY id DESC LIMIT $9" const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" @@ -219,6 +224,10 @@ func (s *outputRoomEventsStatements) insertEvent( txnID = &transactionID.TransactionID } + var content map[string]interface{} + json.Unmarshal(event.Content(), content) + _, containsURL := content["url"] + stmt := common.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, @@ -226,6 +235,7 @@ func (s *outputRoomEventsStatements) insertEvent( event.EventID(), event.Type(), event.Sender(), + containsURL, event.JSON(), pq.StringArray(addState), pq.StringArray(removeState), @@ -246,6 +256,7 @@ func (s *outputRoomEventsStatements) selectRoomRecentEvents( pq.StringArray(timelineFilter.NotSenders), pq.StringArray(filterConvertWildcardToSQL(timelineFilter.Types)), pq.StringArray(filterConvertWildcardToSQL(timelineFilter.NotTypes)), + timelineFilter.ContainsURL, timelineFilter.Limit+1, // TODO: limit abusive values? This can also be done in gomatrix.Filter.Validate ) if err != nil { diff --git a/vendor/src/github.com/matrix-org/gomatrix/filter.go b/vendor/src/github.com/matrix-org/gomatrix/filter.go index 52294bf1c..e9c0c18a8 100644 --- a/vendor/src/github.com/matrix-org/gomatrix/filter.go +++ b/vendor/src/github.com/matrix-org/gomatrix/filter.go @@ -37,13 +37,14 @@ type FilterRoom struct { } type FilterPart struct { - NotRooms []string `json:"not_rooms,omitempty"` - Rooms []string `json:"rooms,omitempty"` - Limit int `json:"limit,omitempty"` - NotSenders []string `json:"not_senders,omitempty"` - NotTypes []string `json:"not_types,omitempty"` - Senders []string `json:"senders,omitempty"` - Types []string `json:"types,omitempty"` + NotRooms []string `json:"not_rooms,omitempty"` + Rooms []string `json:"rooms,omitempty"` + Limit int `json:"limit,omitempty"` + NotSenders []string `json:"not_senders,omitempty"` + NotTypes []string `json:"not_types,omitempty"` + Senders []string `json:"senders,omitempty"` + Types []string `json:"types,omitempty"` + ContainsURL *bool `json:"contains_url,omitempty"` } func (filter *Filter) Validate() error {