mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-14 18:33:09 -06:00
Code Issues Packages Projects Releases Wiki Activity

Filter contains_url implementation

Browse source 
Signed-off-by: Thibaut CHARLES cromfr@gmail.com
This commit is contained in:
Crom (Thibaut CHARLES) 2018-01-18 02:03:43 +01:00
parent 97a5882d81
commit be96370e30
No known key found for this signature in database
GPG key ID: 45A3D5F880B9E6D0
4 changed files with 50 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/github.com/matrix-org/dendrite/syncapi/storage/current_room_state_table.go
View file

@ -17,6 +17,7 @@ package storage
import ( import (
"context" "context"
"database/sql" "database/sql"
"encoding/json"
"github.com/lib/pq" "github.com/lib/pq"
"github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common"
@ -35,6 +36,8 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state (
type TEXT NOT NULL, type TEXT NOT NULL,
-- The 'sender' property for the event. -- The 'sender' property for the event.
sender TEXT NOT NULL, sender TEXT NOT NULL,
-- true if the event content contains a url key
contains_url BOOL NOT NULL,
-- The state_key value for this state event e.g '' -- The state_key value for this state event e.g ''
state_key TEXT NOT NULL, state_key TEXT NOT NULL,
-- The JSON for the event. Stored as TEXT because this should be valid UTF-8. -- The JSON for the event. Stored as TEXT because this should be valid UTF-8.
@ -49,17 +52,17 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state (
CONSTRAINT syncapi_room_state_unique UNIQUE (room_id, type, state_key) CONSTRAINT syncapi_room_state_unique UNIQUE (room_id, type, state_key)
); );
-- for event deletion -- for event deletion
CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender); CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender, contains_url);
-- for querying membership states of users -- for querying membership states of users
CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave'; CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave';
` `
const upsertRoomStateSQL = "" + const upsertRoomStateSQL = "" +
"INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, state_key, event_json, membership, added_at)" + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" +
" VALUES ($1, $2, $3, $4, $5, $6, $7, 8)" + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" +
" ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" + " ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" +
" DO UPDATE SET event_id = $2, sender=$4, event_json = $6, membership = $7, added_at = $8" " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9"
const deleteRoomStateByEventIDSQL = "" + const deleteRoomStateByEventIDSQL = "" +
"DELETE FROM syncapi_current_room_state WHERE event_id = $1" "DELETE FROM syncapi_current_room_state WHERE event_id = $1"
@ -73,7 +76,8 @@ const selectCurrentStateSQL = "" +
" AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" +
" AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" +
" AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" +
" LIMIT $6" " AND ( $6::bool IS NULL OR contains_url = $6 )" +
" LIMIT $7"
const selectJoinedUsersSQL = "" + const selectJoinedUsersSQL = "" +
"SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'"
@ -190,6 +194,7 @@ func (s *currentRoomStateStatements) selectCurrentState(
pq.StringArray(filter.NotSenders), pq.StringArray(filter.NotSenders),
pq.StringArray(filterConvertWildcardToSQL(filter.Types)), pq.StringArray(filterConvertWildcardToSQL(filter.Types)),
pq.StringArray(filterConvertWildcardToSQL(filter.NotTypes)), pq.StringArray(filterConvertWildcardToSQL(filter.NotTypes)),
filter.ContainsURL,
stateFilter.Limit, stateFilter.Limit,
) )
if err != nil { if err != nil {
@ -212,6 +217,10 @@ func (s *currentRoomStateStatements) upsertRoomState(
ctx context.Context, txn *sql.Tx, ctx context.Context, txn *sql.Tx,
event gomatrixserverlib.Event, membership *string, addedAt int64, event gomatrixserverlib.Event, membership *string, addedAt int64,
) error { ) error {
var content map[string]interface{}
json.Unmarshal(event.Content(), content)
_, containsURL := content["url"]
stmt := common.TxStmt(txn, s.upsertRoomStateStmt) stmt := common.TxStmt(txn, s.upsertRoomStateStmt)
_, err := stmt.ExecContext( _, err := stmt.ExecContext(
ctx, ctx,
@ -219,6 +228,7 @@ func (s *currentRoomStateStatements) upsertRoomState(
event.EventID(), event.EventID(),
event.Type(), event.Type(),
event.Sender(), event.Sender(),
containsURL,
*event.StateKey(), *event.StateKey(),
event.JSON(), event.JSON(),
membership, membership,

15
src/github.com/matrix-org/dendrite/syncapi/storage/invites_table.go
View file

@ -3,6 +3,7 @@ package storage
import ( import (
"context" "context"
"database/sql" "database/sql"
"encoding/json"
"github.com/lib/pq" "github.com/lib/pq"
"github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common"
@ -17,13 +18,14 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events (
room_id TEXT NOT NULL, room_id TEXT NOT NULL,
type TEXT NOT NULL, type TEXT NOT NULL,
sender TEXT NOT NULL, sender TEXT NOT NULL,
contains_url BOOL NOT NULL,
target_user_id TEXT NOT NULL, target_user_id TEXT NOT NULL,
event_json TEXT NOT NULL event_json TEXT NOT NULL
); );
-- For looking up the invites for a given user. -- For looking up the invites for a given user.
CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx
ON syncapi_invite_events (target_user_id, id, room_id, type, sender); ON syncapi_invite_events (target_user_id, id, room_id, type, sender, contains_url);
-- For deleting old invites -- For deleting old invites
CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx
@ -32,8 +34,8 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx
const insertInviteEventSQL = "" + const insertInviteEventSQL = "" +
"INSERT INTO syncapi_invite_events (" + "INSERT INTO syncapi_invite_events (" +
" room_id, event_id, type, sender, target_user_id, event_json" + " room_id, event_id, type, sender, contains_url, target_user_id, event_json" +
") VALUES ($1, $2, $3, $4, $5, $6) RETURNING id" ") VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id"
const deleteInviteEventSQL = "" + const deleteInviteEventSQL = "" +
"DELETE FROM syncapi_invite_events WHERE event_id = $1" "DELETE FROM syncapi_invite_events WHERE event_id = $1"
@ -47,6 +49,7 @@ const selectInviteEventsInRangeSQL = "" +
" AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" +
" AND ( $8::text[] IS NULL OR room_id = ANY($8) )" + " AND ( $8::text[] IS NULL OR room_id = ANY($8) )" +
" AND ( $9::text[] IS NULL OR NOT(room_id = ANY($9)) )" + " AND ( $9::text[] IS NULL OR NOT(room_id = ANY($9)) )" +
" AND ( $10::bool IS NULL OR contains_url = $10 )" +
" ORDER BY id DESC" " ORDER BY id DESC"
const selectMaxInviteIDSQL = "" + const selectMaxInviteIDSQL = "" +
@ -82,12 +85,17 @@ func (s *inviteEventsStatements) prepare(db *sql.DB) (err error) {
func (s *inviteEventsStatements) insertInviteEvent( func (s *inviteEventsStatements) insertInviteEvent(
ctx context.Context, inviteEvent gomatrixserverlib.Event, ctx context.Context, inviteEvent gomatrixserverlib.Event,
) (streamPos int64, err error) { ) (streamPos int64, err error) {
var content map[string]interface{}
json.Unmarshal(inviteEvent.Content(), content)
_, containsURL := content["url"]
err = s.insertInviteEventStmt.QueryRowContext( err = s.insertInviteEventStmt.QueryRowContext(
ctx, ctx,
inviteEvent.RoomID(), inviteEvent.RoomID(),
inviteEvent.EventID(), inviteEvent.EventID(),
inviteEvent.Type(), inviteEvent.Type(),
inviteEvent.Sender(), inviteEvent.Sender(),
containsURL,
*inviteEvent.StateKey(), *inviteEvent.StateKey(),
inviteEvent.JSON(), inviteEvent.JSON(),
).Scan(&streamPos) ).Scan(&streamPos)
@ -115,6 +123,7 @@ func (s *inviteEventsStatements) selectInviteEventsInRange(
pq.StringArray(filter.Room.State.NotSenders), pq.StringArray(filter.Room.State.NotSenders),
pq.StringArray(append(filter.Room.Rooms, filter.Room.State.Rooms...)), pq.StringArray(append(filter.Room.Rooms, filter.Room.State.Rooms...)),
pq.StringArray(append(filter.Room.NotRooms, filter.Room.State.NotRooms...)), pq.StringArray(append(filter.Room.NotRooms, filter.Room.State.NotRooms...)),
filter.Room.State.ContainsURL,
) )
if err != nil { if err != nil {
return nil, err return nil, err

19
src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go
View file

@ -17,6 +17,7 @@ package storage
import ( import (
"context" "context"
"database/sql" "database/sql"
"encoding/json"
"github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrix"
@ -46,6 +47,8 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
type TEXT NOT NULL, type TEXT NOT NULL,
-- The 'sender' property for the event. -- The 'sender' property for the event.
sender TEXT NOT NULL, sender TEXT NOT NULL,
-- true if the event content contains a url key
contains_url BOOL NOT NULL,
-- The JSON for the event. Stored as TEXT because this should be valid UTF-8. -- The JSON for the event. Stored as TEXT because this should be valid UTF-8.
event_json TEXT NOT NULL, event_json TEXT NOT NULL,
-- A list of event IDs which represent a delta of added/removed room state. This can be NULL -- A list of event IDs which represent a delta of added/removed room state. This can be NULL
@ -60,13 +63,14 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_output_room_ev
event_id, event_id,
room_id, room_id,
type, type,
sender); sender,
contains_url);
` `
const insertEventSQL = "" + const insertEventSQL = "" +
"INSERT INTO syncapi_output_room_events (" + "INSERT INTO syncapi_output_room_events (" +
" room_id, event_id, type, sender, event_json, add_state_ids, remove_state_ids, device_id, transaction_id" + " room_id, event_id, type, sender, contains_url, event_json, add_state_ids, remove_state_ids, device_id, transaction_id" +
") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING id" ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id"
const selectEventsSQL = "" + const selectEventsSQL = "" +
"SELECT id, event_json FROM syncapi_output_room_events WHERE event_id = ANY($1)" "SELECT id, event_json FROM syncapi_output_room_events WHERE event_id = ANY($1)"
@ -79,7 +83,8 @@ const selectRoomRecentEventsSQL = "" +
" AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" +
" AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" +
" AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" +
" ORDER BY id DESC LIMIT $8" " AND ( $8::bool IS NULL OR contains_url = $8 )" +
" ORDER BY id DESC LIMIT $9"
const selectMaxEventIDSQL = "" + const selectMaxEventIDSQL = "" +
"SELECT MAX(id) FROM syncapi_output_room_events" "SELECT MAX(id) FROM syncapi_output_room_events"
@ -219,6 +224,10 @@ func (s *outputRoomEventsStatements) insertEvent(
txnID = &transactionID.TransactionID txnID = &transactionID.TransactionID
} }
var content map[string]interface{}
json.Unmarshal(event.Content(), content)
_, containsURL := content["url"]
stmt := common.TxStmt(txn, s.insertEventStmt) stmt := common.TxStmt(txn, s.insertEventStmt)
err = stmt.QueryRowContext( err = stmt.QueryRowContext(
ctx, ctx,
@ -226,6 +235,7 @@ func (s *outputRoomEventsStatements) insertEvent(
event.EventID(), event.EventID(),
event.Type(), event.Type(),
event.Sender(), event.Sender(),
containsURL,
event.JSON(), event.JSON(),
pq.StringArray(addState), pq.StringArray(addState),
pq.StringArray(removeState), pq.StringArray(removeState),
@ -246,6 +256,7 @@ func (s *outputRoomEventsStatements) selectRoomRecentEvents(
pq.StringArray(timelineFilter.NotSenders), pq.StringArray(timelineFilter.NotSenders),
pq.StringArray(filterConvertWildcardToSQL(timelineFilter.Types)), pq.StringArray(filterConvertWildcardToSQL(timelineFilter.Types)),
pq.StringArray(filterConvertWildcardToSQL(timelineFilter.NotTypes)), pq.StringArray(filterConvertWildcardToSQL(timelineFilter.NotTypes)),
timelineFilter.ContainsURL,
timelineFilter.Limit+1, // TODO: limit abusive values? This can also be done in gomatrix.Filter.Validate timelineFilter.Limit+1, // TODO: limit abusive values? This can also be done in gomatrix.Filter.Validate
) )
if err != nil { if err != nil {

15
vendor/src/github.com/matrix-org/gomatrix/filter.go vendored
View file

@ -37,13 +37,14 @@ type FilterRoom struct {
} }
type FilterPart struct { type FilterPart struct {
NotRooms []string `json:"not_rooms,omitempty"` NotRooms []string `json:"not_rooms,omitempty"`
Rooms []string `json:"rooms,omitempty"` Rooms []string `json:"rooms,omitempty"`
Limit int `json:"limit,omitempty"` Limit int `json:"limit,omitempty"`
NotSenders []string `json:"not_senders,omitempty"` NotSenders []string `json:"not_senders,omitempty"`
NotTypes []string `json:"not_types,omitempty"` NotTypes []string `json:"not_types,omitempty"`
Senders []string `json:"senders,omitempty"` Senders []string `json:"senders,omitempty"`
Types []string `json:"types,omitempty"` Types []string `json:"types,omitempty"`
ContainsURL *bool `json:"contains_url,omitempty"`
} }
func (filter *Filter) Validate() error { func (filter *Filter) Validate() error {