From b33d9e5045101f2e232645546ff3a6a324dc3e9e Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Wed, 18 May 2022 17:01:04 +0200 Subject: [PATCH] Remove FTS from storage layer Fix query --- syncapi/storage/postgres/syncserver.go | 4 +-- syncapi/storage/shared/syncserver.go | 32 +------------------ .../sqlite3/output_room_events_table.go | 21 +++++++++--- syncapi/storage/sqlite3/syncserver.go | 4 +-- syncapi/storage/storage.go | 7 ++-- 5 files changed, 23 insertions(+), 45 deletions(-) diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index b2de92171..9cfe7c070 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -20,7 +20,6 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" @@ -37,7 +36,7 @@ type SyncServerDatasource struct { } // NewDatabase creates a new sync server database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, fts *fulltext.Search) (*SyncServerDatasource, error) { +func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil { @@ -122,7 +121,6 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, NotificationData: notificationData, Ignores: ignores, Presence: presence, - FTS: fts, } return &d, nil } diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index c6fef5d09..96d2bc9a4 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -20,15 +20,12 @@ import ( "encoding/json" "fmt" - "github.com/matrix-org/dendrite/internal/fulltext" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/tidwall/gjson" - "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -52,7 +49,6 @@ type Database struct { NotificationData tables.NotificationData Ignores tables.Ignores Presence tables.Presence - FTS *fulltext.Search } func (d *Database) readOnlySnapshot(ctx context.Context) (*sql.Tx, error) { @@ -394,32 +390,6 @@ func (d *Database) WriteEvent( return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition, topoPosition) }) - e := fulltext.IndexElement{ - EventID: ev.EventID(), - RoomID: ev.RoomID(), - StreamPosition: int64(pduPosition), - } - e.SetContentType(ev.Type()) - - switch ev.Type() { - case "m.room.message": - e.Content = gjson.GetBytes(ev.Content(), "body").String() - case gomatrixserverlib.MRoomName: - e.Content = gjson.GetBytes(ev.Content(), "name").String() - case gomatrixserverlib.MRoomTopic: - e.Content = gjson.GetBytes(ev.Content(), "topic").String() - case gomatrixserverlib.MRoomRedaction: - if err := d.FTS.Delete(ev.Redacts()); err != nil { - logrus.WithError(err).Warn("failed to delete entry from fulltext index") - } - } - if e.Content != "" { - logrus.Debugf("Indexing element: %+v", e) - if err := d.FTS.Index(e); err != nil { - logrus.WithError(err).Warn("failed to write to fulltext index") - } - } - return pduPosition, returnErr } diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index ee12719d5..39ec26339 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -114,7 +114,7 @@ const selectContextAfterEventSQL = "" + // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters -const selectSearchSQL = "SELECT event_id, headered_event_json FROM syncapi_output_room_events WHERE type = ANY($1) LIMIT $2 OFFSET $3 ORDER BY id ASC" +const selectSearchSQL = "SELECT event_id, headered_event_json FROM syncapi_output_room_events WHERE type IN ($1) LIMIT $2 OFFSET $3 ORDER BY id ASC" type outputRoomEventsStatements struct { db *sql.DB @@ -126,7 +126,7 @@ type outputRoomEventsStatements struct { selectContextEventStmt *sql.Stmt selectContextBeforeEventStmt *sql.Stmt selectContextAfterEventStmt *sql.Stmt - selectSearchStmt *sql.Stmt + //selectSearchStmt *sql.Stmt - prepared at runtime } func NewSqliteEventsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Events, error) { @@ -146,7 +146,7 @@ func NewSqliteEventsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Even {&s.selectContextEventStmt, selectContextEventSQL}, {&s.selectContextBeforeEventStmt, selectContextBeforeEventSQL}, {&s.selectContextAfterEventStmt, selectContextAfterEventSQL}, - {&s.selectSearchStmt, selectSearchSQL}, + //{&s.selectSearchStmt, selectSearchSQL}, - prepared at runtime }.Prepare(db) } @@ -619,7 +619,20 @@ func unmarshalStateIDs(addIDsJSON, delIDsJSON string) (addIDs []string, delIDs [ } func (s *outputRoomEventsStatements) ReIndex(ctx context.Context, txn *sql.Tx, limit, offset int64, types []string) ([]gomatrixserverlib.HeaderedEvent, error) { - rows, err := sqlutil.TxStmt(txn, s.selectSearchStmt).QueryContext(ctx, types, limit, offset) + params := make([]interface{}, len(types)) + for i := range types { + params[i] = types[i] + } + params = append(params, limit) + params = append(params, offset) + selectSQL := strings.Replace(selectSearchSQL, "($1)", sqlutil.QueryVariadic(len(types)), 1) + + stmt, err := s.db.Prepare(selectSQL) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, stmt, "selectEvents: stmt.close() failed") + rows, err := stmt.QueryContext(ctx, params...) if err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index a279748c6..e08a0ba82 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -18,7 +18,6 @@ package sqlite3 import ( "database/sql" - "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" @@ -37,7 +36,7 @@ type SyncServerDatasource struct { // NewDatabase creates a new sync server database // nolint: gocyclo -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, fts *fulltext.Search) (*SyncServerDatasource, error) { +func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil { @@ -46,7 +45,6 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, if err = d.prepare(dbProperties); err != nil { return nil, err } - d.FTS = fts return &d, nil } diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index a31621a55..5b20c6cc2 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -20,7 +20,6 @@ package storage import ( "fmt" - "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage/postgres" @@ -28,12 +27,12 @@ import ( ) // NewSyncServerDatasource opens a database connection. -func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, fts *fulltext.Search) (Database, error) { +func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties, fts) + return sqlite3.NewDatabase(base, dbProperties) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties, fts) + return postgres.NewDatabase(base, dbProperties) default: return nil, fmt.Errorf("unexpected database type") }