From 74939ffb61ab609c878bd15ab9e2fe5158e48a39 Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Thu, 19 May 2022 10:10:15 +0200 Subject: [PATCH] Add config, move to base --- internal/fulltext/bleve.go | 53 +++++++++++++++++++-------------- internal/fulltext/bleve_test.go | 15 +++++++--- setup/base/base.go | 11 +++++++ setup/config/config_syncapi.go | 26 ++++++++++++++++ 4 files changed, 79 insertions(+), 26 deletions(-) diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go index 2b6e151ee..c41c76aa9 100644 --- a/internal/fulltext/bleve.go +++ b/internal/fulltext/bleve.go @@ -18,7 +18,8 @@ import ( "strings" "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/analysis/lang/en" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) @@ -49,9 +50,9 @@ func (i *IndexElement) SetContentType(v string) { } // New opens a new/existing fulltext index -func New(path string) (fts *Search, err error) { +func New(cfg config.Fulltext) (fts *Search, err error) { fts = &Search{} - fts.FulltextIndex, err = openIndex(path) + fts.FulltextIndex, err = openIndex(cfg) if err != nil { return nil, err } @@ -128,30 +129,38 @@ func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, or return f.FulltextIndex.Search(s) } -func openIndex(path string) (bleve.Index, error) { - if index, err := bleve.Open(path); err == nil { +func openIndex(cfg config.Fulltext) (bleve.Index, error) { + m := getMapping(cfg) + if cfg.InMemory { + return bleve.NewMemOnly(m) + } + if index, err := bleve.Open(string(cfg.IndexPath)); err == nil { return index, nil } - enFieldMapping := bleve.NewTextFieldMapping() - enFieldMapping.Analyzer = en.AnalyzerName - - eventMapping := bleve.NewDocumentMapping() - eventMapping.AddFieldMappingsAt("Content", enFieldMapping) - eventMapping.AddFieldMappingsAt("StreamPosition", bleve.NewNumericFieldMapping()) - - idFieldMapping := bleve.NewKeywordFieldMapping() - eventMapping.AddFieldMappingsAt("ContentType", idFieldMapping) - eventMapping.AddFieldMappingsAt("RoomID", idFieldMapping) - eventMapping.AddFieldMappingsAt("EventID", idFieldMapping) - - mapping := bleve.NewIndexMapping() - mapping.AddDocumentMapping("Event", eventMapping) - mapping.DefaultType = "Event" - - index, err := bleve.New(path, mapping) + index, err := bleve.New(string(cfg.IndexPath), m) if err != nil { return nil, err } return index, nil } + +func getMapping(cfg config.Fulltext) *mapping.IndexMappingImpl { + enFieldMapping := bleve.NewTextFieldMapping() + enFieldMapping.Analyzer = cfg.Language + + eventMapping := bleve.NewDocumentMapping() + eventMapping.AddFieldMappingsAt("Content", enFieldMapping) + eventMapping.AddFieldMappingsAt("StreamPosition", bleve.NewNumericFieldMapping()) + + // Index entries as is + idFieldMapping := bleve.NewKeywordFieldMapping() + eventMapping.AddFieldMappingsAt("ContentType", idFieldMapping) + eventMapping.AddFieldMappingsAt("RoomID", idFieldMapping) + eventMapping.AddFieldMappingsAt("EventID", idFieldMapping) + + indexMapping := bleve.NewIndexMapping() + indexMapping.AddDocumentMapping("Event", eventMapping) + indexMapping.DefaultType = "Event" + return indexMapping +} diff --git a/internal/fulltext/bleve_test.go b/internal/fulltext/bleve_test.go index 21db78e7e..a819733e0 100644 --- a/internal/fulltext/bleve_test.go +++ b/internal/fulltext/bleve_test.go @@ -19,13 +19,20 @@ import ( "testing" "github.com/matrix-org/dendrite/internal/fulltext" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) func mustOpenIndex(t *testing.T, tempDir string) *fulltext.Search { t.Helper() - fts, err := fulltext.New(tempDir) + cfg := config.Fulltext{} + cfg.Defaults(true) + if tempDir != "" { + cfg.IndexPath = config.Path(tempDir) + cfg.InMemory = false + } + fts, err := fulltext.New(cfg) if err != nil { t.Fatal("failed to open fulltext index:", err) } @@ -93,7 +100,7 @@ func TestOpen(t *testing.T) { } func TestIndex(t *testing.T) { - fts := mustOpenIndex(t, t.TempDir()) + fts := mustOpenIndex(t, "") defer fts.Close() // add some data @@ -117,7 +124,7 @@ func TestIndex(t *testing.T) { } func TestDelete(t *testing.T) { - fts := mustOpenIndex(t, t.TempDir()) + fts := mustOpenIndex(t, "") defer fts.Close() eventIDs, roomIDs := mustAddTestData(t, fts, 0) res1, err := fts.Search("lorem", roomIDs[:1], nil, 50, 0, false) @@ -213,7 +220,7 @@ func TestSearch(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - f := mustOpenIndex(t, t.TempDir()) + f := mustOpenIndex(t, "") eventIDs, roomIDs := mustAddTestData(t, f, 0) var searchRooms []string for _, x := range tt.args.roomIndex { diff --git a/setup/base/base.go b/setup/base/base.go index 5cbd7da9c..d46624f63 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -31,6 +31,7 @@ import ( "github.com/getsentry/sentry-go" sentryhttp "github.com/getsentry/sentry-go/http" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -87,6 +88,7 @@ type BaseDendrite struct { Database *sql.DB DatabaseWriter sqlutil.Writer EnableMetrics bool + Fulltext *fulltext.Search } const NoListener = "" @@ -146,6 +148,14 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base logrus.WithError(err).Panicf("failed to start opentracing") } + var fts *fulltext.Search + if cfg.SyncAPI.Fulltext.Enabled { + fts, err = fulltext.New(cfg.SyncAPI.Fulltext) + if err != nil { + logrus.WithError(err).Panicf("failed to create full text") + } + } + if cfg.Global.Sentry.Enabled { logrus.Info("Setting up Sentry for debugging...") err = sentry.Init(sentry.ClientOptions{ @@ -248,6 +258,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base Database: db, // set if monolith with global connection pool only DatabaseWriter: writer, // set if monolith with global connection pool only EnableMetrics: enableMetrics, + Fulltext: fts, } } diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go index 7d5e3808a..371ae2708 100644 --- a/setup/config/config_syncapi.go +++ b/setup/config/config_syncapi.go @@ -9,6 +9,8 @@ type SyncAPI struct { Database DatabaseOptions `yaml:"database"` RealIPHeader string `yaml:"real_ip_header"` + + Fulltext Fulltext `yaml:"fulltext"` } func (c *SyncAPI) Defaults(generate bool) { @@ -16,12 +18,14 @@ func (c *SyncAPI) Defaults(generate bool) { c.InternalAPI.Connect = "http://localhost:7773" c.ExternalAPI.Listen = "http://localhost:8073" c.Database.Defaults(10) + c.Fulltext.Defaults(generate) if generate { c.Database.ConnectionString = "file:syncapi.db" } } func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + c.Fulltext.Verify(configErrs, isMonolith) if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString)) } @@ -32,3 +36,25 @@ func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect)) checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen)) } + +type Fulltext struct { + Enabled bool `yaml:"enabled"` + IndexPath Path `yaml:"index_path"` + InMemory bool `yaml:"in_memory"` // only useful in tests + Language string `yaml:"language"` // the language to use when analysing content +} + +func (f *Fulltext) Defaults(generate bool) { + f.Enabled = false + f.IndexPath = "./fulltextindex" + f.Language = "en" + if generate { + f.Enabled = true + f.InMemory = true + } +} + +func (f *Fulltext) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "syncapi.fulltext.index_path", string(f.IndexPath)) + checkNotEmpty(configErrs, "syncapi.fulltext.language", f.Language) +}