Add config, move to base

internal/fulltext/bleve.go

@@ -18,7 +18,8 @@ import (
 	"strings"
 
 	"github.com/blevesearch/bleve/v2"
-	"github.com/blevesearch/bleve/v2/analysis/lang/en"
+	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -49,9 +50,9 @@ func (i *IndexElement) SetContentType(v string) {
 }
 
 // New opens a new/existing fulltext index
-func New(path string) (fts *Search, err error) {
+func New(cfg config.Fulltext) (fts *Search, err error) {
 	fts = &Search{}
-	fts.FulltextIndex, err = openIndex(path)
+	fts.FulltextIndex, err = openIndex(cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -128,30 +129,38 @@ func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, or
 	return f.FulltextIndex.Search(s)
 }
 
-func openIndex(path string) (bleve.Index, error) {
+func openIndex(cfg config.Fulltext) (bleve.Index, error) {
-	if index, err := bleve.Open(path); err == nil {
+	m := getMapping(cfg)
+	if cfg.InMemory {
+		return bleve.NewMemOnly(m)
+	}
+	if index, err := bleve.Open(string(cfg.IndexPath)); err == nil {
 		return index, nil
 	}
 
-	enFieldMapping := bleve.NewTextFieldMapping()
+	index, err := bleve.New(string(cfg.IndexPath), m)
-	enFieldMapping.Analyzer = en.AnalyzerName
-
-	eventMapping := bleve.NewDocumentMapping()
-	eventMapping.AddFieldMappingsAt("Content", enFieldMapping)
-	eventMapping.AddFieldMappingsAt("StreamPosition", bleve.NewNumericFieldMapping())
-
-	idFieldMapping := bleve.NewKeywordFieldMapping()
-	eventMapping.AddFieldMappingsAt("ContentType", idFieldMapping)
-	eventMapping.AddFieldMappingsAt("RoomID", idFieldMapping)
-	eventMapping.AddFieldMappingsAt("EventID", idFieldMapping)
-
-	mapping := bleve.NewIndexMapping()
-	mapping.AddDocumentMapping("Event", eventMapping)
-	mapping.DefaultType = "Event"
-
-	index, err := bleve.New(path, mapping)
 	if err != nil {
 		return nil, err
 	}
 	return index, nil
 }
+
+func getMapping(cfg config.Fulltext) *mapping.IndexMappingImpl {
+	enFieldMapping := bleve.NewTextFieldMapping()
+	enFieldMapping.Analyzer = cfg.Language
+
+	eventMapping := bleve.NewDocumentMapping()
+	eventMapping.AddFieldMappingsAt("Content", enFieldMapping)
+	eventMapping.AddFieldMappingsAt("StreamPosition", bleve.NewNumericFieldMapping())
+
+	// Index entries as is
+	idFieldMapping := bleve.NewKeywordFieldMapping()
+	eventMapping.AddFieldMappingsAt("ContentType", idFieldMapping)
+	eventMapping.AddFieldMappingsAt("RoomID", idFieldMapping)
+	eventMapping.AddFieldMappingsAt("EventID", idFieldMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("Event", eventMapping)
+	indexMapping.DefaultType = "Event"
+	return indexMapping
+}

internal/fulltext/bleve_test.go

@@ -19,13 +19,20 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/internal/fulltext"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 func mustOpenIndex(t *testing.T, tempDir string) *fulltext.Search {
 	t.Helper()
-	fts, err := fulltext.New(tempDir)
+	cfg := config.Fulltext{}
+	cfg.Defaults(true)
+	if tempDir != "" {
+		cfg.IndexPath = config.Path(tempDir)
+		cfg.InMemory = false
+	}
+	fts, err := fulltext.New(cfg)
 	if err != nil {
 		t.Fatal("failed to open fulltext index:", err)
 	}
@@ -93,7 +100,7 @@ func TestOpen(t *testing.T) {
 }
 
 func TestIndex(t *testing.T) {
-	fts := mustOpenIndex(t, t.TempDir())
+	fts := mustOpenIndex(t, "")
 	defer fts.Close()
 
 	// add some data
@@ -117,7 +124,7 @@ func TestIndex(t *testing.T) {
 }
 
 func TestDelete(t *testing.T) {
-	fts := mustOpenIndex(t, t.TempDir())
+	fts := mustOpenIndex(t, "")
 	defer fts.Close()
 	eventIDs, roomIDs := mustAddTestData(t, fts, 0)
 	res1, err := fts.Search("lorem", roomIDs[:1], nil, 50, 0, false)
@@ -213,7 +220,7 @@ func TestSearch(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			f := mustOpenIndex(t, t.TempDir())
+			f := mustOpenIndex(t, "")
 			eventIDs, roomIDs := mustAddTestData(t, f, 0)
 			var searchRooms []string
 			for _, x := range tt.args.roomIndex {

setup/base/base.go

@@ -31,6 +31,7 @@ import (
 	"github.com/getsentry/sentry-go"
 	sentryhttp "github.com/getsentry/sentry-go/http"
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/fulltext"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/pushgateway"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -87,6 +88,7 @@ type BaseDendrite struct {
 	Database               *sql.DB
 	DatabaseWriter         sqlutil.Writer
 	EnableMetrics          bool
+	Fulltext               *fulltext.Search
 }
 
 const NoListener = ""
@@ -146,6 +148,14 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		logrus.WithError(err).Panicf("failed to start opentracing")
 	}
 
+	var fts *fulltext.Search
+	if cfg.SyncAPI.Fulltext.Enabled {
+		fts, err = fulltext.New(cfg.SyncAPI.Fulltext)
+		if err != nil {
+			logrus.WithError(err).Panicf("failed to create full text")
+		}
+	}
+
 	if cfg.Global.Sentry.Enabled {
 		logrus.Info("Setting up Sentry for debugging...")
 		err = sentry.Init(sentry.ClientOptions{
@@ -248,6 +258,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		Database:               db,     // set if monolith with global connection pool only
 		DatabaseWriter:         writer, // set if monolith with global connection pool only
 		EnableMetrics:          enableMetrics,
+		Fulltext:               fts,
 	}
 }
 

setup/config/config_syncapi.go

@@ -9,6 +9,8 @@ type SyncAPI struct {
 	Database DatabaseOptions `yaml:"database"`
 
 	RealIPHeader string `yaml:"real_ip_header"`
+
+	Fulltext Fulltext `yaml:"fulltext"`
 }
 
 func (c *SyncAPI) Defaults(generate bool) {
@@ -16,12 +18,14 @@ func (c *SyncAPI) Defaults(generate bool) {
 	c.InternalAPI.Connect = "http://localhost:7773"
 	c.ExternalAPI.Listen = "http://localhost:8073"
 	c.Database.Defaults(10)
+	c.Fulltext.Defaults(generate)
 	if generate {
 		c.Database.ConnectionString = "file:syncapi.db"
 	}
 }
 
 func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	c.Fulltext.Verify(configErrs, isMonolith)
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
 	}
@@ -32,3 +36,25 @@ func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect))
 	checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
 }
+
+type Fulltext struct {
+	Enabled   bool   `yaml:"enabled"`
+	IndexPath Path   `yaml:"index_path"`
+	InMemory  bool   `yaml:"in_memory"` // only useful in tests
+	Language  string `yaml:"language"`  // the language to use when analysing content
+}
+
+func (f *Fulltext) Defaults(generate bool) {
+	f.Enabled = false
+	f.IndexPath = "./fulltextindex"
+	f.Language = "en"
+	if generate {
+		f.Enabled = true
+		f.InMemory = true
+	}
+}
+
+func (f *Fulltext) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "syncapi.fulltext.index_path", string(f.IndexPath))
+	checkNotEmpty(configErrs, "syncapi.fulltext.language", f.Language)
+}