mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-06 13:43:09 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add config, move to base

Browse source
This commit is contained in:
Till Faelligen 2022-05-19 10:10:15 +02:00
parent baacb8ac6e
commit 74939ffb61
4 changed files with 79 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
internal/fulltext/bleve.go
View file

@ -18,7 +18,8 @@ import (
"strings"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/lang/en"
"github.com/blevesearch/bleve/v2/mapping"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
@ -49,9 +50,9 @@ func (i *IndexElement) SetContentType(v string) {
}
// New opens a new/existing fulltext index
func New(path string) (fts *Search, err error) {
func New(cfg config.Fulltext) (fts *Search, err error) {
fts = &Search{}
fts.FulltextIndex, err = openIndex(path)
fts.FulltextIndex, err = openIndex(cfg)
if err != nil {
return nil, err
}
@ -128,30 +129,38 @@ func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, or
return f.FulltextIndex.Search(s)
}
func openIndex(path string) (bleve.Index, error) {
if index, err := bleve.Open(path); err == nil {
func openIndex(cfg config.Fulltext) (bleve.Index, error) {
m := getMapping(cfg)
if cfg.InMemory {
return bleve.NewMemOnly(m)
}
if index, err := bleve.Open(string(cfg.IndexPath)); err == nil {
return index, nil
}
enFieldMapping := bleve.NewTextFieldMapping()
enFieldMapping.Analyzer = en.AnalyzerName
eventMapping := bleve.NewDocumentMapping()
eventMapping.AddFieldMappingsAt("Content", enFieldMapping)
eventMapping.AddFieldMappingsAt("StreamPosition", bleve.NewNumericFieldMapping())
idFieldMapping := bleve.NewKeywordFieldMapping()
eventMapping.AddFieldMappingsAt("ContentType", idFieldMapping)
eventMapping.AddFieldMappingsAt("RoomID", idFieldMapping)
eventMapping.AddFieldMappingsAt("EventID", idFieldMapping)
mapping := bleve.NewIndexMapping()
mapping.AddDocumentMapping("Event", eventMapping)
mapping.DefaultType = "Event"
index, err := bleve.New(path, mapping)
index, err := bleve.New(string(cfg.IndexPath), m)
if err != nil {
return nil, err
}
return index, nil
}
func getMapping(cfg config.Fulltext) *mapping.IndexMappingImpl {
enFieldMapping := bleve.NewTextFieldMapping()
enFieldMapping.Analyzer = cfg.Language
eventMapping := bleve.NewDocumentMapping()
eventMapping.AddFieldMappingsAt("Content", enFieldMapping)
eventMapping.AddFieldMappingsAt("StreamPosition", bleve.NewNumericFieldMapping())
// Index entries as is
idFieldMapping := bleve.NewKeywordFieldMapping()
eventMapping.AddFieldMappingsAt("ContentType", idFieldMapping)
eventMapping.AddFieldMappingsAt("RoomID", idFieldMapping)
eventMapping.AddFieldMappingsAt("EventID", idFieldMapping)
indexMapping := bleve.NewIndexMapping()
indexMapping.AddDocumentMapping("Event", eventMapping)
indexMapping.DefaultType = "Event"
return indexMapping
}

15
internal/fulltext/bleve_test.go
View file

@ -19,13 +19,20 @@ import (
"testing"
"github.com/matrix-org/dendrite/internal/fulltext"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
func mustOpenIndex(t *testing.T, tempDir string) *fulltext.Search {
t.Helper()
fts, err := fulltext.New(tempDir)
cfg := config.Fulltext{}
cfg.Defaults(true)
if tempDir != "" {
cfg.IndexPath = config.Path(tempDir)
cfg.InMemory = false
}
fts, err := fulltext.New(cfg)
if err != nil {
t.Fatal("failed to open fulltext index:", err)
}
@ -93,7 +100,7 @@ func TestOpen(t *testing.T) {
}
func TestIndex(t *testing.T) {
fts := mustOpenIndex(t, t.TempDir())
fts := mustOpenIndex(t, "")
defer fts.Close()
// add some data
@ -117,7 +124,7 @@ func TestIndex(t *testing.T) {
}
func TestDelete(t *testing.T) {
fts := mustOpenIndex(t, t.TempDir())
fts := mustOpenIndex(t, "")
defer fts.Close()
eventIDs, roomIDs := mustAddTestData(t, fts, 0)
res1, err := fts.Search("lorem", roomIDs[:1], nil, 50, 0, false)
@ -213,7 +220,7 @@ func TestSearch(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := mustOpenIndex(t, t.TempDir())
f := mustOpenIndex(t, "")
eventIDs, roomIDs := mustAddTestData(t, f, 0)
var searchRooms []string
for _, x := range tt.args.roomIndex {

11
setup/base/base.go
View file

@ -31,6 +31,7 @@ import (
"github.com/getsentry/sentry-go"
sentryhttp "github.com/getsentry/sentry-go/http"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/fulltext"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/pushgateway"
"github.com/matrix-org/dendrite/internal/sqlutil"
@ -87,6 +88,7 @@ type BaseDendrite struct {
Database *sql.DB
DatabaseWriter sqlutil.Writer
EnableMetrics bool
Fulltext *fulltext.Search
}
const NoListener = ""
@ -146,6 +148,14 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
logrus.WithError(err).Panicf("failed to start opentracing")
}
var fts *fulltext.Search
if cfg.SyncAPI.Fulltext.Enabled {
fts, err = fulltext.New(cfg.SyncAPI.Fulltext)
if err != nil {
logrus.WithError(err).Panicf("failed to create full text")
}
}
if cfg.Global.Sentry.Enabled {
logrus.Info("Setting up Sentry for debugging...")
err = sentry.Init(sentry.ClientOptions{
@ -248,6 +258,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
Database: db, // set if monolith with global connection pool only
DatabaseWriter: writer, // set if monolith with global connection pool only
EnableMetrics: enableMetrics,
Fulltext: fts,
}
}

26
setup/config/config_syncapi.go
View file

@ -9,6 +9,8 @@ type SyncAPI struct {
Database DatabaseOptions `yaml:"database"`
RealIPHeader string `yaml:"real_ip_header"`
Fulltext Fulltext `yaml:"fulltext"`
}
func (c *SyncAPI) Defaults(generate bool) {
@ -16,12 +18,14 @@ func (c *SyncAPI) Defaults(generate bool) {
c.InternalAPI.Connect = "http://localhost:7773"
c.ExternalAPI.Listen = "http://localhost:8073"
c.Database.Defaults(10)
c.Fulltext.Defaults(generate)
if generate {
c.Database.ConnectionString = "file:syncapi.db"
}
}
func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
c.Fulltext.Verify(configErrs, isMonolith)
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
}
@ -32,3 +36,25 @@ func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect))
checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
}
type Fulltext struct {
Enabled bool `yaml:"enabled"`
IndexPath Path `yaml:"index_path"`
InMemory bool `yaml:"in_memory"` // only useful in tests
Language string `yaml:"language"` // the language to use when analysing content
}
func (f *Fulltext) Defaults(generate bool) {
f.Enabled = false
f.IndexPath = "./fulltextindex"
f.Language = "en"
if generate {
f.Enabled = true
f.InMemory = true
}
}
func (f *Fulltext) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "syncapi.fulltext.index_path", string(f.IndexPath))
checkNotEmpty(configErrs, "syncapi.fulltext.language", f.Language)
}