diff --git a/go.mod b/go.mod index ecdeb77fa..d4baac122 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/MFAshby/stdemuxerhook v1.0.0 github.com/Masterminds/semver/v3 v3.1.1 + github.com/blevesearch/bleve/v2 v2.3.2 github.com/codeclysm/extract v2.2.0+incompatible github.com/containerd/containerd v1.6.2 // indirect github.com/docker/docker v20.10.14+incompatible diff --git a/go.sum b/go.sum index 145c2a04c..662adebb0 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,8 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= +github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= +github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA= github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -158,10 +160,45 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blevesearch/bleve/v2 v2.3.2 h1:BJUnMhi2nrkl+vboHmKfW+9l+tJSj39HeWa5c3BN3/Y= +github.com/blevesearch/bleve/v2 v2.3.2/go.mod h1:96+xE5pZUOsr3Y4vHzV1cBC837xZCpwLlX0hrrxnvIg= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A= +github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= +github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= +github.com/blevesearch/goleveldb v1.0.1/go.mod h1:WrU8ltZbIp0wAoig/MHbrPCXSOLpe79nz5lv5nqfYrQ= +github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= +github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4= +github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE= +github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= +github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= +github.com/blevesearch/snowball v0.6.1/go.mod h1:ZF0IBg5vgpeoUhnMza2v0A/z8m1cWPlwhke08LpNusg= +github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= +github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= +github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= +github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= +github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ= +github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE= +github.com/blevesearch/zapx/v11 v11.3.3 h1:8vQMO5hdA2qPCmicIMuKS+qcvUAEh6Vcb0uve4Nh8e4= +github.com/blevesearch/zapx/v11 v11.3.3/go.mod h1:YzTfUm4kS3e8OmTXDHVV8OzC5MWPO/VPJZQgPNVb4Lc= +github.com/blevesearch/zapx/v12 v12.3.3 h1:MQO5YNI8MqdPz12ALCoXiJw5cl9QQamYZSp285Z/+Mo= +github.com/blevesearch/zapx/v12 v12.3.3/go.mod h1:RMl6lOZqF+sTxKvhQDJ5yK2LT3Mu7E2p/jGdjAaiRxs= +github.com/blevesearch/zapx/v13 v13.3.3 h1:TS4xpMK1ARPYHq+1WwuEOKMOiwvKpTK3RuWOkKlI7BE= +github.com/blevesearch/zapx/v13 v13.3.3/go.mod h1:eppobNM35U4C22yDvTuxV9xPqo10pwfP/jugL4INWG4= +github.com/blevesearch/zapx/v14 v14.3.3 h1:dqqAzGphKl0yehHKKntDHKlEMhi9B/tJrD4OsWpY7YE= +github.com/blevesearch/zapx/v14 v14.3.3/go.mod h1:zXNcVzukh0AvG57oUtT1T0ndi09H0kELNaNmekEy0jw= +github.com/blevesearch/zapx/v15 v15.3.3 h1:60oE+qsJkveLenJmbc0eaH59GWYCbJJsPDV6Z5hEoYY= +github.com/blevesearch/zapx/v15 v15.3.3/go.mod h1:C+f/97ZzTzK6vt/7sVlZdzZxKu+5+j4SrGCvr9dJzaY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= @@ -332,6 +369,8 @@ github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= +github.com/couchbase/moss v0.2.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -534,6 +573,8 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo= github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4= @@ -874,6 +915,8 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -909,6 +952,7 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -922,6 +966,7 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -1200,6 +1245,7 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -1438,6 +1484,7 @@ golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go new file mode 100644 index 000000000..c41c76aa9 --- /dev/null +++ b/internal/fulltext/bleve.go @@ -0,0 +1,166 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext + +import ( + "strings" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" +) + +// Search contains all existing bleve.Index +type Search struct { + FulltextIndex bleve.Index +} + +// IndexElement describes the layout of an element to index +type IndexElement struct { + EventID string + RoomID string + Content string + ContentType string + StreamPosition int64 +} + +// SetContentType sets i.ContentType given an identifier +func (i *IndexElement) SetContentType(v string) { + switch v { + case "m.room.message": + i.ContentType = "content.body" + case gomatrixserverlib.MRoomName: + i.ContentType = "content.name" + case gomatrixserverlib.MRoomTopic: + i.ContentType = "content.topic" + } +} + +// New opens a new/existing fulltext index +func New(cfg config.Fulltext) (fts *Search, err error) { + fts = &Search{} + fts.FulltextIndex, err = openIndex(cfg) + if err != nil { + return nil, err + } + return fts, nil +} + +// Close closes the fulltext index +func (f *Search) Close() error { + return f.FulltextIndex.Close() +} + +// FulltextIndex indexes a given element +func (f *Search) Index(e IndexElement) error { + return f.FulltextIndex.Index(e.EventID, e) +} + +// BatchIndex indexes the given elements +func (f *Search) BatchIndex(elements []IndexElement) error { + batch := f.FulltextIndex.NewBatch() + + for _, element := range elements { + err := batch.Index(element.EventID, element) + if err != nil { + return err + } + } + return f.FulltextIndex.Batch(batch) +} + +// Delete deletes an indexed element by the eventID +func (f *Search) Delete(eventID string) error { + return f.FulltextIndex.Delete(eventID) +} + +// Search searches the index given a search term, roomIDs and keys. +func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (*bleve.SearchResult, error) { + qry := bleve.NewConjunctionQuery() + termQuery := bleve.NewBooleanQuery() + + terms := strings.Split(term, " ") + for _, term := range terms { + matchQuery := bleve.NewMatchQuery(term) + matchQuery.SetField("Content") + termQuery.AddMust(matchQuery) + } + qry.AddQuery(termQuery) + + roomQuery := bleve.NewBooleanQuery() + for _, roomID := range roomIDs { + roomSearch := bleve.NewMatchQuery(roomID) + roomSearch.SetField("RoomID") + roomQuery.AddShould(roomSearch) + } + if len(roomIDs) > 0 { + qry.AddQuery(roomQuery) + } + keyQuery := bleve.NewBooleanQuery() + for _, key := range keys { + keySearch := bleve.NewMatchQuery(key) + keySearch.SetField("ContentType") + keyQuery.AddShould(keySearch) + } + if len(keys) > 0 { + qry.AddQuery(keyQuery) + } + + s := bleve.NewSearchRequestOptions(qry, limit, from, false) + s.Fields = []string{"*"} + s.SortBy([]string{"_score"}) + if orderByStreamPos { + s.SortBy([]string{"-StreamPosition"}) + } + + return f.FulltextIndex.Search(s) +} + +func openIndex(cfg config.Fulltext) (bleve.Index, error) { + m := getMapping(cfg) + if cfg.InMemory { + return bleve.NewMemOnly(m) + } + if index, err := bleve.Open(string(cfg.IndexPath)); err == nil { + return index, nil + } + + index, err := bleve.New(string(cfg.IndexPath), m) + if err != nil { + return nil, err + } + return index, nil +} + +func getMapping(cfg config.Fulltext) *mapping.IndexMappingImpl { + enFieldMapping := bleve.NewTextFieldMapping() + enFieldMapping.Analyzer = cfg.Language + + eventMapping := bleve.NewDocumentMapping() + eventMapping.AddFieldMappingsAt("Content", enFieldMapping) + eventMapping.AddFieldMappingsAt("StreamPosition", bleve.NewNumericFieldMapping()) + + // Index entries as is + idFieldMapping := bleve.NewKeywordFieldMapping() + eventMapping.AddFieldMappingsAt("ContentType", idFieldMapping) + eventMapping.AddFieldMappingsAt("RoomID", idFieldMapping) + eventMapping.AddFieldMappingsAt("EventID", idFieldMapping) + + indexMapping := bleve.NewIndexMapping() + indexMapping.AddDocumentMapping("Event", eventMapping) + indexMapping.DefaultType = "Event" + return indexMapping +} diff --git a/internal/fulltext/bleve_test.go b/internal/fulltext/bleve_test.go new file mode 100644 index 000000000..a819733e0 --- /dev/null +++ b/internal/fulltext/bleve_test.go @@ -0,0 +1,246 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext_test + +import ( + "reflect" + "testing" + + "github.com/matrix-org/dendrite/internal/fulltext" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +func mustOpenIndex(t *testing.T, tempDir string) *fulltext.Search { + t.Helper() + cfg := config.Fulltext{} + cfg.Defaults(true) + if tempDir != "" { + cfg.IndexPath = config.Path(tempDir) + cfg.InMemory = false + } + fts, err := fulltext.New(cfg) + if err != nil { + t.Fatal("failed to open fulltext index:", err) + } + return fts +} + +func mustAddTestData(t *testing.T, fts *fulltext.Search, firstStreamPos int64) (eventIDs, roomIDs []string) { + t.Helper() + // create some more random data + var batchItems []fulltext.IndexElement + streamPos := firstStreamPos + + wantRoomID := util.RandomString(16) + + for i := 0; i < 30; i++ { + streamPos++ + eventID := util.RandomString(16) + // Create more data for the first room + if i > 15 { + wantRoomID = util.RandomString(16) + } + e := fulltext.IndexElement{ + EventID: eventID, + RoomID: wantRoomID, + Content: "lorem ipsum", + StreamPosition: streamPos, + } + e.SetContentType("m.room.message") + batchItems = append(batchItems, e) + roomIDs = append(roomIDs, wantRoomID) + eventIDs = append(eventIDs, eventID) + } + e := fulltext.IndexElement{ + EventID: util.RandomString(16), + RoomID: wantRoomID, + Content: "Roomname testing", + StreamPosition: streamPos, + } + e.SetContentType(gomatrixserverlib.MRoomName) + batchItems = append(batchItems, e) + e = fulltext.IndexElement{ + EventID: util.RandomString(16), + RoomID: wantRoomID, + Content: "Room topic fulltext", + StreamPosition: streamPos, + } + e.SetContentType(gomatrixserverlib.MRoomTopic) + batchItems = append(batchItems, e) + if err := fts.BatchIndex(batchItems); err != nil { + t.Fatalf("failed to batch insert elements: %v", err) + } + return eventIDs, roomIDs +} + +func TestOpen(t *testing.T) { + dataDir := t.TempDir() + fts := mustOpenIndex(t, dataDir) + if err := fts.Close(); err != nil { + t.Fatal("unable to close fulltext index", err) + } + + // open existing index + fts = mustOpenIndex(t, dataDir) + defer fts.Close() +} + +func TestIndex(t *testing.T) { + fts := mustOpenIndex(t, "") + defer fts.Close() + + // add some data + var streamPos int64 = 1 + roomID := util.RandomString(8) + eventID := util.RandomString(16) + e := fulltext.IndexElement{ + EventID: eventID, + RoomID: roomID, + Content: "lorem ipsum", + StreamPosition: streamPos, + } + e.SetContentType("m.room.message") + + if err := fts.Index(e); err != nil { + t.Fatal("failed to index element", err) + } + + // create some more random data + mustAddTestData(t, fts, streamPos) +} + +func TestDelete(t *testing.T) { + fts := mustOpenIndex(t, "") + defer fts.Close() + eventIDs, roomIDs := mustAddTestData(t, fts, 0) + res1, err := fts.Search("lorem", roomIDs[:1], nil, 50, 0, false) + if err != nil { + t.Fatal(err) + } + + if err = fts.Delete(eventIDs[0]); err != nil { + t.Fatal(err) + } + + res2, err := fts.Search("lorem", roomIDs[:1], nil, 50, 0, false) + if err != nil { + t.Fatal(err) + } + + if res1.Total <= res2.Total { + t.Fatalf("got unexpected result: %d <= %d", res1.Total, res2.Total) + } +} + +func TestSearch(t *testing.T) { + type args struct { + term string + keys []string + limit int + from int + orderByStreamPos bool + roomIndex []int + } + tests := []struct { + name string + args args + wantCount int + wantErr bool + }{ + { + name: "Can search for many results in one room", + wantCount: 16, + args: args{ + term: "lorem", + roomIndex: []int{0}, + limit: 20, + }, + }, + { + name: "Can search for one result in one room", + wantCount: 1, + args: args{ + term: "lorem", + roomIndex: []int{16}, + limit: 20, + }, + }, + { + name: "Can search for many results in multiple rooms", + wantCount: 17, + args: args{ + term: "lorem", + roomIndex: []int{0, 16}, + limit: 20, + }, + }, + { + name: "Can search for many results in all rooms, reversed", + wantCount: 30, + args: args{ + term: "lorem", + limit: 30, + orderByStreamPos: true, + }, + }, + { + name: "Can search for specific search room name", + wantCount: 1, + args: args{ + term: "testing", + roomIndex: []int{}, + limit: 20, + keys: []string{"content.name"}, + }, + }, + { + name: "Can search for specific search room topic", + wantCount: 1, + args: args{ + term: "fulltext", + roomIndex: []int{}, + limit: 20, + keys: []string{"content.topic"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := mustOpenIndex(t, "") + eventIDs, roomIDs := mustAddTestData(t, f, 0) + var searchRooms []string + for _, x := range tt.args.roomIndex { + searchRooms = append(searchRooms, roomIDs[x]) + } + t.Logf("searching in rooms: %v - %v\n", searchRooms, tt.args.keys) + + got, err := f.Search(tt.args.term, searchRooms, tt.args.keys, tt.args.limit, tt.args.from, tt.args.orderByStreamPos) + if (err != nil) != tt.wantErr { + t.Errorf("Search() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(len(got.Hits), tt.wantCount) { + t.Errorf("Search() got = %v, want %v", len(got.Hits), tt.wantCount) + } + if tt.args.orderByStreamPos { + if got.Hits[0].ID != eventIDs[29] { + t.Fatalf("expected ID %s, got %s", eventIDs[29], got.Hits[0].ID) + } + } + }) + } +} diff --git a/setup/base/base.go b/setup/base/base.go index 5cbd7da9c..9a4b8ea64 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -31,6 +31,7 @@ import ( "github.com/getsentry/sentry-go" sentryhttp "github.com/getsentry/sentry-go/http" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -87,6 +88,7 @@ type BaseDendrite struct { Database *sql.DB DatabaseWriter sqlutil.Writer EnableMetrics bool + Fulltext *fulltext.Search } const NoListener = "" @@ -146,6 +148,15 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base logrus.WithError(err).Panicf("failed to start opentracing") } + var fts *fulltext.Search + isSyncOrMonolith := componentName == "syncapi" || isMonolith + if cfg.SyncAPI.Fulltext.Enabled && isSyncOrMonolith { + fts, err = fulltext.New(cfg.SyncAPI.Fulltext) + if err != nil { + logrus.WithError(err).Panicf("failed to create full text") + } + } + if cfg.Global.Sentry.Enabled { logrus.Info("Setting up Sentry for debugging...") err = sentry.Init(sentry.ClientOptions{ @@ -248,6 +259,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base Database: db, // set if monolith with global connection pool only DatabaseWriter: writer, // set if monolith with global connection pool only EnableMetrics: enableMetrics, + Fulltext: fts, } } diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go index 7d5e3808a..371ae2708 100644 --- a/setup/config/config_syncapi.go +++ b/setup/config/config_syncapi.go @@ -9,6 +9,8 @@ type SyncAPI struct { Database DatabaseOptions `yaml:"database"` RealIPHeader string `yaml:"real_ip_header"` + + Fulltext Fulltext `yaml:"fulltext"` } func (c *SyncAPI) Defaults(generate bool) { @@ -16,12 +18,14 @@ func (c *SyncAPI) Defaults(generate bool) { c.InternalAPI.Connect = "http://localhost:7773" c.ExternalAPI.Listen = "http://localhost:8073" c.Database.Defaults(10) + c.Fulltext.Defaults(generate) if generate { c.Database.ConnectionString = "file:syncapi.db" } } func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + c.Fulltext.Verify(configErrs, isMonolith) if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString)) } @@ -32,3 +36,25 @@ func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect)) checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen)) } + +type Fulltext struct { + Enabled bool `yaml:"enabled"` + IndexPath Path `yaml:"index_path"` + InMemory bool `yaml:"in_memory"` // only useful in tests + Language string `yaml:"language"` // the language to use when analysing content +} + +func (f *Fulltext) Defaults(generate bool) { + f.Enabled = false + f.IndexPath = "./fulltextindex" + f.Language = "en" + if generate { + f.Enabled = true + f.InMemory = true + } +} + +func (f *Fulltext) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "syncapi.fulltext.index_path", string(f.IndexPath)) + checkNotEmpty(configErrs, "syncapi.fulltext.language", f.Language) +} diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index 6594e6941..8dd563098 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -27,6 +27,7 @@ var ( OutputReadUpdate = "OutputReadUpdate" RequestPresence = "GetPresence" OutputPresenceEvent = "OutputPresenceEvent" + InputFulltextReindex = "InputFulltextReindex" ) var safeCharacters = regexp.MustCompile("[^A-Za-z0-9$]+") diff --git a/test/testrig/base.go b/test/testrig/base.go index facb49f3e..daa9dfe71 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -35,8 +35,9 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f switch dbType { case test.DBTypePostgres: - cfg.Global.Defaults(true) // autogen a signing key - cfg.MediaAPI.Defaults(true) // autogen a media path + cfg.Global.Defaults(true) // autogen a signing key + cfg.MediaAPI.Defaults(true) // autogen a media path + cfg.SyncAPI.Fulltext.Defaults(true) // use in memory fts // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use // the file system event with InMemory=true :( cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType) @@ -85,6 +86,7 @@ func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nat cfg.Defaults(true) } cfg.Global.JetStream.InMemory = true + cfg.SyncAPI.Fulltext.InMemory = true base := base.NewBaseDendrite(cfg, "Tests") js, jc := base.NATS.Prepare(base.ProcessContext, &cfg.Global.JetStream) return base, js, jc