mirror of
https://github.com/matrix-org/dendrite.git
synced 2026-01-07 14:13:11 -06:00
FTS experiments
This commit is contained in:
Till Faelligen
parent
fc670f03a2
commit
18650d4e41
1
go.mod
1
go.mod
|
|
@ -11,6 +11,7 @@ require (
|
|||
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
|
||||
github.com/MFAshby/stdemuxerhook v1.0.0
|
||||
github.com/Masterminds/semver/v3 v3.1.1
|
||||
github.com/blevesearch/bleve/v2 v2.3.2 // indirect
|
||||
github.com/codeclysm/extract v2.2.0+incompatible
|
||||
github.com/containerd/containerd v1.6.2 // indirect
|
||||
github.com/docker/docker v20.10.14+incompatible
|
||||
|
|
|
|||
47
go.sum
47
go.sum
|
|
@ -114,6 +114,8 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
|
|||
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
|
||||
github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo=
|
||||
github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
|
||||
github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA=
|
||||
github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30=
|
||||
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
|
||||
|
|
@ -158,10 +160,45 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
|||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
|
||||
github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
|
||||
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
github.com/blevesearch/bleve/v2 v2.3.2 h1:BJUnMhi2nrkl+vboHmKfW+9l+tJSj39HeWa5c3BN3/Y=
|
||||
github.com/blevesearch/bleve/v2 v2.3.2/go.mod h1:96+xE5pZUOsr3Y4vHzV1cBC837xZCpwLlX0hrrxnvIg=
|
||||
github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk=
|
||||
github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4=
|
||||
github.com/blevesearch/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
|
||||
github.com/blevesearch/goleveldb v1.0.1/go.mod h1:WrU8ltZbIp0wAoig/MHbrPCXSOLpe79nz5lv5nqfYrQ=
|
||||
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
|
||||
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
|
||||
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
|
||||
github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4=
|
||||
github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE=
|
||||
github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac=
|
||||
github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
|
||||
github.com/blevesearch/snowball v0.6.1/go.mod h1:ZF0IBg5vgpeoUhnMza2v0A/z8m1cWPlwhke08LpNusg=
|
||||
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
|
||||
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU=
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q=
|
||||
github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ=
|
||||
github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE=
|
||||
github.com/blevesearch/zapx/v11 v11.3.3 h1:8vQMO5hdA2qPCmicIMuKS+qcvUAEh6Vcb0uve4Nh8e4=
|
||||
github.com/blevesearch/zapx/v11 v11.3.3/go.mod h1:YzTfUm4kS3e8OmTXDHVV8OzC5MWPO/VPJZQgPNVb4Lc=
|
||||
github.com/blevesearch/zapx/v12 v12.3.3 h1:MQO5YNI8MqdPz12ALCoXiJw5cl9QQamYZSp285Z/+Mo=
|
||||
github.com/blevesearch/zapx/v12 v12.3.3/go.mod h1:RMl6lOZqF+sTxKvhQDJ5yK2LT3Mu7E2p/jGdjAaiRxs=
|
||||
github.com/blevesearch/zapx/v13 v13.3.3 h1:TS4xpMK1ARPYHq+1WwuEOKMOiwvKpTK3RuWOkKlI7BE=
|
||||
github.com/blevesearch/zapx/v13 v13.3.3/go.mod h1:eppobNM35U4C22yDvTuxV9xPqo10pwfP/jugL4INWG4=
|
||||
github.com/blevesearch/zapx/v14 v14.3.3 h1:dqqAzGphKl0yehHKKntDHKlEMhi9B/tJrD4OsWpY7YE=
|
||||
github.com/blevesearch/zapx/v14 v14.3.3/go.mod h1:zXNcVzukh0AvG57oUtT1T0ndi09H0kELNaNmekEy0jw=
|
||||
github.com/blevesearch/zapx/v15 v15.3.3 h1:60oE+qsJkveLenJmbc0eaH59GWYCbJJsPDV6Z5hEoYY=
|
||||
github.com/blevesearch/zapx/v15 v15.3.3/go.mod h1:C+f/97ZzTzK6vt/7sVlZdzZxKu+5+j4SrGCvr9dJzaY=
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
|
||||
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
|
||||
github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
|
||||
|
|
@ -332,6 +369,8 @@ github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+
|
|||
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
|
||||
github.com/couchbase/moss v0.2.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
|
|
@ -534,6 +573,8 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx
|
|||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
|
||||
github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo=
|
||||
github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4=
|
||||
|
|
@ -874,6 +915,8 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
|
|||
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
|
||||
github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
|
||||
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
|
||||
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
|
||||
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
|
|
@ -909,6 +952,7 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v
|
|||
github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
|
|
@ -922,6 +966,7 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
|
|||
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
|
||||
github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
|
|
@ -1200,6 +1245,7 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go
|
|||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
|
||||
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
|
||||
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
|
||||
go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
|
||||
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
|
||||
|
|
@ -1438,6 +1484,7 @@ golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
|
|||
56
internal/fulltext/bleve.go
Normal file
56
internal/fulltext/bleve.go
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
package fulltext
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
)
|
||||
|
||||
type Search struct {
|
||||
Index bleve.Index
|
||||
}
|
||||
|
||||
type IndexElement struct {
|
||||
EventID string
|
||||
Type string
|
||||
RoomID string
|
||||
Content string
|
||||
}
|
||||
|
||||
func New(path string) (*Search, error) {
|
||||
fts := &Search{}
|
||||
var err error
|
||||
fts.Index, err = openIndex(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fts, nil
|
||||
}
|
||||
|
||||
func (f *Search) Close() error {
|
||||
return f.Index.Close()
|
||||
}
|
||||
|
||||
func (f *Search) IndexElement(e IndexElement) error {
|
||||
return f.Index.Index(e.EventID, e)
|
||||
}
|
||||
|
||||
func (f *Search) DeleteElement(eventID string) error {
|
||||
return f.Index.Delete(eventID)
|
||||
}
|
||||
|
||||
func (f *Search) Search(term string) (*bleve.SearchResult, error) {
|
||||
qry := bleve.NewQueryStringQuery(term)
|
||||
search := bleve.NewSearchRequest(qry)
|
||||
return f.Index.Search(search)
|
||||
}
|
||||
|
||||
func openIndex(path string) (bleve.Index, error) {
|
||||
if index, err := bleve.Open(path); err == nil {
|
||||
return index, nil
|
||||
}
|
||||
|
||||
index, err := bleve.New(path, bleve.NewIndexMapping())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return index, nil
|
||||
}
|
||||
75
internal/fulltext/bleve_test.go
Normal file
75
internal/fulltext/bleve_test.go
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
package fulltext
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/matrix-org/util"
|
||||
)
|
||||
|
||||
func TestSearch(t *testing.T) {
|
||||
// create new index
|
||||
dataDir := t.TempDir()
|
||||
fts, err := New(dataDir)
|
||||
if err != nil {
|
||||
t.Fatal("failed to open fulltext index:", err)
|
||||
}
|
||||
if err = fts.Close(); err != nil {
|
||||
t.Fatal("unable to close fulltext index", err)
|
||||
}
|
||||
|
||||
// open existing index
|
||||
fts, err = New(dataDir)
|
||||
if err != nil {
|
||||
t.Fatal("failed to open fulltext index:", err)
|
||||
}
|
||||
defer fts.Close()
|
||||
if fts == nil {
|
||||
t.Fatal("fts is nil")
|
||||
}
|
||||
|
||||
// add some data
|
||||
e := IndexElement{
|
||||
EventID: util.RandomString(16),
|
||||
Type: "m.room.message",
|
||||
RoomID: util.RandomString(8),
|
||||
Content: "lorem ipsum",
|
||||
}
|
||||
|
||||
if err = fts.IndexElement(e); err != nil {
|
||||
t.Fatal("failed to index element", err)
|
||||
}
|
||||
|
||||
eventID := util.RandomString(16)
|
||||
e = IndexElement{
|
||||
EventID: eventID,
|
||||
Type: "m.room.message",
|
||||
RoomID: util.RandomString(8),
|
||||
Content: "lorem ipsum",
|
||||
}
|
||||
|
||||
if err = fts.IndexElement(e); err != nil {
|
||||
t.Fatal("failed to index element", err)
|
||||
}
|
||||
|
||||
// search data
|
||||
res, err := fts.Search("lorem")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 2 {
|
||||
t.Fatalf("expected %d results, got %d", 2, res.Total)
|
||||
}
|
||||
|
||||
// remove element
|
||||
if err = fts.DeleteElement(eventID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
res, err = fts.Search("lorem")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 1 {
|
||||
t.Fatalf("expected %d results, got %d", 2, res.Total)
|
||||
}
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ import (
|
|||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/matrix-org/dendrite/internal/caching"
|
||||
"github.com/matrix-org/dendrite/internal/fulltext"
|
||||
"github.com/matrix-org/dendrite/internal/httputil"
|
||||
"github.com/matrix-org/dendrite/roomserver/api"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
|
|
@ -40,6 +41,7 @@ func Setup(
|
|||
rsAPI api.SyncRoomserverAPI,
|
||||
cfg *config.SyncAPI,
|
||||
lazyLoadCache caching.LazyLoadCache,
|
||||
fts *fulltext.Search,
|
||||
) {
|
||||
v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
|
||||
|
||||
|
|
@ -95,4 +97,10 @@ func Setup(
|
|||
)
|
||||
}),
|
||||
).Methods(http.MethodGet, http.MethodOptions)
|
||||
|
||||
v3mux.Handle("/search",
|
||||
httputil.MakeAuthAPI("search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
|
||||
return Search(req, device, fts)
|
||||
}),
|
||||
).Methods(http.MethodPost, http.MethodOptions)
|
||||
}
|
||||
|
|
|
|||
122
syncapi/routing/search.go
Normal file
122
syncapi/routing/search.go
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
package routing
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/matrix-org/dendrite/clientapi/httputil"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/internal/fulltext"
|
||||
"github.com/matrix-org/dendrite/userapi/api"
|
||||
"github.com/matrix-org/util"
|
||||
)
|
||||
|
||||
func Search(req *http.Request, device *api.Device, fts *fulltext.Search) util.JSONResponse {
|
||||
var searchReq SearchRequest
|
||||
resErr := httputil.UnmarshalJSONRequest(req, &searchReq)
|
||||
if resErr != nil {
|
||||
return *resErr
|
||||
}
|
||||
|
||||
result, err := fts.Search(searchReq.SearchCategories.RoomEvents.SearchTerm)
|
||||
if err != nil {
|
||||
return jsonerror.InternalServerError()
|
||||
}
|
||||
|
||||
results := []Results{}
|
||||
for _, x := range result.Hits {
|
||||
results = append(results, Results{
|
||||
Rank: x.Score,
|
||||
Result: Result{
|
||||
Content: Content{
|
||||
Body: x.String(),
|
||||
},
|
||||
EventID: x.ID,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
res := SearchResponse{
|
||||
SearchCategories: SearchCategories{
|
||||
RoomEvents: RoomEvents{
|
||||
Count: len(results),
|
||||
Groups: Groups{},
|
||||
Results: results,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusOK,
|
||||
JSON: res,
|
||||
}
|
||||
}
|
||||
|
||||
type SearchRequest struct {
|
||||
SearchCategories struct {
|
||||
RoomEvents struct {
|
||||
Filter struct {
|
||||
NotRooms []interface{} `json:"not_rooms"`
|
||||
NotSenders []interface{} `json:"not_senders"`
|
||||
NotTypes []interface{} `json:"not_types"`
|
||||
Rooms []interface{} `json:"rooms"`
|
||||
Senders []interface{} `json:"senders"`
|
||||
Types []interface{} `json:"types"`
|
||||
} `json:"filter"`
|
||||
Groupings struct {
|
||||
GroupBy []struct {
|
||||
Key string `json:"key"`
|
||||
} `json:"group_by"`
|
||||
} `json:"groupings"`
|
||||
Keys []string `json:"keys"`
|
||||
OrderBy string `json:"order_by"`
|
||||
SearchTerm string `json:"search_term"`
|
||||
} `json:"room_events"`
|
||||
} `json:"search_categories"`
|
||||
}
|
||||
|
||||
type SearchResponse struct {
|
||||
SearchCategories SearchCategories `json:"search_categories"`
|
||||
}
|
||||
type RoomResult struct {
|
||||
NextBatch string `json:"next_batch"`
|
||||
Order int `json:"order"`
|
||||
Results []string `json:"results"`
|
||||
}
|
||||
type RoomID struct {
|
||||
Room RoomResult
|
||||
}
|
||||
type Groups struct {
|
||||
RoomID RoomID `json:"room_id"`
|
||||
}
|
||||
type Content struct {
|
||||
Body string `json:"body"`
|
||||
Format string `json:"format"`
|
||||
FormattedBody string `json:"formatted_body"`
|
||||
Msgtype string `json:"msgtype"`
|
||||
}
|
||||
type Unsigned struct {
|
||||
Age int `json:"age"`
|
||||
}
|
||||
type Result struct {
|
||||
Content Content `json:"content"`
|
||||
EventID string `json:"event_id"`
|
||||
OriginServerTs int64 `json:"origin_server_ts"`
|
||||
RoomID string `json:"room_id"`
|
||||
Sender string `json:"sender"`
|
||||
Type string `json:"type"`
|
||||
Unsigned Unsigned `json:"unsigned"`
|
||||
}
|
||||
type Results struct {
|
||||
Rank float64 `json:"rank"`
|
||||
Result Result `json:"result"`
|
||||
}
|
||||
type RoomEvents struct {
|
||||
Count int `json:"count"`
|
||||
Groups Groups `json:"groups"`
|
||||
Highlights []string `json:"highlights"`
|
||||
NextBatch string `json:"next_batch"`
|
||||
Results []Results `json:"results"`
|
||||
}
|
||||
type SearchCategories struct {
|
||||
RoomEvents RoomEvents `json:"room_events"`
|
||||
}
|
||||
|
|
@ -20,6 +20,7 @@ import (
|
|||
|
||||
// Import the postgres database driver.
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/matrix-org/dendrite/internal/fulltext"
|
||||
"github.com/matrix-org/dendrite/internal/sqlutil"
|
||||
"github.com/matrix-org/dendrite/setup/base"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
|
|
@ -36,7 +37,7 @@ type SyncServerDatasource struct {
|
|||
}
|
||||
|
||||
// NewDatabase creates a new sync server database
|
||||
func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) {
|
||||
func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, fts *fulltext.Search) (*SyncServerDatasource, error) {
|
||||
var d SyncServerDatasource
|
||||
var err error
|
||||
if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
|
||||
|
|
@ -121,6 +122,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
|
|||
NotificationData: notificationData,
|
||||
Ignores: ignores,
|
||||
Presence: presence,
|
||||
FTS: fts,
|
||||
}
|
||||
return &d, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,9 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/matrix-org/dendrite/internal/fulltext"
|
||||
userapi "github.com/matrix-org/dendrite/userapi/api"
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
"github.com/matrix-org/dendrite/internal/eventutil"
|
||||
"github.com/matrix-org/dendrite/internal/sqlutil"
|
||||
|
|
@ -50,6 +52,7 @@ type Database struct {
|
|||
NotificationData tables.NotificationData
|
||||
Ignores tables.Ignores
|
||||
Presence tables.Presence
|
||||
FTS *fulltext.Search
|
||||
}
|
||||
|
||||
func (d *Database) readOnlySnapshot(ctx context.Context) (*sql.Tx, error) {
|
||||
|
|
@ -391,6 +394,30 @@ func (d *Database) WriteEvent(
|
|||
return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition, topoPosition)
|
||||
})
|
||||
|
||||
e := fulltext.IndexElement{
|
||||
EventID: ev.EventID(),
|
||||
Type: ev.Type(),
|
||||
RoomID: ev.RoomID(),
|
||||
}
|
||||
|
||||
switch ev.Type() {
|
||||
case "m.room.message":
|
||||
e.Content = gjson.GetBytes(ev.Content(), "body").String()
|
||||
case gomatrixserverlib.MRoomName:
|
||||
e.Content = gjson.GetBytes(ev.Content(), "name").String()
|
||||
case gomatrixserverlib.MRoomTopic:
|
||||
e.Content = gjson.GetBytes(ev.Content(), "topic").String()
|
||||
case gomatrixserverlib.MRoomRedaction:
|
||||
if err := d.FTS.DeleteElement(ev.Redacts()); err != nil {
|
||||
logrus.WithError(err).Warn("failed to delete entry from fulltext index")
|
||||
}
|
||||
}
|
||||
if e.Content != "" {
|
||||
if err := d.FTS.IndexElement(e); err != nil {
|
||||
logrus.WithError(err).Warn("failed to write to fulltext index")
|
||||
}
|
||||
}
|
||||
|
||||
return pduPosition, returnErr
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ package sqlite3
|
|||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/matrix-org/dendrite/internal/fulltext"
|
||||
"github.com/matrix-org/dendrite/internal/sqlutil"
|
||||
"github.com/matrix-org/dendrite/setup/base"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
|
|
@ -36,7 +37,7 @@ type SyncServerDatasource struct {
|
|||
|
||||
// NewDatabase creates a new sync server database
|
||||
// nolint: gocyclo
|
||||
func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) {
|
||||
func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, fts *fulltext.Search) (*SyncServerDatasource, error) {
|
||||
var d SyncServerDatasource
|
||||
var err error
|
||||
if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil {
|
||||
|
|
@ -45,6 +46,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
|
|||
if err = d.prepare(dbProperties); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.FTS = fts
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ package storage
|
|||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/matrix-org/dendrite/internal/fulltext"
|
||||
"github.com/matrix-org/dendrite/setup/base"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/postgres"
|
||||
|
|
@ -27,12 +28,12 @@ import (
|
|||
)
|
||||
|
||||
// NewSyncServerDatasource opens a database connection.
|
||||
func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
|
||||
func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, fts *fulltext.Search) (Database, error) {
|
||||
switch {
|
||||
case dbProperties.ConnectionString.IsSQLite():
|
||||
return sqlite3.NewDatabase(base, dbProperties)
|
||||
return sqlite3.NewDatabase(base, dbProperties, fts)
|
||||
case dbProperties.ConnectionString.IsPostgres():
|
||||
return postgres.NewDatabase(base, dbProperties)
|
||||
return postgres.NewDatabase(base, dbProperties, fts)
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected database type")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/matrix-org/dendrite/internal/fulltext"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage"
|
||||
"github.com/matrix-org/dendrite/syncapi/types"
|
||||
|
|
@ -16,14 +17,21 @@ import (
|
|||
var ctx = context.Background()
|
||||
|
||||
func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
|
||||
fts, err := fulltext.New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal("failed to create full text")
|
||||
}
|
||||
connStr, close := test.PrepareDBConnectionString(t, dbType)
|
||||
db, err := storage.NewSyncServerDatasource(nil, &config.DatabaseOptions{
|
||||
ConnectionString: config.DataSource(connStr),
|
||||
})
|
||||
}, fts)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSyncServerDatasource returned %s", err)
|
||||
}
|
||||
return db, close
|
||||
return db, func() {
|
||||
close()
|
||||
fts.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserverlib.HeaderedEvent) (positions []types.StreamPosition) {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"context"
|
||||
|
||||
"github.com/matrix-org/dendrite/internal/caching"
|
||||
"github.com/matrix-org/dendrite/internal/fulltext"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
keyapi "github.com/matrix-org/dendrite/keyserver/api"
|
||||
|
|
@ -47,7 +48,12 @@ func AddPublicRoutes(
|
|||
|
||||
js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
|
||||
|
||||
syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database)
|
||||
fts, err := fulltext.New("./fts.bleve")
|
||||
if err != nil {
|
||||
logrus.WithError(err).Panicf("failed to create full text")
|
||||
}
|
||||
|
||||
syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database, fts)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Panicf("failed to connect to sync db")
|
||||
}
|
||||
|
|
@ -142,6 +148,6 @@ func AddPublicRoutes(
|
|||
|
||||
routing.Setup(
|
||||
base.PublicClientAPIMux, requestPool, syncDB, userAPI,
|
||||
rsAPI, cfg, base.Caches,
|
||||
rsAPI, cfg, base.Caches, fts,
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue