mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-16 11:23:11 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add an LRU cache around the key DB

Browse source 
We actually need this for P2P because otherwise we can *segfault*
with things like: "runtime: unexpected return pc for runtime.handleEvent"
where the event is a `syscall/js` event, caused by spamming sql.js
caused by "Checking event signatures for 14 events of room state" which
hammers the key DB repeatedly in quick succession.

Using a cache fixes this, though the underlying cause is probably a bug
in the version of Go I'm on (1.13.7)
This commit is contained in:
Kegan Dougal 2020-03-04 18:00:12 +00:00
parent 2f0622226d
commit cfe97cd874
3 changed files with 33 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
common/keydb/sqlite3/server_key_table.go
View file

@ -20,8 +20,10 @@ import (
"database/sql"
"strings"
lru "github.com/hashicorp/golang-lru"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
const serverKeysSchema = `
@ -64,10 +66,16 @@ type serverKeyStatements struct {
db *sql.DB
bulkSelectServerKeysStmt *sql.Stmt
upsertServerKeysStmt *sql.Stmt
cache *lru.Cache // nameAndKeyID => gomatrixserverlib.PublicKeyLookupResult
}
func (s *serverKeyStatements) prepare(db *sql.DB) (err error) {
s.db = db
s.cache, err = lru.New(64)
if err != nil {
return
}
_, err = db.Exec(serverKeysSchema)
if err != nil {
return
@ -89,6 +97,22 @@ func (s *serverKeyStatements) bulkSelectServerKeys(
for request := range requests {
nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request))
}
// If we can satisfy all of the requests from the cache, do so. TODO: Allow partial matches with merges.
cacheResults := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
for request := range requests {
r, ok := s.cache.Get(nameAndKeyID(request))
if !ok {
break
}
cacheResult := r.(gomatrixserverlib.PublicKeyLookupResult)
cacheResults[request] = cacheResult
}
if len(cacheResults) == len(requests) {
util.GetLogger(ctx).Infof("KeyDB cache hit for %d keys", len(cacheResults))
return cacheResults, nil
}
query := strings.Replace(bulkSelectServerKeysSQL, "($1)", common.QueryVariadic(len(nameAndKeyIDs)), 1)
iKeyIDs := make([]interface{}, len(nameAndKeyIDs))
@ -134,6 +158,7 @@ func (s *serverKeyStatements) upsertServerKeys(
request gomatrixserverlib.PublicKeyLookupRequest,
key gomatrixserverlib.PublicKeyLookupResult,
) error {
s.cache.Add(nameAndKeyID(request), key)
_, err := s.upsertServerKeysStmt.ExecContext(
ctx,
string(request.ServerName),

3
go.mod
View file

@ -6,10 +6,11 @@ replace github.com/prometheus/client_golang => ./prometheus
require (
github.com/gorilla/mux v1.7.3
github.com/hashicorp/golang-lru v0.5.4
github.com/lib/pq v1.2.0
github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5
github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4
github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074
github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e
github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1

6
go.sum
View file

@ -26,6 +26,8 @@ github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@ -43,6 +45,10 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 h1:oF
github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ=
github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4=
github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI=
github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04=
github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI=
github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4=