mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-21 13:53:09 -06:00
Code Issues Packages Projects Releases Wiki Activity

Initial work on roomserver NID caches

Browse source
This commit is contained in:
Neil Alexander 2020-08-25 11:14:14 +01:00
parent 733abe912b
commit 6b5be72503
No known key found for this signature in database
GPG key ID: A02A2019A2BB0944
8 changed files with 177 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
internal/caching/cache_roomservernids.go Normal file
View file

@ -0,0 +1,79 @@
package caching
import (
"github.com/matrix-org/dendrite/roomserver/types"
)
const (
RoomServerStateKeyNIDsCacheName = "roomserver_statekey_nids"
RoomServerStateKeyNIDsCacheMaxEntries = 1024
RoomServerStateKeyNIDsCacheMutable = false
RoomServerEventTypeNIDsCacheName = "roomserver_eventtype_nids"
RoomServerEventTypeNIDsCacheMaxEntries = 64
RoomServerEventTypeNIDsCacheMutable = false
RoomServerRoomNIDsCacheName = "roomserver_room_nids"
RoomServerRoomNIDsCacheMaxEntries = 1024
RoomServerRoomNIDsCacheMutable = false
)
type RoomServerCaches interface {
RoomServerNIDsCache
RoomVersionCache
}
// RoomServerNIDsCache contains the subset of functions needed for
// a roomserver NID cache.
type RoomServerNIDsCache interface {
GetRoomServerStateKeyNID(stateKey string) (types.EventStateKeyNID, bool)
StoreRoomServerStateKeyNID(stateKey string, nid types.EventStateKeyNID)
GetRoomServerEventTypeNID(eventType string) (types.EventTypeNID, bool)
StoreRoomServerEventTypeNID(eventType string, nid types.EventTypeNID)
GetRoomServerRoomNID(roomID string) (types.RoomNID, bool)
StoreRoomServerRoomNID(roomID string, nid types.RoomNID)
}
func (c Caches) GetRoomServerStateKeyNID(stateKey string) (types.EventStateKeyNID, bool) {
val, found := c.RoomServerStateKeyNIDs.Get(stateKey)
if found && val != nil {
if stateKeyNID, ok := val.(types.EventStateKeyNID); ok {
return stateKeyNID, true
}
}
return 0, false
}
func (c Caches) StoreRoomServerStateKeyNID(stateKey string, nid types.EventStateKeyNID) {
c.RoomVersions.Set(stateKey, nid)
}
func (c Caches) GetRoomServerEventTypeNID(eventType string) (types.EventTypeNID, bool) {
val, found := c.RoomServerEventTypeNIDs.Get(eventType)
if found && val != nil {
if eventTypeNID, ok := val.(types.EventTypeNID); ok {
return eventTypeNID, true
}
}
return 0, false
}
func (c Caches) StoreRoomServerEventTypeNID(eventType string, nid types.EventTypeNID) {
c.RoomVersions.Set(eventType, nid)
}
func (c Caches) GetRoomServerRoomNID(roomID string) (types.RoomNID, bool) {
val, found := c.RoomServerRoomNIDs.Get(roomID)
if found && val != nil {
if roomNID, ok := val.(types.RoomNID); ok {
return roomNID, true
}
}
return 0, false
}
func (c Caches) StoreRoomServerRoomNID(roomID string, nid types.RoomNID) {
c.RoomVersions.Set(roomID, nid)
}

7
internal/caching/caches.go
View file

@ -4,8 +4,11 @@ package caching
// different implementations as long as they satisfy the Cache
// interface.
type Caches struct {
RoomVersions Cache // implements RoomVersionCache
ServerKeys Cache // implements ServerKeyCache
RoomVersions Cache // RoomVersionCache
ServerKeys Cache // ServerKeyCache
RoomServerStateKeyNIDs Cache // RoomServerNIDsCache
RoomServerEventTypeNIDs Cache // RoomServerNIDsCache
RoomServerRoomNIDs Cache // RoomServerNIDsCache
}
// Cache is the interface that an implementation must satisfy.

34
internal/caching/impl_inmemorylru.go
View file

@ -27,9 +27,39 @@ func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) {
if err != nil {
return nil, err
}
roomServerStateKeyNIDs, err := NewInMemoryLRUCachePartition(
RoomServerStateKeyNIDsCacheName,
RoomServerStateKeyNIDsCacheMutable,
RoomServerStateKeyNIDsCacheMaxEntries,
enablePrometheus,
)
if err != nil {
return nil, err
}
roomServerEventTypeNIDs, err := NewInMemoryLRUCachePartition(
RoomServerEventTypeNIDsCacheName,
RoomServerEventTypeNIDsCacheMutable,
RoomServerEventTypeNIDsCacheMaxEntries,
enablePrometheus,
)
if err != nil {
return nil, err
}
roomServerRoomNIDs, err := NewInMemoryLRUCachePartition(
RoomServerRoomNIDsCacheName,
RoomServerRoomNIDsCacheMutable,
RoomServerRoomNIDsCacheMaxEntries,
enablePrometheus,
)
if err != nil {
return nil, err
}
return &Caches{
RoomVersions: roomVersions,
ServerKeys: serverKeys,
RoomVersions: roomVersions,
ServerKeys: serverKeys,
RoomServerStateKeyNIDs: roomServerStateKeyNIDs,
RoomServerEventTypeNIDs: roomServerEventTypeNIDs,
RoomServerRoomNIDs: roomServerRoomNIDs,
}, nil
}

2
roomserver/internal/api.go
View file

@ -16,7 +16,7 @@ type RoomserverInternalAPI struct {
DB storage.Database
Cfg *config.RoomServer
Producer sarama.SyncProducer
Cache caching.RoomVersionCache
Cache caching.RoomServerCaches
ServerName gomatrixserverlib.ServerName
KeyRing gomatrixserverlib.JSONVerifier
FedClient *gomatrixserverlib.FederationClient

4
roomserver/storage/postgres/storage.go
View file

@ -18,6 +18,7 @@ package postgres
import (
"database/sql"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/sqlutil"
@ -33,7 +34,7 @@ type Database struct {
// Open a postgres database.
// nolint: gocyclo
func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
var d Database
var db *sql.DB
var err error
@ -98,6 +99,7 @@ func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
}
d.Database = shared.Database{
DB: db,
Cache: cache,
Writer: sqlutil.NewDummyWriter(),
EventTypesTable: eventTypes,
EventStateKeysTable: eventStateKeys,

55
roomserver/storage/shared/storage.go
View file

@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
@ -27,6 +28,7 @@ const redactionsArePermanent = false
type Database struct {
DB *sql.DB
Cache caching.RoomServerCaches
Writer sqlutil.Writer
EventsTable tables.Events
EventJSONTable tables.EventJSON
@ -51,7 +53,25 @@ func (d *Database) SupportsConcurrentRoomInputs() bool {
func (d *Database) EventTypeNIDs(
ctx context.Context, eventTypes []string,
) (map[string]types.EventTypeNID, error) {
return d.EventTypesTable.BulkSelectEventTypeNID(ctx, eventTypes)
result := make(map[string]types.EventTypeNID)
remaining := []string{}
for _, eventType := range eventTypes {
if nid, ok := d.Cache.GetRoomServerEventTypeNID(eventType); ok {
result[eventType] = nid
} else {
remaining = append(remaining, eventType)
}
}
if len(remaining) > 0 {
nids, err := d.EventTypesTable.BulkSelectEventTypeNID(ctx, remaining)
if err != nil {
return nil, err
}
for eventType, nid := range nids {
result[eventType] = nid
}
}
return result, nil
}
func (d *Database) EventStateKeys(
@ -63,7 +83,25 @@ func (d *Database) EventStateKeys(
func (d *Database) EventStateKeyNIDs(
ctx context.Context, eventStateKeys []string,
) (map[string]types.EventStateKeyNID, error) {
return d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, eventStateKeys)
result := make(map[string]types.EventStateKeyNID)
remaining := []string{}
for _, eventStateKey := range eventStateKeys {
if nid, ok := d.Cache.GetRoomServerStateKeyNID(eventStateKey); ok {
result[eventStateKey] = nid
} else {
remaining = append(remaining, eventStateKey)
}
}
if len(remaining) > 0 {
nids, err := d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, remaining)
if err != nil {
return nil, err
}
for eventStateKey, nid := range nids {
result[eventStateKey] = nid
}
}
return result, nil
}
func (d *Database) StateEntriesForEventIDs(
@ -157,6 +195,9 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type
}
func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) {
if nid, ok := d.Cache.GetRoomServerRoomNID(roomID); ok {
return nid, nil
}
roomNID, err := d.RoomsTable.SelectRoomNID(ctx, nil, roomID)
if err == sql.ErrNoRows {
return 0, nil
@ -165,9 +206,13 @@ func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, e
}
func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) {
roomNID, err = d.RoomNID(ctx, roomID)
if err != nil {
return
if nid, ok := d.Cache.GetRoomServerRoomNID(roomID); ok {
roomNID = nid
} else {
roomNID, err = d.RoomNID(ctx, roomID)
if err != nil {
return
}
}
latestEvents, _, err := d.RoomsTable.SelectLatestEventNIDs(ctx, nil, roomNID)
if err != nil {

4
roomserver/storage/sqlite3/storage.go
View file

@ -19,6 +19,7 @@ import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/shared"
@ -46,7 +47,7 @@ type Database struct {
// Open a sqlite database.
// nolint: gocyclo
func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
var d Database
var err error
if d.db, err = sqlutil.Open(dbProperties); err != nil {
@ -120,6 +121,7 @@ func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
}
d.Database = shared.Database{
DB: d.db,
Cache: cache,
Writer: sqlutil.NewExclusiveWriter(),
EventsTable: d.events,
EventTypesTable: d.eventTypes,

7
roomserver/storage/storage.go
View file

@ -19,18 +19,19 @@ package storage
import (
"fmt"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/storage/postgres"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
)
// Open opens a database connection.
func Open(dbProperties *config.DatabaseOptions) (Database, error) {
func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.Open(dbProperties)
return sqlite3.Open(dbProperties, cache)
case dbProperties.ConnectionString.IsPostgres():
return postgres.Open(dbProperties)
return postgres.Open(dbProperties, cache)
default:
return nil, fmt.Errorf("unexpected database type")
}