Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/storeedutype

This commit is contained in:
Till Faelligen 2022-04-25 10:59:33 +02:00
commit 83867dfcbc
14 changed files with 108 additions and 43 deletions

View file

@ -6,11 +6,11 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ
- Efficient: A small memory footprint with better baseline performance than an out-of-the-box Synapse.
- Reliable: Implements the Matrix specification as written, using the
[same test suite](https://github.com/matrix-org/sytest) as Synapse as well as
a [brand new Go test suite](https://github.com/matrix-org/complement).
[same test suite](https://github.com/matrix-org/sytest) as Synapse as well as
a [brand new Go test suite](https://github.com/matrix-org/complement).
- Scalable: can run on multiple machines and eventually scale to massive homeserver deployments.
As of October 2020, Dendrite has now entered **beta** which means:
As of October 2020 (current [progress below](#progress)), Dendrite has now entered **beta** which means:
- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database.
- Dendrite has periodic semver releases. We intend to release new versions as we land significant features.
@ -21,7 +21,7 @@ This does not mean:
- Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
- All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular,
presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
- Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine.
Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
@ -78,7 +78,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co
Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
## Progress
## <a id="progress"></a> Progress
We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it

View file

@ -212,11 +212,12 @@ plv Users cannot set kick powerlevel higher than their own (2 subtests)
plv Users cannot set redact powerlevel higher than their own (2 subtests)
v1s Check that event streams started after a client joined a room work (SYT-1)
v1s Event stream catches up fully after many messages
xxx POST /rooms/:room_id/redact/:event_id as power user redacts message
xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message
xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message
xxx POST /redact disallows redaction of event in different room
xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message
xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message
xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as random user does not redact message
xxx PUT /redact disallows redaction of event in different room
xxx Redaction of a redaction redacts the redaction reason
xxx PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
v1s A departed room is still included in /initialSync (SPEC-216)
v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216)
rst Can get rooms/{roomId}/state for a departed room (SPEC-216)
@ -921,3 +922,18 @@ msc We can't peek into rooms with invited history_visibility
msc We can't peek into rooms with joined history_visibility
msc Local users can peek by room alias
msc Peeked rooms only turn up in the sync for the device who peeked them
ban 'ban' event respects room powerlevel (2 subtests)
inv Test that we can be reinvited to a room we created (11 subtests)
fiv Rejecting invite over federation doesn't break incremental /sync
pre Presence can be set from sync
fst /state returns M_NOT_FOUND for an outlier
fst /state_ids returns M_NOT_FOUND for an outlier
fst /state returns M_NOT_FOUND for a rejected message event
fst /state_ids returns M_NOT_FOUND for a rejected message event
fst /state returns M_NOT_FOUND for a rejected state event
fst /state_ids returns M_NOT_FOUND for a rejected state event
fst Room state after a rejected message event is the same as before
fst Room state after a rejected state event is the same as before
fpb Federation publicRoom Name/topic keys are correct
fed New federated private chats get full presence information (SYN-115) (10 subtests)
dvk Rejects invalid device keys

View file

@ -43,6 +43,9 @@ func GetUserDevices(
},
}
sigRes := &keyapi.QuerySignaturesResponse{}
for _, dev := range res.Devices {
sigReq.TargetIDs[userID] = append(sigReq.TargetIDs[userID], gomatrixserverlib.KeyID(dev.DeviceID))
}
keyAPI.QuerySignatures(req.Context(), sigReq, sigRes)
response := gomatrixserverlib.RespUserDevices{

View file

@ -455,10 +455,10 @@ func (a *KeyInternalAPI) processOtherSignatures(
func (a *KeyInternalAPI) crossSigningKeysFromDatabase(
ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse,
) {
for userID := range req.UserToDevices {
keys, err := a.DB.CrossSigningKeysForUser(ctx, userID)
for targetUserID := range req.UserToDevices {
keys, err := a.DB.CrossSigningKeysForUser(ctx, targetUserID)
if err != nil {
logrus.WithError(err).Errorf("Failed to get cross-signing keys for user %q", userID)
logrus.WithError(err).Errorf("Failed to get cross-signing keys for user %q", targetUserID)
continue
}
@ -469,9 +469,9 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase(
break
}
sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, userID, keyID)
sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, keyID)
if err != nil && err != sql.ErrNoRows {
logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", userID, keyID)
logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", targetUserID, keyID)
continue
}
@ -491,7 +491,7 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase(
case req.UserID != "" && originUserID == req.UserID:
// Include signatures that we created
appendSignature(originUserID, originKeyID, signature)
case originUserID == userID:
case originUserID == targetUserID:
// Include signatures that were created by the person whose key
// we are processing
appendSignature(originUserID, originKeyID, signature)
@ -501,13 +501,13 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase(
switch keyType {
case gomatrixserverlib.CrossSigningKeyPurposeMaster:
res.MasterKeys[userID] = key
res.MasterKeys[targetUserID] = key
case gomatrixserverlib.CrossSigningKeyPurposeSelfSigning:
res.SelfSigningKeys[userID] = key
res.SelfSigningKeys[targetUserID] = key
case gomatrixserverlib.CrossSigningKeyPurposeUserSigning:
res.UserSigningKeys[userID] = key
res.UserSigningKeys[targetUserID] = key
}
}
}
@ -546,7 +546,8 @@ func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySign
}
for _, targetKeyID := range forTargetUser {
sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, targetUserID, targetKeyID)
// Get own signatures only.
sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, targetUserID, targetUserID, targetKeyID)
if err != nil && err != sql.ErrNoRows {
res.Error = &api.KeyError{
Err: fmt.Sprintf("a.DB.CrossSigningSigsForTarget: %s", err),

View file

@ -313,9 +313,31 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
// Finally, append signatures that we know about
// TODO: This is horrible because we need to round-trip the signature from
// JSON, add the signatures and marshal it again, for some reason?
for userID, forUserID := range res.DeviceKeys {
for keyID, key := range forUserID {
sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, userID, gomatrixserverlib.KeyID(keyID))
for targetUserID, masterKey := range res.MasterKeys {
for targetKeyID := range masterKey.Keys {
sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID)
if err != nil {
logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
continue
}
if len(sigMap) == 0 {
continue
}
for sourceUserID, forSourceUser := range sigMap {
for sourceKeyID, sourceSig := range forSourceUser {
if _, ok := masterKey.Signatures[sourceUserID]; !ok {
masterKey.Signatures[sourceUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{}
}
masterKey.Signatures[sourceUserID][sourceKeyID] = sourceSig
}
}
}
}
for targetUserID, forUserID := range res.DeviceKeys {
for targetKeyID, key := range forUserID {
sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, gomatrixserverlib.KeyID(targetKeyID))
if err != nil {
logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
continue
@ -339,7 +361,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
}
}
if js, err := json.Marshal(deviceKey); err == nil {
res.DeviceKeys[userID][keyID] = js
res.DeviceKeys[targetUserID][targetKeyID] = js
}
}
}

View file

@ -81,7 +81,7 @@ type Database interface {
CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error)
CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error)
CrossSigningSigsForTarget(ctx context.Context, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error)
CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error)
StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error
StoreCrossSigningSigsForTarget(ctx context.Context, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error

View file

@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs (
const selectCrossSigningSigsForTargetSQL = "" +
"SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" +
" WHERE target_user_id = $1 AND target_key_id = $2"
" WHERE (origin_user_id = $1 OR origin_user_id = target_user_id) AND target_user_id = $2 AND target_key_id = $3"
const upsertCrossSigningSigsForTargetSQL = "" +
"INSERT INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" +
@ -72,9 +72,9 @@ func NewPostgresCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, erro
}
func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget(
ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID,
ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID,
) (r types.CrossSigningSigMap, err error) {
rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, targetUserID, targetKeyID)
rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, originUserID, targetUserID, targetKeyID)
if err != nil {
return nil, err
}

View file

@ -190,7 +190,7 @@ func (d *Database) CrossSigningKeysForUser(ctx context.Context, userID string) (
keyID: key,
},
}
sigMap, err := d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, userID, keyID)
sigMap, err := d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, userID, userID, keyID)
if err != nil {
continue
}
@ -219,8 +219,8 @@ func (d *Database) CrossSigningKeysDataForUser(ctx context.Context, userID strin
}
// CrossSigningSigsForTarget returns the signatures for a given user's key ID, if any.
func (d *Database) CrossSigningSigsForTarget(ctx context.Context, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) {
return d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, targetUserID, targetKeyID)
func (d *Database) CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) {
return d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, originUserID, targetUserID, targetKeyID)
}
// StoreCrossSigningKeysForUser stores the latest known cross-signing keys for a user.

View file

@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs (
const selectCrossSigningSigsForTargetSQL = "" +
"SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" +
" WHERE target_user_id = $1 AND target_key_id = $2"
" WHERE (origin_user_id = $1 OR origin_user_id = target_user_id) AND target_user_id = $2 AND target_key_id = $3"
const upsertCrossSigningSigsForTargetSQL = "" +
"INSERT OR REPLACE INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" +
@ -71,13 +71,13 @@ func NewSqliteCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error)
}
func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget(
ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID,
ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID,
) (r types.CrossSigningSigMap, err error) {
rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, targetUserID, targetKeyID)
rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, originUserID, targetUserID, targetKeyID)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningSigsForTargetStmt: rows.close() failed")
defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningSigsForOriginTargetStmt: rows.close() failed")
r = types.CrossSigningSigMap{}
for rows.Next() {
var userID string

View file

@ -64,7 +64,7 @@ type CrossSigningKeys interface {
}
type CrossSigningSigs interface {
SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error)
SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error)
UpsertCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error
DeleteCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) error
}

View file

@ -22,6 +22,7 @@ import (
"strconv"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/caching"
roomserver "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/storage"
userapi "github.com/matrix-org/dendrite/userapi/api"
@ -44,6 +45,7 @@ func Context(
rsAPI roomserver.RoomserverInternalAPI,
syncDB storage.Database,
roomID, eventID string,
lazyLoadCache *caching.LazyLoadCache,
) util.JSONResponse {
filter, err := parseRoomEventFilter(req)
if err != nil {
@ -129,7 +131,7 @@ func Context(
eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatAll)
eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatAll)
newState := applyLazyLoadMembers(filter, eventsAfterClient, eventsBeforeClient, state)
newState := applyLazyLoadMembers(device, filter, eventsAfterClient, eventsBeforeClient, state, lazyLoadCache)
response := ContextRespsonse{
Event: gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll),
@ -148,15 +150,25 @@ func Context(
}
}
func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent, state []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
func applyLazyLoadMembers(
device *userapi.Device,
filter *gomatrixserverlib.RoomEventFilter,
eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent,
state []*gomatrixserverlib.HeaderedEvent,
lazyLoadCache *caching.LazyLoadCache,
) []*gomatrixserverlib.HeaderedEvent {
if filter == nil || !filter.LazyLoadMembers {
return state
}
allEvents := append(eventsBefore, eventsAfter...)
x := make(map[string]bool)
x := make(map[string]struct{})
// get members who actually send an event
for _, e := range allEvents {
x[e.Sender] = true
// Don't add membership events the client should already know about
if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, e.RoomID, e.Sender); cached {
continue
}
x[e.Sender] = struct{}{}
}
newState := []*gomatrixserverlib.HeaderedEvent{}
@ -166,8 +178,9 @@ func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter
newState = append(newState, event)
} else {
// did the user send an event?
if x[event.Sender()] {
if _, ok := x[event.Sender()]; ok {
membershipEvents = append(membershipEvents, event)
lazyLoadCache.StoreLazyLoadedUser(device, event.RoomID(), event.Sender(), event.EventID())
}
}
}

View file

@ -21,6 +21,7 @@ import (
"sort"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/syncapi/storage"
@ -64,6 +65,7 @@ func OnIncomingMessagesRequest(
rsAPI api.RoomserverInternalAPI,
cfg *config.SyncAPI,
srp *sync.RequestPool,
lazyLoadCache *caching.LazyLoadCache,
) util.JSONResponse {
var err error
@ -200,6 +202,10 @@ func OnIncomingMessagesRequest(
if filter.LazyLoadMembers {
membershipToUser := make(map[string]*gomatrixserverlib.HeaderedEvent)
for _, evt := range clientEvents {
// Don't add membership events the client should already know about
if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, roomID, evt.Sender); cached {
continue
}
membership, err := db.GetStateEvent(req.Context(), roomID, gomatrixserverlib.MRoomMember, evt.Sender)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("failed to get membership event for user")
@ -207,10 +213,11 @@ func OnIncomingMessagesRequest(
}
if membership != nil {
membershipToUser[evt.Sender] = membership
lazyLoadCache.StoreLazyLoadedUser(device, roomID, evt.Sender, membership.EventID())
}
}
for _, evt := range membershipToUser {
state = append(state, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatAll))
state = append(state, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatSync))
}
}

View file

@ -18,6 +18,7 @@ import (
"net/http"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
@ -38,6 +39,7 @@ func Setup(
userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient,
rsAPI api.RoomserverInternalAPI,
cfg *config.SyncAPI,
lazyLoadCache *caching.LazyLoadCache,
) {
v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
@ -51,7 +53,7 @@ func Setup(
if err != nil {
return util.ErrorResponse(err)
}
return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, federation, rsAPI, cfg, srp)
return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, federation, rsAPI, cfg, srp, lazyLoadCache)
})).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/user/{userId}/filter",
@ -89,6 +91,7 @@ func Setup(
req, device,
rsAPI, syncDB,
vars["roomId"], vars["eventId"],
lazyLoadCache,
)
}),
).Methods(http.MethodGet, http.MethodOptions)

View file

@ -148,5 +148,5 @@ func AddPublicRoutes(
logrus.WithError(err).Panicf("failed to start presence consumer")
}
routing.Setup(router, requestPool, syncDB, userAPI, federation, rsAPI, cfg)
routing.Setup(router, requestPool, syncDB, userAPI, federation, rsAPI, cfg, lazyLoadCache)
}