From e0703522932ab4bf41b23d5229c7a4540b59d830 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 5 Oct 2022 11:14:33 +0200 Subject: [PATCH 01/12] Side effect import bleve analyzer languages (#2763) ... to actually allow different languages. Fixes #2761 Binary size increases by ~1MB. --- internal/fulltext/bleve.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go index da8932f5c..7187861dd 100644 --- a/internal/fulltext/bleve.go +++ b/internal/fulltext/bleve.go @@ -21,6 +21,29 @@ import ( "strings" "github.com/blevesearch/bleve/v2" + + // side effect imports to allow all possible languages + _ "github.com/blevesearch/bleve/v2/analysis/lang/ar" + _ "github.com/blevesearch/bleve/v2/analysis/lang/cjk" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ckb" + _ "github.com/blevesearch/bleve/v2/analysis/lang/da" + _ "github.com/blevesearch/bleve/v2/analysis/lang/de" + _ "github.com/blevesearch/bleve/v2/analysis/lang/en" + _ "github.com/blevesearch/bleve/v2/analysis/lang/es" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fa" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fi" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fr" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hi" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hr" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hu" + _ "github.com/blevesearch/bleve/v2/analysis/lang/it" + _ "github.com/blevesearch/bleve/v2/analysis/lang/nl" + _ "github.com/blevesearch/bleve/v2/analysis/lang/no" + _ "github.com/blevesearch/bleve/v2/analysis/lang/pt" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ro" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ru" + _ "github.com/blevesearch/bleve/v2/analysis/lang/sv" + _ "github.com/blevesearch/bleve/v2/analysis/lang/tr" "github.com/blevesearch/bleve/v2/mapping" "github.com/matrix-org/gomatrixserverlib" From ebd137cf6b2fbd767625dc5289b0bef6d1e51971 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 11:07:17 +0100 Subject: [PATCH 02/12] Check PostgreSQL connection count (#2760) This PR queries PostgreSQL for the `max_connections` and `superuser_reserved_connections` settings and then ensures that Dendrite's `max_open_conns` doesn't exceed the allowed value. This is a really common source of configuration problems and can either result in blocking queries or deadlocks, so it seems reasonable that we complain as loudly as possible when it happens. --- internal/sqlutil/sqlutil.go | 39 +++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/internal/sqlutil/sqlutil.go b/internal/sqlutil/sqlutil.go index 0cdae6d30..789bceeac 100644 --- a/internal/sqlutil/sqlutil.go +++ b/internal/sqlutil/sqlutil.go @@ -2,6 +2,7 @@ package sqlutil import ( "database/sql" + "flag" "fmt" "regexp" @@ -9,6 +10,8 @@ import ( "github.com/sirupsen/logrus" ) +var skipSanityChecks = flag.Bool("skip-db-sanity", false, "Ignore sanity checks on the database connections (NOT RECOMMENDED!)") + // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 @@ -37,15 +40,39 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) return nil, err } if driverName != "sqlite3" { - logrus.WithFields(logrus.Fields{ - "MaxOpenConns": dbProperties.MaxOpenConns(), - "MaxIdleConns": dbProperties.MaxIdleConns(), - "ConnMaxLifetime": dbProperties.ConnMaxLifetime(), - "dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), - }).Debug("Setting DB connection limits") + logger := logrus.WithFields(logrus.Fields{ + "max_open_conns": dbProperties.MaxOpenConns(), + "max_idle_conns": dbProperties.MaxIdleConns(), + "conn_max_lifetime": dbProperties.ConnMaxLifetime(), + "data_source_name": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), + }) + logger.Debug("Setting DB connection limits") db.SetMaxOpenConns(dbProperties.MaxOpenConns()) db.SetMaxIdleConns(dbProperties.MaxIdleConns()) db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime()) + + if !*skipSanityChecks { + if dbProperties.MaxOpenConns() == 0 { + logrus.Warnf("WARNING: Configuring 'max_open_conns' to be unlimited is not recommended. This can result in bad performance or deadlocks.") + } + + switch driverName { + case "postgres": + // Perform a quick sanity check if possible that we aren't trying to use more database + // connections than PostgreSQL is willing to give us. + var max, reserved int + if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='max_connections';").Scan(&max); err != nil { + return nil, fmt.Errorf("failed to find maximum connections: %w", err) + } + if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='superuser_reserved_connections';").Scan(&reserved); err != nil { + return nil, fmt.Errorf("failed to find reserved connections: %w", err) + } + if configured, allowed := dbProperties.MaxOpenConns(), max-reserved; configured > allowed { + logrus.Errorf("ERROR: The configured 'max_open_conns' is greater than the %d non-superuser connections that PostgreSQL is configured to allow. This can result in bad performance or deadlocks. Please pay close attention to your configured database connection counts. If you REALLY know what you are doing and want to override this error, pass the --skip-db-sanity option to Dendrite.", allowed) + return nil, fmt.Errorf("database sanity checks failed") + } + } + } } return db, nil } From 8c0c3441d88a612ca8e9ba4f83c6ff29ca73f5d0 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 5 Oct 2022 12:12:42 +0200 Subject: [PATCH 03/12] Add `RoomEventType` nats.Header to avoid unneeded unmarshalling (#2765) --- appservice/consumers/roomserver.go | 5 +++++ federationapi/consumers/roomserver.go | 7 +++++++ roomserver/producers/roomevent.go | 13 ++++++------- setup/jetstream/streams.go | 7 ++++--- test/testrig/jetstream.go | 9 ++++----- userapi/consumers/roomserver.go | 7 ++++--- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index d44f32b38..ac68f4bd4 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -101,6 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage( log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs)) events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs)) for _, msg := range msgs { + // Only handle events we care about + receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType)) + if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInviteEvent { + continue + } // Parse out the event JSON var output api.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go index 349b50b05..a42733628 100644 --- a/federationapi/consumers/roomserver.go +++ b/federationapi/consumers/roomserver.go @@ -79,6 +79,13 @@ func (s *OutputRoomEventConsumer) Start() error { // realises that it cannot update the room state using the deltas. func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called + receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType)) + + // Only handle events we care about + if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInboundPeek { + return true + } + // Parse out the event JSON var output api.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go index 987e6c942..9c4521986 100644 --- a/roomserver/producers/roomevent.go +++ b/roomserver/producers/roomevent.go @@ -17,12 +17,13 @@ package producers import ( "encoding/json" - "github.com/matrix-org/dendrite/roomserver/acls" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/jetstream" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" + + "github.com/matrix-org/dendrite/roomserver/acls" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/jetstream" ) var keyContentFields = map[string]string{ @@ -40,10 +41,8 @@ type RoomEventProducer struct { func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.OutputEvent) error { var err error for _, update := range updates { - msg := &nats.Msg{ - Subject: r.Topic, - Header: nats.Header{}, - } + msg := nats.NewMsg(r.Topic) + msg.Header.Set(jetstream.RoomEventType, string(update.Type)) msg.Header.Set(jetstream.RoomID, roomID) msg.Data, err = json.Marshal(update) if err != nil { diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index ee9810dae..590f0cbd9 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -9,9 +9,10 @@ import ( ) const ( - UserID = "user_id" - RoomID = "room_id" - EventID = "event_id" + UserID = "user_id" + RoomID = "room_id" + EventID = "event_id" + RoomEventType = "output_room_event_type" ) var ( diff --git a/test/testrig/jetstream.go b/test/testrig/jetstream.go index 74cf95062..b880eea43 100644 --- a/test/testrig/jetstream.go +++ b/test/testrig/jetstream.go @@ -4,10 +4,11 @@ import ( "encoding/json" "testing" + "github.com/nats-io/nats.go" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" - "github.com/nats-io/nats.go" ) func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Msg) { @@ -21,10 +22,8 @@ func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Ms func NewOutputEventMsg(t *testing.T, base *base.BaseDendrite, roomID string, update api.OutputEvent) *nats.Msg { t.Helper() - msg := &nats.Msg{ - Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), - Header: nats.Header{}, - } + msg := nats.NewMsg(base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) + msg.Header.Set(jetstream.RoomEventType, string(update.Type)) msg.Header.Set(jetstream.RoomID, roomID) var err error msg.Data, err = json.Marshal(update) diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 952de98f7..a12876946 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -72,15 +72,16 @@ func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called + // Only handle events we care about + if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent { + return true + } var output rsapi.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { // If the message was invalid, log it and move on to the next message in the stream log.WithError(err).Errorf("roomserver output log: message parse failure") return true } - if output.Type != rsapi.OutputTypeNewRoomEvent { - return true - } event := output.NewRoomEvent.Event if event == nil { log.Errorf("userapi consumer: expected event") From 6f602bb0969fac6d455de1088bf3980f77a4017f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 11:16:05 +0100 Subject: [PATCH 04/12] Demote `Failed to query device keys for some users` warning to `level=debug` Many of these warnings are due to dead servers and are quite annoying when they fill up the logs. --- keyserver/internal/device_list_update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index fcfcd092d..8b02f3d6c 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -424,7 +424,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam "succeeded": successCount, "failed": len(userIDs) - successCount, "wait_time": waitTime, - }).Warn("Failed to query device keys for some users") + }).Debug("Failed to query device keys for some users") } return waitTime, !allUsersSucceeded } From c85bc3434fc930423e9e27c8bca9b31b5ad7a441 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 12:47:53 +0100 Subject: [PATCH 05/12] Optimise `QuerySharedUsers` so that we can only work on local users (#2766) Otherwise the sync API key change consumer wastes a lot of time trying to wake up the notifiers for non-local users. --- roomserver/api/query.go | 1 + roomserver/internal/query/query.go | 2 +- roomserver/storage/interface.go | 2 +- .../storage/postgres/membership_table.go | 17 +++++++++----- roomserver/storage/shared/storage.go | 4 ++-- .../storage/sqlite3/membership_table.go | 23 +++++++++++-------- roomserver/storage/tables/interface.go | 2 +- .../storage/tables/membership_table_test.go | 2 +- syncapi/consumers/keychange.go | 6 +++-- 9 files changed, 36 insertions(+), 23 deletions(-) diff --git a/roomserver/api/query.go b/roomserver/api/query.go index aa7dc4735..d63c24785 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -278,6 +278,7 @@ type QuerySharedUsersRequest struct { OtherUserIDs []string ExcludeRoomIDs []string IncludeRoomIDs []string + LocalOnly bool } type QuerySharedUsersResponse struct { diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index b41a92e94..ee8e1cfe7 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -799,7 +799,7 @@ func (r *Queryer) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUser } roomIDs = roomIDs[:j] - users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs) + users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs, req.LocalOnly) if err != nil { return err } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 43e8da7bb..11e175f55 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -157,7 +157,7 @@ type Database interface { // If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned. GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) // JoinedUsersSetInRooms returns how many times each of the given users appears across the given rooms. - JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error) + JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error) // GetLocalServerInRoom returns true if we think we're in a given room or false otherwise. GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) // GetServerInRoom returns true if we think a server is in a given room or false otherwise. diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index bd3fd5592..0150534e1 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -68,14 +68,18 @@ CREATE TABLE IF NOT EXISTS roomserver_membership ( var selectJoinedUsersSetForRoomsAndUserSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid = ANY($1) AND target_nid = ANY($2) AND" + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false)" + + " AND room_nid = ANY($2) AND target_nid = ANY($3)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" var selectJoinedUsersSetForRoomsSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid = ANY($1) AND" + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false) " + + " AND room_nid = ANY($2)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" // Insert a row in to membership table so that it can be locked by the @@ -334,6 +338,7 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms( ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, + localOnly bool, ) (map[types.EventStateKeyNID]int, error) { var ( rows *sql.Rows @@ -342,9 +347,9 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms( stmt := sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsStmt) if len(userNIDs) > 0 { stmt = sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsAndUserStmt) - rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs), pq.Array(userNIDs)) + rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs), pq.Array(userNIDs)) } else { - rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs)) + rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs)) } if err != nil { diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 593abbea1..d83a1ff74 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -1280,7 +1280,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu } // JoinedUsersSetInRooms returns a map of how many times the given users appear in the specified rooms. -func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error) { +func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error) { roomNIDs, err := d.RoomsTable.BulkSelectRoomNIDs(ctx, nil, roomIDs) if err != nil { return nil, err @@ -1295,7 +1295,7 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs [ userNIDs = append(userNIDs, nid) nidToUserID[nid] = id } - userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs) + userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs, localOnly) if err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index f3303eb0e..cd149f0ed 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -44,14 +44,18 @@ const membershipSchema = ` var selectJoinedUsersSetForRoomsAndUserSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid IN ($1) AND target_nid IN ($2) AND" + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false)" + + " AND room_nid IN ($2) AND target_nid IN ($3)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" var selectJoinedUsersSetForRoomsSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid IN ($1) AND " + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false)" + + " AND room_nid IN ($2)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" // Insert a row in to membership table so that it can be locked by the @@ -305,8 +309,9 @@ func (s *membershipStatements) SelectRoomsWithMembership( return roomNIDs, nil } -func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error) { - params := make([]interface{}, 0, len(roomNIDs)+len(userNIDs)) +func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) { + params := make([]interface{}, 0, 1+len(roomNIDs)+len(userNIDs)) + params = append(params, localOnly) for _, v := range roomNIDs { params = append(params, v) } @@ -314,10 +319,10 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, params = append(params, v) } - query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1) + query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1) if len(userNIDs) > 0 { - query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1) - query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)), 1) + query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1) + query = strings.Replace(query, "($3)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)+1), 1) } var rows *sql.Rows var err error diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 68d30f994..d7bcc95ab 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -137,7 +137,7 @@ type Membership interface { UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID, forgotten bool) (bool, error) SelectRoomsWithMembership(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, membershipState MembershipState) ([]types.RoomNID, error) // SelectJoinedUsersSetForRooms returns how many times each of the given users appears across the given rooms. - SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error) + SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) SelectKnownUsers(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, searchString string, limit int) ([]string, error) UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error SelectLocalServerInRoom(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID) (bool, error) diff --git a/roomserver/storage/tables/membership_table_test.go b/roomserver/storage/tables/membership_table_test.go index f789ef4ac..c9541d9d2 100644 --- a/roomserver/storage/tables/membership_table_test.go +++ b/roomserver/storage/tables/membership_table_test.go @@ -79,7 +79,7 @@ func TestMembershipTable(t *testing.T) { assert.NoError(t, err) assert.True(t, inRoom) - userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs) + userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs, false) assert.NoError(t, err) assert.Equal(t, 1, len(userJoinedToRooms)) diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go index dc7d9e207..96ebba7ef 100644 --- a/syncapi/consumers/keychange.go +++ b/syncapi/consumers/keychange.go @@ -111,7 +111,8 @@ func (s *OutputKeyChangeEventConsumer) onDeviceKeyMessage(m api.DeviceMessage, d // work out who we need to notify about the new key var queryRes roomserverAPI.QuerySharedUsersResponse err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{ - UserID: output.UserID, + UserID: output.UserID, + LocalOnly: true, }, &queryRes) if err != nil { logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") @@ -135,7 +136,8 @@ func (s *OutputKeyChangeEventConsumer) onCrossSigningMessage(m api.DeviceMessage // work out who we need to notify about the new key var queryRes roomserverAPI.QuerySharedUsersResponse err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{ - UserID: output.UserID, + UserID: output.UserID, + LocalOnly: true, }, &queryRes) if err != nil { logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") From 0f777d421c56e386b4c483233277f2b96c4da3a0 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 5 Oct 2022 14:47:13 +0200 Subject: [PATCH 06/12] Remove empty fields from `/sync` response (#2755) First attempt at removing empty fields from `/sync` responses. Needs https://github.com/matrix-org/sytest/pull/1298 to keep Sytest happy. Co-authored-by: Neil Alexander --- syncapi/internal/keychange_test.go | 33 +++-- syncapi/streams/stream_accountdata.go | 6 +- syncapi/streams/stream_invite.go | 4 +- syncapi/streams/stream_pdu.go | 10 +- syncapi/streams/stream_receipt.go | 5 +- syncapi/streams/stream_typing.go | 9 +- syncapi/types/types.go | 201 ++++++++++++++++++-------- 7 files changed, 178 insertions(+), 90 deletions(-) diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 3b9c8221c..53f3e5a40 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -170,9 +170,12 @@ func joinResponseWithRooms(syncResponse *types.Response, userID string, roomIDs Content: []byte(`{"membership":"join"}`), }, } - - jr := syncResponse.Rooms.Join[roomID] - jr.State.Events = roomEvents + jr, ok := syncResponse.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() + } + jr.Timeline = &types.Timeline{} + jr.State = &types.ClientEvents{Events: roomEvents} syncResponse.Rooms.Join[roomID] = jr } return syncResponse @@ -191,8 +194,11 @@ func leaveResponseWithRooms(syncResponse *types.Response, userID string, roomIDs }, } - lr := syncResponse.Rooms.Leave[roomID] - lr.Timeline.Events = roomEvents + lr, ok := syncResponse.Rooms.Leave[roomID] + if !ok { + lr = types.NewLeaveResponse() + } + lr.Timeline = &types.Timeline{Events: roomEvents} syncResponse.Rooms.Leave[roomID] = lr } return syncResponse @@ -328,9 +334,13 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) { }, } - jr := syncResponse.Rooms.Join[roomID] - jr.State.Events = roomStateEvents - jr.Timeline.Events = roomTimelineEvents + jr, ok := syncResponse.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() + } + + jr.State = &types.ClientEvents{Events: roomStateEvents} + jr.Timeline = &types.Timeline{Events: roomTimelineEvents} syncResponse.Rooms.Join[roomID] = jr rsAPI := &mockRoomserverAPI{ @@ -442,8 +452,11 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) { }, } - lr := syncResponse.Rooms.Leave[roomID] - lr.Timeline.Events = roomEvents + lr, ok := syncResponse.Rooms.Leave[roomID] + if !ok { + lr = types.NewLeaveResponse() + } + lr.Timeline = &types.Timeline{Events: roomEvents} syncResponse.Rooms.Leave[roomID] = lr rsAPI := &mockRoomserverAPI{ diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 3f2f7d134..3593a6563 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -90,9 +90,9 @@ func (p *AccountDataStreamProvider) IncrementalSync( } } else { if roomData, ok := dataRes.RoomAccountData[roomID][dataType]; ok { - joinData := *types.NewJoinResponse() - if existing, ok := req.Response.Rooms.Join[roomID]; ok { - joinData = existing + joinData, ok := req.Response.Rooms.Join[roomID] + if !ok { + joinData = types.NewJoinResponse() } joinData.AccountData.Events = append( joinData.AccountData.Events, diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 17b3b8434..7875ffa35 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -65,7 +65,7 @@ func (p *InviteStreamProvider) IncrementalSync( continue } ir := types.NewInviteResponse(inviteEvent) - req.Response.Rooms.Invite[roomID] = *ir + req.Response.Rooms.Invite[roomID] = ir } // When doing an initial sync, we don't want to add retired invites, as this @@ -87,7 +87,7 @@ func (p *InviteStreamProvider) IncrementalSync( Type: "m.room.member", Content: gomatrixserverlib.RawJSON(`{"membership":"leave"}`), }) - req.Response.Rooms.Leave[roomID] = *lr + req.Response.Rooms.Leave[roomID] = lr } } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index d252265ff..613ac434f 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -106,7 +106,7 @@ func (p *PDUStreamProvider) CompleteSync( } continue } - req.Response.Rooms.Join[roomID] = *jr + req.Response.Rooms.Join[roomID] = jr req.Rooms[roomID] = gomatrixserverlib.Join } @@ -129,7 +129,7 @@ func (p *PDUStreamProvider) CompleteSync( } continue } - req.Response.Rooms.Peek[peek.RoomID] = *jr + req.Response.Rooms.Peek[peek.RoomID] = jr } } @@ -320,7 +320,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // didn't "remove" events, return that the response is limited. jr.Timeline.Limited = limited && len(events) == len(recentEvents) jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Join[delta.RoomID] = *jr + res.Rooms.Join[delta.RoomID] = jr case gomatrixserverlib.Peek: jr := types.NewJoinResponse() @@ -329,7 +329,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = limited jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Peek[delta.RoomID] = *jr + res.Rooms.Peek[delta.RoomID] = jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -342,7 +342,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // didn't "remove" events, return that the response is limited. lr.Timeline.Limited = limited && len(events) == len(recentEvents) lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Leave[delta.RoomID] = *lr + res.Rooms.Leave[delta.RoomID] = lr } return latestPosition, nil diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 8818a5533..76927cc36 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -4,9 +4,10 @@ import ( "context" "encoding/json" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) type ReceiptStreamProvider struct { @@ -76,7 +77,7 @@ func (p *ReceiptStreamProvider) IncrementalSync( continue } - jr := *types.NewJoinResponse() + jr := types.NewJoinResponse() if existing, ok := req.Response.Rooms.Join[roomID]; ok { jr = existing } diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go index a6f7c7a06..84c199b39 100644 --- a/syncapi/streams/stream_typing.go +++ b/syncapi/streams/stream_typing.go @@ -4,10 +4,11 @@ import ( "context" "encoding/json" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) type TypingStreamProvider struct { @@ -35,9 +36,9 @@ func (p *TypingStreamProvider) IncrementalSync( continue } - jr := *types.NewJoinResponse() - if existing, ok := req.Response.Rooms.Join[roomID]; ok { - jr = existing + jr, ok := req.Response.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() } if users, updated := p.EDUCache.GetTypingUsersIfUpdatedAfter( diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 3b85db4a4..b6d340f93 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -327,29 +327,57 @@ type PrevEventRef struct { PrevSender string `json:"prev_sender"` } +type DeviceLists struct { + Changed []string `json:"changed,omitempty"` + Left []string `json:"left,omitempty"` +} + +type RoomsResponse struct { + Join map[string]*JoinResponse `json:"join,omitempty"` + Peek map[string]*JoinResponse `json:"peek,omitempty"` + Invite map[string]*InviteResponse `json:"invite,omitempty"` + Leave map[string]*LeaveResponse `json:"leave,omitempty"` +} + +type ToDeviceResponse struct { + Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"` +} + // Response represents a /sync API response. See https://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-client-r0-sync type Response struct { - NextBatch StreamingToken `json:"next_batch"` - AccountData struct { - Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` - } `json:"account_data,omitempty"` - Presence struct { - Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` - } `json:"presence,omitempty"` - Rooms struct { - Join map[string]JoinResponse `json:"join,omitempty"` - Peek map[string]JoinResponse `json:"peek,omitempty"` - Invite map[string]InviteResponse `json:"invite,omitempty"` - Leave map[string]LeaveResponse `json:"leave,omitempty"` - } `json:"rooms,omitempty"` - ToDevice struct { - Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"` - } `json:"to_device,omitempty"` - DeviceLists struct { - Changed []string `json:"changed,omitempty"` - Left []string `json:"left,omitempty"` - } `json:"device_lists,omitempty"` - DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"` + NextBatch StreamingToken `json:"next_batch"` + AccountData *ClientEvents `json:"account_data,omitempty"` + Presence *ClientEvents `json:"presence,omitempty"` + Rooms *RoomsResponse `json:"rooms,omitempty"` + ToDevice *ToDeviceResponse `json:"to_device,omitempty"` + DeviceLists *DeviceLists `json:"device_lists,omitempty"` + DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"` +} + +func (r Response) MarshalJSON() ([]byte, error) { + type alias Response + a := alias(r) + if r.AccountData != nil && len(r.AccountData.Events) == 0 { + a.AccountData = nil + } + if r.Presence != nil && len(r.Presence.Events) == 0 { + a.Presence = nil + } + if r.DeviceLists != nil { + if len(r.DeviceLists.Left) == 0 && len(r.DeviceLists.Changed) == 0 { + a.DeviceLists = nil + } + } + if r.Rooms != nil { + if len(r.Rooms.Join) == 0 && len(r.Rooms.Peek) == 0 && + len(r.Rooms.Invite) == 0 && len(r.Rooms.Leave) == 0 { + a.Rooms = nil + } + } + if r.ToDevice != nil && len(r.ToDevice.Events) == 0 { + a.ToDevice = nil + } + return json.Marshal(a) } func (r *Response) HasUpdates() bool { @@ -370,18 +398,21 @@ func NewResponse() *Response { res := Response{} // Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section, // so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors. - res.Rooms.Join = map[string]JoinResponse{} - res.Rooms.Peek = map[string]JoinResponse{} - res.Rooms.Invite = map[string]InviteResponse{} - res.Rooms.Leave = map[string]LeaveResponse{} + res.Rooms = &RoomsResponse{ + Join: map[string]*JoinResponse{}, + Peek: map[string]*JoinResponse{}, + Invite: map[string]*InviteResponse{}, + Leave: map[string]*LeaveResponse{}, + } // Also pre-intialise empty slices or else we'll insert 'null' instead of '[]' for the value. // TODO: We really shouldn't have to do all this to coerce encoding/json to Do The Right Thing. We should // really be using our own Marshal/Unmarshal implementations otherwise this may prove to be a CPU bottleneck. // This also applies to NewJoinResponse, NewInviteResponse and NewLeaveResponse. - res.AccountData.Events = []gomatrixserverlib.ClientEvent{} - res.Presence.Events = []gomatrixserverlib.ClientEvent{} - res.ToDevice.Events = []gomatrixserverlib.SendToDeviceEvent{} + res.AccountData = &ClientEvents{} + res.Presence = &ClientEvents{} + res.DeviceLists = &DeviceLists{} + res.ToDevice = &ToDeviceResponse{} res.DeviceListsOTKCount = map[string]int{} return &res @@ -403,38 +434,73 @@ type UnreadNotifications struct { NotificationCount int `json:"notification_count"` } +type ClientEvents struct { + Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` +} + +type Timeline struct { + Events []gomatrixserverlib.ClientEvent `json:"events"` + Limited bool `json:"limited"` + PrevBatch *TopologyToken `json:"prev_batch,omitempty"` +} + +type Summary struct { + Heroes []string `json:"m.heroes,omitempty"` + JoinedMemberCount *int `json:"m.joined_member_count,omitempty"` + InvitedMemberCount *int `json:"m.invited_member_count,omitempty"` +} + // JoinResponse represents a /sync response for a room which is under the 'join' or 'peek' key. type JoinResponse struct { - Summary struct { - Heroes []string `json:"m.heroes,omitempty"` - JoinedMemberCount *int `json:"m.joined_member_count,omitempty"` - InvitedMemberCount *int `json:"m.invited_member_count,omitempty"` - } `json:"summary"` - State struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"state"` - Timeline struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - Limited bool `json:"limited"` - PrevBatch *TopologyToken `json:"prev_batch,omitempty"` - } `json:"timeline"` - Ephemeral struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"ephemeral"` - AccountData struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"account_data"` + Summary *Summary `json:"summary,omitempty"` + State *ClientEvents `json:"state,omitempty"` + Timeline *Timeline `json:"timeline,omitempty"` + Ephemeral *ClientEvents `json:"ephemeral,omitempty"` + AccountData *ClientEvents `json:"account_data,omitempty"` *UnreadNotifications `json:"unread_notifications,omitempty"` } +func (jr JoinResponse) MarshalJSON() ([]byte, error) { + type alias JoinResponse + a := alias(jr) + if jr.State != nil && len(jr.State.Events) == 0 { + a.State = nil + } + if jr.Ephemeral != nil && len(jr.Ephemeral.Events) == 0 { + a.Ephemeral = nil + } + if jr.AccountData != nil && len(jr.AccountData.Events) == 0 { + a.AccountData = nil + } + if jr.Timeline != nil && len(jr.Timeline.Events) == 0 { + a.Timeline = nil + } + if jr.Summary != nil { + var nilPtr int + joinedEmpty := jr.Summary.JoinedMemberCount == nil || jr.Summary.JoinedMemberCount == &nilPtr + invitedEmpty := jr.Summary.InvitedMemberCount == nil || jr.Summary.InvitedMemberCount == &nilPtr + if joinedEmpty && invitedEmpty && len(jr.Summary.Heroes) == 0 { + a.Summary = nil + } + + } + if jr.UnreadNotifications != nil && + jr.UnreadNotifications.NotificationCount == 0 && jr.UnreadNotifications.HighlightCount == 0 { + a.UnreadNotifications = nil + } + return json.Marshal(a) +} + // NewJoinResponse creates an empty response with initialised arrays. func NewJoinResponse() *JoinResponse { - res := JoinResponse{} - res.State.Events = []gomatrixserverlib.ClientEvent{} - res.Timeline.Events = []gomatrixserverlib.ClientEvent{} - res.Ephemeral.Events = []gomatrixserverlib.ClientEvent{} - res.AccountData.Events = []gomatrixserverlib.ClientEvent{} - return &res + return &JoinResponse{ + Summary: &Summary{}, + State: &ClientEvents{}, + Timeline: &Timeline{}, + Ephemeral: &ClientEvents{}, + AccountData: &ClientEvents{}, + UnreadNotifications: &UnreadNotifications{}, + } } // InviteResponse represents a /sync response for a room which is under the 'invite' key. @@ -469,21 +535,28 @@ func NewInviteResponse(event *gomatrixserverlib.HeaderedEvent) *InviteResponse { // LeaveResponse represents a /sync response for a room which is under the 'leave' key. type LeaveResponse struct { - State struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"state"` - Timeline struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - Limited bool `json:"limited"` - PrevBatch *TopologyToken `json:"prev_batch,omitempty"` - } `json:"timeline"` + State *ClientEvents `json:"state,omitempty"` + Timeline *Timeline `json:"timeline,omitempty"` +} + +func (lr LeaveResponse) MarshalJSON() ([]byte, error) { + type alias LeaveResponse + a := alias(lr) + if lr.State != nil && len(lr.State.Events) == 0 { + a.State = nil + } + if lr.Timeline != nil && len(lr.Timeline.Events) == 0 { + a.Timeline = nil + } + return json.Marshal(a) } // NewLeaveResponse creates an empty response with initialised arrays. func NewLeaveResponse() *LeaveResponse { - res := LeaveResponse{} - res.State.Events = []gomatrixserverlib.ClientEvent{} - res.Timeline.Events = []gomatrixserverlib.ClientEvent{} + res := LeaveResponse{ + State: &ClientEvents{}, + Timeline: &Timeline{}, + } return &res } From 9ba3103f8839243ea8529bf1f563afded3e7b591 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 14:54:19 +0100 Subject: [PATCH 07/12] Document database connection limits --- docs/installation/11_optimisation.md | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/installation/11_optimisation.md b/docs/installation/11_optimisation.md index f2f67c947..686ec2eb9 100644 --- a/docs/installation/11_optimisation.md +++ b/docs/installation/11_optimisation.md @@ -11,6 +11,41 @@ permalink: /installation/start/optimisation Now that you have Dendrite running, the following tweaks will improve the reliability and performance of your installation. +## PostgreSQL connection limit + +A PostgreSQL database engine is configured to allow only a certain number of connections. +This is typically controlled by the `max_connections` and `superuser_reserved_connections` +configuration items in `postgresql.conf`. Once these limits are violated, **PostgreSQL will +immediately stop accepting new connections** until some of the existing connections are closed. +This is a common source of misconfiguration and requires particular care. + +If your PostgreSQL `max_connections` is set to `100` and `superuser_reserved_connections` is +set to `3` then you have an effective connection limit of 97 database connections. It is +therefore important to ensure that Dendrite doesn't violate that limit, otherwise database +queries will unexpectedly fail and this will cause problems both within Dendrite and for users. + +If you are also running other software that uses the same PostgreSQL database engine, then you +must also take into account that some connections will be already used by your other software +and therefore will not be available to Dendrite. Check the configuration of any other software +using the same database engine for their configured connection limits and adjust your calculations +accordingly. + +Dendrite has a `max_open_conns` configuration item in each `database` block to control how many +connections it will open to the database. + +**If you are using the `global` database pool** then you only need to configure the +`max_open_conns` setting once in the `global` section. + +**If you are defining a `database` config per component** then you will need to ensure that +the **sum total** of all configured `max_open_conns` to a given database server do not exceed +the connection limit. If you configure a total that adds up to more connections than are available +then this will cause database queries to fail. + +You may wish to raise the `max_connections` limit on your PostgreSQL server to accommodate +additional connections, in which case you should also update the `max_open_conns` in your +Dendrite configuration accordingly. However be aware that this is only advisable on particularly +powerful servers that can handle the concurrent load of additional queries running at one time. + ## File descriptor limit Most platforms have a limit on how many file descriptors a single process can open. All From e53dcb25a9a0258b48fd4f7afddd2c3c54d55b7a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Oct 2022 10:07:13 +0100 Subject: [PATCH 08/12] Tweak logging for federated room joins --- federationapi/internal/perform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 84702f4ce..4cdd3a5eb 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -259,7 +259,7 @@ func (r *FederationInternalAPI) performJoinUsingServer( if err != nil { return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err) } - logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts") + logrus.WithField("room", roomID).Infof("Joined federated room with %d hosts", len(joinedHosts)) if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil { return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err) } From 3f82bceb70050c1233b7de6d87ffa5510596d145 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Oct 2022 10:51:06 +0100 Subject: [PATCH 09/12] Don't try to talk to ourselves when finding missing events --- roomserver/internal/input/input_events.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 01fd62010..d1b6bc73e 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -173,12 +173,15 @@ func (r *Inputer) processRoomEvent( for _, server := range serverRes.ServerNames { servers[server] = struct{}{} } + // Don't try to talk to ourselves. + delete(servers, r.Cfg.Matrix.ServerName) + // Now build up the list of servers. serverRes.ServerNames = serverRes.ServerNames[:0] - if input.Origin != "" { + if input.Origin != "" && input.Origin != r.Cfg.Matrix.ServerName { serverRes.ServerNames = append(serverRes.ServerNames, input.Origin) delete(servers, input.Origin) } - if senderDomain != input.Origin { + if senderDomain != input.Origin && senderDomain != r.Cfg.Matrix.ServerName { serverRes.ServerNames = append(serverRes.ServerNames, senderDomain) delete(servers, senderDomain) } From ec5d1d681d1362f5746c5cb45e93829d6a68aa4d Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 6 Oct 2022 12:30:24 +0200 Subject: [PATCH 10/12] Always return `one_time_key_counts` on `/keys/upload` (#2769) The OTK count is [required](https://spec.matrix.org/v1.4/client-server-api/#post_matrixclientv3keysupload) in responses to `/keys/upload`, so return those. --- clientapi/routing/keys.go | 4 ++-- keyserver/internal/internal.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go index b7a76b47e..5c3681382 100644 --- a/clientapi/routing/keys.go +++ b/clientapi/routing/keys.go @@ -19,11 +19,12 @@ import ( "net/http" "time" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/keyserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/util" ) type uploadKeysRequest struct { @@ -77,7 +78,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devi } } keyCount := make(map[string]int) - // we only return key counts when the client uploads OTKs if len(uploadRes.OneTimeKeyCounts) > 0 { keyCount = uploadRes.OneTimeKeyCounts[0].KeyCount } diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 017c29e84..a0280dff4 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -70,6 +70,11 @@ func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.Perform if len(req.OneTimeKeys) > 0 { a.uploadOneTimeKeys(ctx, req, res) } + otks, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) + if err != nil { + return err + } + res.OneTimeKeyCounts = []api.OneTimeKeysCount{*otks} return nil } From d605d928bce87b381e2f64b8835619d803e67a54 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Oct 2022 11:56:00 +0100 Subject: [PATCH 11/12] Allow specifying old signing keys with the public key and key ID only (#2770) If the private key file is lost, it's often possible to retrieve the public key from another server elsewhere, so we should make it possible to configure it in that way. --- dendrite-sample.monolith.yaml | 7 ++++- dendrite-sample.polylith.yaml | 7 ++++- federationapi/routing/keys.go | 2 +- setup/config/config.go | 48 +++++++++++++++++++++++------------ setup/config/config_global.go | 7 +++-- 5 files changed, 50 insertions(+), 21 deletions(-) diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index f0fa386d1..eadb74a2a 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -18,12 +18,17 @@ global: private_key: matrix_key.pem # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) - # to old signing private keys that were formerly in use on this domain. These + # to old signing keys that were formerly in use on this domain name. These # keys will not be used for federation request or event signing, but will be # provided to any other homeserver that asks when trying to verify old events. old_private_keys: + # If the old private key file is available: # - private_key: old_matrix_key.pem # expired_at: 1601024554498 + # If only the public key (in base64 format) and key ID are known: + # - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM= + # key_id: ed25519:mykeyid + # expired_at: 1601024554498 # How long a remote server can cache our server signing key before requesting it # again. Increasing this number will reduce the number of requests made by other diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index 0ae4cc8fb..aa7e0cc38 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -18,12 +18,17 @@ global: private_key: matrix_key.pem # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) - # to old signing private keys that were formerly in use on this domain. These + # to old signing keys that were formerly in use on this domain name. These # keys will not be used for federation request or event signing, but will be # provided to any other homeserver that asks when trying to verify old events. old_private_keys: + # If the old private key file is available: # - private_key: old_matrix_key.pem # expired_at: 1601024554498 + # If only the public key (in base64 format) and key ID are known: + # - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM= + # key_id: ed25519:mykeyid + # expired_at: 1601024554498 # How long a remote server can cache our server signing key before requesting it # again. Increasing this number will reduce the number of requests made by other diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index b03d4c1d6..8931830f3 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -160,7 +160,7 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver for _, oldVerifyKey := range cfg.Matrix.OldVerifyKeys { keys.OldVerifyKeys[oldVerifyKey.KeyID] = gomatrixserverlib.OldVerifyKey{ VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PrivateKey.Public().(ed25519.PublicKey)), + Key: oldVerifyKey.PublicKey, }, ExpiredTS: oldVerifyKey.ExpiredAt, } diff --git a/setup/config/config.go b/setup/config/config.go index 5a618d671..e99852ec9 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -231,24 +231,40 @@ func loadConfig( return nil, err } - for i, oldPrivateKey := range c.Global.OldVerifyKeys { - var oldPrivateKeyData []byte + for _, key := range c.Global.OldVerifyKeys { + switch { + case key.PrivateKeyPath != "": + var oldPrivateKeyData []byte + oldPrivateKeyPath := absPath(basePath, key.PrivateKeyPath) + oldPrivateKeyData, err = readFile(oldPrivateKeyPath) + if err != nil { + return nil, fmt.Errorf("failed to read %q: %w", oldPrivateKeyPath, err) + } - oldPrivateKeyPath := absPath(basePath, oldPrivateKey.PrivateKeyPath) - oldPrivateKeyData, err = readFile(oldPrivateKeyPath) - if err != nil { - return nil, err + // NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are + // a number of private keys out there with non-compatible symbols in them due + // to lack of validation in Synapse, we won't enforce that for old verify keys. + keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false) + if perr != nil { + return nil, fmt.Errorf("failed to parse %q: %w", oldPrivateKeyPath, perr) + } + + key.KeyID = keyID + key.PrivateKey = privateKey + key.PublicKey = gomatrixserverlib.Base64Bytes(privateKey.Public().(ed25519.PublicKey)) + + case key.KeyID == "": + return nil, fmt.Errorf("'key_id' must be specified if 'public_key' is specified") + + case len(key.PublicKey) == ed25519.PublicKeySize: + continue + + case len(key.PublicKey) > 0: + return nil, fmt.Errorf("the supplied 'public_key' is the wrong length") + + default: + return nil, fmt.Errorf("either specify a 'private_key' path or supply both 'public_key' and 'key_id'") } - - // NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are - // a number of private keys out there with non-compatible symbols in them due - // to lack of validation in Synapse, we won't enforce that for old verify keys. - keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false) - if perr != nil { - return nil, perr - } - - c.Global.OldVerifyKeys[i].KeyID, c.Global.OldVerifyKeys[i].PrivateKey = keyID, privateKey } c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath)) diff --git a/setup/config/config_global.go b/setup/config/config_global.go index acc608dd7..2efae0d5a 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -27,7 +27,7 @@ type Global struct { // Information about old private keys that used to be used to sign requests and // events on this domain. They will not be used but will be advertised to other // servers that ask for them to help verify old events. - OldVerifyKeys []OldVerifyKeys `yaml:"old_private_keys"` + OldVerifyKeys []*OldVerifyKeys `yaml:"old_private_keys"` // How long a remote server can cache our server key for before requesting it again. // Increasing this number will reduce the number of requests made by remote servers @@ -127,8 +127,11 @@ type OldVerifyKeys struct { // The private key itself. PrivateKey ed25519.PrivateKey `yaml:"-"` + // The public key, in case only that part is known. + PublicKey gomatrixserverlib.Base64Bytes `yaml:"public_key"` + // The key ID of the private key. - KeyID gomatrixserverlib.KeyID `yaml:"-"` + KeyID gomatrixserverlib.KeyID `yaml:"key_id"` // When the private key was designed as "expired", as a UNIX timestamp // in millisecond precision. From 8c5b16678475112f5a1c13d616931283421aee05 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 6 Oct 2022 12:57:13 +0200 Subject: [PATCH 12/12] Use the stream positions of the notifier (#2768) Use the stream positions of the notifier, which might have advanced since setting it at the beginning of the loop. This possibly helps in reducing roundtrips to the SyncAPI, just because we didn't fetch the latest data. Also fixes a minor oversight in the receipts stream. --- syncapi/streams/stream_receipt.go | 6 +++--- syncapi/sync/requestpool.go | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 76927cc36..bba911022 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -77,9 +77,9 @@ func (p *ReceiptStreamProvider) IncrementalSync( continue } - jr := types.NewJoinResponse() - if existing, ok := req.Response.Rooms.Join[roomID]; ok { - jr = existing + jr, ok := req.Response.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() } ev := gomatrixserverlib.ClientEvent{ diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index a71d32ab8..29d92b293 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -407,7 +407,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.PDUStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.PDUPosition, currentPos.PDUPosition, + syncReq.Since.PDUPosition, rp.Notifier.CurrentPosition().PDUPosition, ) }, ), @@ -416,7 +416,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.TypingStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.TypingPosition, currentPos.TypingPosition, + syncReq.Since.TypingPosition, rp.Notifier.CurrentPosition().TypingPosition, ) }, ), @@ -425,7 +425,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.ReceiptStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, + syncReq.Since.ReceiptPosition, rp.Notifier.CurrentPosition().ReceiptPosition, ) }, ), @@ -434,7 +434,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.InviteStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.InvitePosition, currentPos.InvitePosition, + syncReq.Since.InvitePosition, rp.Notifier.CurrentPosition().InvitePosition, ) }, ), @@ -443,7 +443,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.SendToDeviceStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, + syncReq.Since.SendToDevicePosition, rp.Notifier.CurrentPosition().SendToDevicePosition, ) }, ), @@ -452,7 +452,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.AccountDataStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, + syncReq.Since.AccountDataPosition, rp.Notifier.CurrentPosition().AccountDataPosition, ) }, ), @@ -461,7 +461,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.NotificationDataStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, + syncReq.Since.NotificationDataPosition, rp.Notifier.CurrentPosition().NotificationDataPosition, ) }, ), @@ -470,7 +470,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.DeviceListStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, + syncReq.Since.DeviceListPosition, rp.Notifier.CurrentPosition().DeviceListPosition, ) }, ), @@ -479,7 +479,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.PresenceStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.PresencePosition, currentPos.PresencePosition, + syncReq.Since.PresencePosition, rp.Notifier.CurrentPosition().PresencePosition, ) }, ),